{"site":"commoncrawl.org","rank":19435,"mapped":"2026-11-20","status":200,"home":"https://commoncrawl.org","title":"Common Crawl - Open Repository of Web Crawl Data","description":"We build and maintain an open repository of web crawl data that can be accessed and analyzed by anyone.","language":"en","search":{"template":"https://commoncrawl.org/search?query={q}","from":"the site's search form","example":"https://commoncrawl.org/search?query=example"},"readable":{"words":411,"withoutJavaScript":true},"sections":[{"name":"Overview","url":"https://commoncrawl.org/overview"},{"name":"CDXJ Index","url":"https://commoncrawl.org/cdxj-index"},{"name":"URL Index","url":"https://commoncrawl.org/url-index"},{"name":"Web Graphs","url":"https://commoncrawl.org/web-graphs"},{"name":"Latest Crawl","url":"https://commoncrawl.org/latest-crawl"},{"name":"Errata","url":"https://commoncrawl.org/errata"},{"name":"Get Started","url":"https://commoncrawl.org/get-started"},{"name":"Blog","url":"https://commoncrawl.org/blog"},{"name":"CCBot","url":"https://commoncrawl.org/ccbot"},{"name":"Opt-Out Ledger","url":"https://commoncrawl.org/blog/common-crawl-foundation-opt-out-registry"},{"name":"FAQ","url":"https://commoncrawl.org/faq"},{"name":"Research Papers","url":"https://commoncrawl.org/research-papers"},{"name":"Collaborators","url":"https://commoncrawl.org/collaborators"},{"name":"About","url":"https://commoncrawl.org/about"}],"sitemaps":["https://commoncrawl.org/sitemap.xml"],"robots":"robots.txt read","agents":[{"name":"ChatGPT-User","op":"OpenAI","role":"live","allowed":true},{"name":"OAI-SearchBot","op":"OpenAI","role":"search","allowed":true},{"name":"GPTBot","op":"OpenAI","role":"training","allowed":true},{"name":"Claude-User","op":"Anthropic","role":"live","allowed":true},{"name":"Claude-SearchBot","op":"Anthropic","role":"search","allowed":true},{"name":"ClaudeBot","op":"Anthropic","role":"training","allowed":true},{"name":"Perplexity-User","op":"Perplexity","role":"live","allowed":true},{"name":"PerplexityBot","op":"Perplexity","role":"search","allowed":true},{"name":"Googlebot","op":"Google","role":"search","allowed":true},{"name":"Google-Extended","op":"Google","role":"training","allowed":true},{"name":"Bingbot","op":"Microsoft","role":"search","allowed":true},{"name":"Applebot-Extended","op":"Apple","role":"training","allowed":true},{"name":"Meta-ExternalAgent","op":"Meta","role":"training","allowed":true},{"name":"Amazonbot","op":"Amazon","role":"search","allowed":true},{"name":"DuckAssistBot","op":"DuckDuckGo","role":"live","allowed":true},{"name":"MistralAI-User","op":"Mistral","role":"live","allowed":true},{"name":"CCBot","op":"Common Crawl","role":"training","allowed":true},{"name":"Bytespider","op":"ByteDance","role":"training","allowed":true}],"score":65,"grade":"C","use":{"read":"https://mareliberum.dev/r/https://commoncrawl.org/","check":"https://mareliberum.dev/check/commoncrawl.org","searchThroughReader":"https://mareliberum.dev/r/https://commoncrawl.org/search?query=<words>"},"source":"read from what the site publishes (robots.txt, the home page and the files it names) by MareLiberum-Atlas; open data, CC BY 4.0"}