{"site":"robotstxt.org","rank":10606,"mapped":"2026-11-20","status":200,"home":"https://www.robotstxt.org","title":"The Web Robots Pages","language":"en","readable":{"words":188,"withoutJavaScript":true},"sections":[{"name":"The /robots.txt","url":"https://www.robotstxt.org/robotstxt.html"},{"name":"<META> tags","url":"https://www.robotstxt.org/meta.html"},{"name":"Frequently Asked Questions","url":"https://www.robotstxt.org/faq.html"},{"name":"Mailing list","url":"https://www.robotstxt.org/mailinglist.html"},{"name":"Other Sites","url":"https://www.robotstxt.org/other.html"},{"name":"About robotstxt.org","url":"https://www.robotstxt.org/about.html"},{"name":"/robots.txt checker","url":"https://www.robotstxt.org/checker.html"},{"name":"Robots Database","url":"https://www.robotstxt.org/db.html"},{"name":"IP lookup","url":"https://www.robotstxt.org/iplookup.html"},{"name":"How do I stop robots visiting my site?","url":"https://www.robotstxt.org/faq/prevent.html"},{"name":"Privacy and cookies policy","url":"https://www.robotstxt.org/privacy.html"},{"name":"Contact us","url":"https://www.robotstxt.org/contact.html"}],"robots":"robots.txt read","agents":[{"name":"ChatGPT-User","op":"OpenAI","role":"live","allowed":true},{"name":"OAI-SearchBot","op":"OpenAI","role":"search","allowed":true},{"name":"GPTBot","op":"OpenAI","role":"training","allowed":true},{"name":"Claude-User","op":"Anthropic","role":"live","allowed":true},{"name":"Claude-SearchBot","op":"Anthropic","role":"search","allowed":true},{"name":"ClaudeBot","op":"Anthropic","role":"training","allowed":true},{"name":"Perplexity-User","op":"Perplexity","role":"live","allowed":true},{"name":"PerplexityBot","op":"Perplexity","role":"search","allowed":true},{"name":"Googlebot","op":"Google","role":"search","allowed":true},{"name":"Google-Extended","op":"Google","role":"training","allowed":true},{"name":"Bingbot","op":"Microsoft","role":"search","allowed":true},{"name":"Applebot-Extended","op":"Apple","role":"training","allowed":true},{"name":"Meta-ExternalAgent","op":"Meta","role":"training","allowed":true},{"name":"Amazonbot","op":"Amazon","role":"search","allowed":true},{"name":"DuckAssistBot","op":"DuckDuckGo","role":"live","allowed":true},{"name":"MistralAI-User","op":"Mistral","role":"live","allowed":true},{"name":"CCBot","op":"Common Crawl","role":"training","allowed":true},{"name":"Bytespider","op":"ByteDance","role":"training","allowed":true}],"score":50,"grade":"C","recipes":{"contact":"https://www.robotstxt.org/contact.html","account":"https://www.robotstxt.org/faq/prevent.html"},"use":{"read":"https://mareliberum.dev/r/https://www.robotstxt.org/","check":"https://mareliberum.dev/check/robotstxt.org"},"source":"read from what the site publishes (robots.txt, the home page and the files it names) by MareLiberum-Atlas; open data, CC BY 4.0"}