{"site":"mit.edu","rank":101,"mapped":"2026-11-20","status":200,"home":"https://web.mit.edu","title":"MIT - Massachusetts Institute of Technology","description":"The mission of MIT is to advance knowledge and educate students in science, technology and other areas of scholarship that will best serve the nation and the world in the 21st century.","language":"en","readable":{"words":363,"withoutJavaScript":true},"sections":[{"name":"Education","url":"https://web.mit.edu/education"},{"name":"Research","url":"https://web.mit.edu/research"},{"name":"Innovation","url":"https://web.mit.edu/innovation"},{"name":"Admissions + Aid","url":"https://web.mit.edu/admissions-aid"},{"name":"Campus Life","url":"https://web.mit.edu/campus-life"},{"name":"Alumni","url":"https://web.mit.edu/alumni"},{"name":"About MIT","url":"https://web.mit.edu/about"},{"name":"Give","url":"https://web.mit.edu/building-a-better-world"}],"robots":"robots.txt read","agents":[{"name":"ChatGPT-User","op":"OpenAI","role":"live","allowed":true},{"name":"OAI-SearchBot","op":"OpenAI","role":"search","allowed":true},{"name":"GPTBot","op":"OpenAI","role":"training","allowed":true},{"name":"Claude-User","op":"Anthropic","role":"live","allowed":true},{"name":"Claude-SearchBot","op":"Anthropic","role":"search","allowed":true},{"name":"ClaudeBot","op":"Anthropic","role":"training","allowed":true},{"name":"Perplexity-User","op":"Perplexity","role":"live","allowed":true},{"name":"PerplexityBot","op":"Perplexity","role":"search","allowed":true},{"name":"Googlebot","op":"Google","role":"search","allowed":true},{"name":"Google-Extended","op":"Google","role":"training","allowed":true},{"name":"Bingbot","op":"Microsoft","role":"search","allowed":true},{"name":"Applebot-Extended","op":"Apple","role":"training","allowed":true},{"name":"Meta-ExternalAgent","op":"Meta","role":"training","allowed":true},{"name":"Amazonbot","op":"Amazon","role":"search","allowed":true},{"name":"DuckAssistBot","op":"DuckDuckGo","role":"live","allowed":true},{"name":"MistralAI-User","op":"Mistral","role":"live","allowed":true},{"name":"CCBot","op":"Common Crawl","role":"training","allowed":true},{"name":"Bytespider","op":"ByteDance","role":"training","allowed":true}],"score":60,"grade":"C","use":{"read":"https://mareliberum.dev/r/https://web.mit.edu/","check":"https://mareliberum.dev/check/mit.edu"},"source":"read from what the site publishes (robots.txt, the home page and the files it names) by MareLiberum-Atlas; open data, CC BY 4.0"}