{ "title": "RadiusRank Public AI Crawler Access Benchmark", "version": 1, "snapshot_date": "2026-08-11", "generated_at": "2026-08-11T16:45:35.143Z", "methodology_url": "https://www.radiusrank.com/research/ai-crawler-access-benchmark#methodology", "user_agent": "RadiusRankResearch/1.0 (+https://www.radiusrank.com/research/ai-crawler-access-benchmark)", "scope": "Root-path access inferred from public robots.txt files. Root access does not prove that every URL is crawlable.", "agents": [ { "id": "GPTBot", "key": "gptbot", "purpose": "training" }, { "id": "OAI-SearchBot", "key": "oai_searchbot", "purpose": "retrieval" }, { "id": "ChatGPT-User", "key": "chatgpt_user", "purpose": "user initiated" }, { "id": "ClaudeBot", "key": "claudebot", "purpose": "training" }, { "id": "Claude-SearchBot", "key": "claude_searchbot", "purpose": "retrieval" }, { "id": "Claude-User", "key": "claude_user", "purpose": "user initiated" }, { "id": "PerplexityBot", "key": "perplexitybot", "purpose": "retrieval" }, { "id": "Perplexity-User", "key": "perplexity_user", "purpose": "user initiated" }, { "id": "Google-Extended", "key": "google_extended", "purpose": "AI model control" }, { "id": "CCBot", "key": "ccbot", "purpose": "training corpus" }, { "id": "Applebot-Extended", "key": "applebot_extended", "purpose": "AI model control" } ], "summary": { "sampled_sites": 50, "available_robots_files": 47, "missing_robots_files": 0, "unavailable_checks": 3, "sites_with_sitemap_declaration": 41, "sites_with_explicit_ai_agent_rules": 11, "agents": { "gptbot": { "id": "GPTBot", "purpose": "training", "allowed_at_root": 46, "blocked_at_root": 1, "unavailable": 3, "explicitly_named": 8 }, "oai_searchbot": { "id": "OAI-SearchBot", "purpose": "retrieval", "allowed_at_root": 47, "blocked_at_root": 0, "unavailable": 3, "explicitly_named": 7 }, "chatgpt_user": { "id": "ChatGPT-User", "purpose": "user initiated", "allowed_at_root": 47, "blocked_at_root": 0, "unavailable": 3, "explicitly_named": 7 }, "claudebot": { "id": "ClaudeBot", "purpose": "training", "allowed_at_root": 46, "blocked_at_root": 1, "unavailable": 3, "explicitly_named": 7 }, "claude_searchbot": { "id": "Claude-SearchBot", "purpose": "retrieval", "allowed_at_root": 47, "blocked_at_root": 0, "unavailable": 3, "explicitly_named": 3 }, "claude_user": { "id": "Claude-User", "purpose": "user initiated", "allowed_at_root": 47, "blocked_at_root": 0, "unavailable": 3, "explicitly_named": 4 }, "perplexitybot": { "id": "PerplexityBot", "purpose": "retrieval", "allowed_at_root": 47, "blocked_at_root": 0, "unavailable": 3, "explicitly_named": 8 }, "perplexity_user": { "id": "Perplexity-User", "purpose": "user initiated", "allowed_at_root": 47, "blocked_at_root": 0, "unavailable": 3, "explicitly_named": 3 }, "google_extended": { "id": "Google-Extended", "purpose": "AI model control", "allowed_at_root": 45, "blocked_at_root": 2, "unavailable": 3, "explicitly_named": 6 }, "ccbot": { "id": "CCBot", "purpose": "training corpus", "allowed_at_root": 46, "blocked_at_root": 1, "unavailable": 3, "explicitly_named": 5 }, "applebot_extended": { "id": "Applebot-Extended", "purpose": "AI model control", "allowed_at_root": 46, "blocked_at_root": 1, "unavailable": 3, "explicitly_named": 6 } } }, "rows": [ { "brand": "OpenAI", "category": "AI lab or platform", "domain": "openai.com", "robots_url": "https://openai.com/robots.txt", "final_url": "https://openai.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 98, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "c53100563b9d289efeebb6f826a20c566578e1337e3096e298eb9da5ecc2e498", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Anthropic", "category": "AI lab or platform", "domain": "anthropic.com", "robots_url": "https://anthropic.com/robots.txt", "final_url": "https://www.anthropic.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 71, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "b8bb03567cf862a9024bd464bd4bd7afaf054b1029de79727b2ba94711480ec7", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Perplexity", "category": "AI lab or platform", "domain": "perplexity.ai", "robots_url": "https://perplexity.ai/robots.txt", "final_url": "https://www.perplexity.ai/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 480, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "a4b48ab940c33c9ef1b6c66576b01c2ccaacdeef5315a93e09eaf67b05bb01dc", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Cohere", "category": "AI lab or platform", "domain": "cohere.com", "robots_url": "https://cohere.com/robots.txt", "final_url": "https://cohere.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 278, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "813b56545d0154d518a0461e7906a0c87ff90dbaa29f27035af7e50f4a233cdf", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Mistral AI", "category": "AI lab or platform", "domain": "mistral.ai", "robots_url": "https://mistral.ai/robots.txt", "final_url": "https://mistral.ai/robots.txt", "http_status": 200, "content_type": "text/plain; charset=UTF-8", "file_bytes": 70, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "e5543d29282e692d17eb88ba69bbaf6076d6707902fd907e240d10e9312e7bfe", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Hugging Face", "category": "AI lab or platform", "domain": "huggingface.co", "robots_url": "https://huggingface.co/robots.txt", "final_url": "https://huggingface.co/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 68, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "045c12bc2b5f1820752a5c149d2c96030ecfb8f32c42dd034cead715be277193", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Together AI", "category": "AI lab or platform", "domain": "together.ai", "robots_url": "https://together.ai/robots.txt", "final_url": "https://www.together.ai/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 357, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "Google-Extended" ], "source_sha256": "f677ebbce49ce67fa3d468a89d40482d22410a607e3e54b8da74f45bd08f541a", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "blocked", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Replicate", "category": "AI lab or platform", "domain": "replicate.com", "robots_url": "https://replicate.com/robots.txt", "final_url": "https://replicate.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 120, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "fb728bd6c9304b5bdc6c802287c1c99494f8e8afb26c8f23104f53a3d1992969", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Groq", "category": "AI lab or platform", "domain": "groq.com", "robots_url": "https://groq.com/robots.txt", "final_url": "https://groq.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 90, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "eabce30b93978e50065cc1b9595b34f5843145313a208dbc07a1834c1e8cdc07", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Ahrefs", "category": "SEO platform", "domain": "ahrefs.com", "robots_url": "https://ahrefs.com/robots.txt", "final_url": "https://ahrefs.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf8", "file_bytes": 1038, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": "1fe9430b59f5156e51739f55dadb22df4bb73c052556eba7d14ffd92d10189fc", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Semrush", "category": "SEO platform", "domain": "semrush.com", "robots_url": "https://semrush.com/robots.txt", "final_url": "https://www.semrush.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 3581, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "d783244848b9a1f74b52f580e2204950ebcb4641e9cdfc95cb85dbe02a5320de", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Similarweb", "category": "SEO platform", "domain": "similarweb.com", "robots_url": "https://similarweb.com/robots.txt", "final_url": "https://similarweb.com/robots.txt", "http_status": null, "content_type": null, "file_bytes": null, "robots_available": false, "access_basis": "unavailable", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": null, "error": "fetch failed", "gptbot": "unavailable", "oai_searchbot": "unavailable", "chatgpt_user": "unavailable", "claudebot": "unavailable", "claude_searchbot": "unavailable", "claude_user": "unavailable", "perplexitybot": "unavailable", "perplexity_user": "unavailable", "google_extended": "unavailable", "ccbot": "unavailable", "applebot_extended": "unavailable" }, { "brand": "SE Ranking", "category": "SEO platform", "domain": "seranking.com", "robots_url": "https://seranking.com/robots.txt", "final_url": "https://seranking.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 2385, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Google-Extended", "Applebot-Extended" ], "source_sha256": "6a836ad82057fcba5de55c693fa99a9da106757621649936310f6403f0f014a2", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Moz", "category": "SEO platform", "domain": "moz.com", "robots_url": "https://moz.com/robots.txt", "final_url": "https://moz.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 1210, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot" ], "source_sha256": "07b090a959e9e282d776b371dbfe1a2130352f70b7fd5d0adaa5fbc19b962ea1", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "SISTRIX", "category": "SEO platform", "domain": "sistrix.com", "robots_url": "https://sistrix.com/robots.txt", "final_url": "https://www.sistrix.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 440, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "c84b501ed329f78544d4768fcbafac92443da6244b98664a26e2f7dd2b9a399a", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "BrightEdge", "category": "SEO platform", "domain": "brightedge.com", "robots_url": "https://brightedge.com/robots.txt", "final_url": "https://www.brightedge.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 2512, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "51c4292d739ece2b88c0817e3e8b6d37652bde5a0438c3630a0ad294a9c61a8f", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Majestic", "category": "SEO platform", "domain": "majestic.com", "robots_url": "https://majestic.com/robots.txt", "final_url": "https://majestic.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 517, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": "25473ba42a9c458a56400540553439f4eb930335cba5deeb5baaee195c801ec5", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "SpyFu", "category": "SEO platform", "domain": "spyfu.com", "robots_url": "https://spyfu.com/robots.txt", "final_url": "https://www.spyfu.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 22246, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "ChatGPT-User", "Claude-User", "Perplexity-User" ], "source_sha256": "0225c1aaf63811de1d529d862ac5264ed60aa7893aa8e6d4fafcc40cb82a1092", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Seobility", "category": "SEO platform", "domain": "seobility.net", "robots_url": "https://seobility.net/robots.txt", "final_url": "https://seobility.net/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 4763, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Applebot-Extended" ], "source_sha256": "1e36162d34164ec507cc769880cdb6ffe13911bf66307ed044c66af8b8ea3ae4", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Mangools", "category": "SEO platform", "domain": "mangools.com", "robots_url": "https://mangools.com/robots.txt", "final_url": "https://mangools.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 483, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": "ef150c795fa3b94fcfdc6580071116b301a86affb5d94c269818a4a250b6d113", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Screaming Frog", "category": "SEO platform", "domain": "screamingfrog.co.uk", "robots_url": "https://screamingfrog.co.uk/robots.txt", "final_url": "https://screamingfrog.co.uk/robots.txt", "http_status": 200, "content_type": "text/plain; charset=UTF-8", "file_bytes": 1883, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": "4daae490a5b51502ebc20ed9a7e83564d7b808a0c7647a370d8542273e55a1e1", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Sitebulb", "category": "SEO platform", "domain": "sitebulb.com", "robots_url": "https://sitebulb.com/robots.txt", "final_url": "https://sitebulb.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 237, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "a8ba2e5501c74650ebd0aaefedf96f92971e64564cc2d9d407c31109949ece55", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Botify", "category": "SEO platform", "domain": "botify.com", "robots_url": "https://botify.com/robots.txt", "final_url": "https://www.botify.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 425, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "8faf809ae34f1ac78dbe06b1de197740119c8ea4cd8635c86cfa0fd5a56fc980", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Oncrawl", "category": "SEO platform", "domain": "oncrawl.com", "robots_url": "https://oncrawl.com/robots.txt", "final_url": "https://oncrawl.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 743, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "427c008df127916675a9c9aa8108120dd8b260f9f05427c5a0462c536ba9a98b", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Surfer", "category": "Content and AI SEO", "domain": "surferseo.com", "robots_url": "https://surferseo.com/robots.txt", "final_url": "https://surferseo.com/robots.txt/", "http_status": 200, "content_type": "text/plain", "file_bytes": 144, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": "4035cf6a2df927c8b8b9b730f841f52d183ff1b60234ef4937884c65fa16d8d0", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Clearscope", "category": "Content and AI SEO", "domain": "clearscope.io", "robots_url": "https://clearscope.io/robots.txt", "final_url": "https://www.clearscope.io/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 247, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": "56076d8a196a1d78246f5a5e24c8fd293f2f7ec32b8d53d63410dca806fd2699", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "MarketMuse", "category": "Content and AI SEO", "domain": "marketmuse.com", "robots_url": "https://marketmuse.com/robots.txt", "final_url": "https://www.marketmuse.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 134, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "a1db4849d66e0245cd826c0588d0a3005e78579f7d0d549f97e4297a72daa9b9", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Frase", "category": "Content and AI SEO", "domain": "frase.io", "robots_url": "https://frase.io/robots.txt", "final_url": "https://www.frase.io/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 1144, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Perplexity-User", "Google-Extended", "CCBot", "Applebot-Extended" ], "source_sha256": "22464f9058852528ac84488ab0b4cc7d999f866ff07b1930f488491a5f0bb2ba", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Jasper", "category": "Content and AI SEO", "domain": "jasper.ai", "robots_url": "https://jasper.ai/robots.txt", "final_url": "https://www.jasper.ai/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 665, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "PerplexityBot", "Google-Extended", "CCBot", "Applebot-Extended" ], "source_sha256": "e03e4247cb85d675d4ed3e50a4dea5e7b4c4039e151cfd0332ac9d79730b1dae", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Copy.ai", "category": "Content and AI SEO", "domain": "copy.ai", "robots_url": "https://copy.ai/robots.txt", "final_url": "https://www.copy.ai/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 834, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "eac7d3141f42d652411d536c2a4e3859bb6fdc52fbcb39f645f28dc717129b19", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Writesonic", "category": "Content and AI SEO", "domain": "writesonic.com", "robots_url": "https://writesonic.com/robots.txt", "final_url": "https://writesonic.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 164, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "d4ea5acd37ebbbb704064ca7fb77883cd86655a48554fd8509648115c1d45c77", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Scalenut", "category": "Content and AI SEO", "domain": "scalenut.com", "robots_url": "https://scalenut.com/robots.txt", "final_url": "https://www.scalenut.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 147, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "d7868395e88c2e5be4d1d11ea10edf61d6bd7799ba30d1c04f0a3839dff765f9", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "NeuronWriter", "category": "Content and AI SEO", "domain": "neuronwriter.com", "robots_url": "https://neuronwriter.com/robots.txt", "final_url": "https://neuronwriter.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 868, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "13d12e297f0284d3c4acc8eb1654b996da77102b12663659813e3623a49bb96a", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "SEOmatic", "category": "Content and AI SEO", "domain": "seomatic.ai", "robots_url": "https://seomatic.ai/robots.txt", "final_url": "https://seomatic.ai/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 396, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "OAI-SearchBot", "Claude-SearchBot", "PerplexityBot" ], "source_sha256": "c78f413c60f60cf8ba3dedf90973d1816886dce7c1a563e9511dcf66c17240a1", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Profound", "category": "AI visibility and AEO", "domain": "tryprofound.com", "robots_url": "https://tryprofound.com/robots.txt", "final_url": "https://www.tryprofound.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 107, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "0fc59b727690ab9cc189d9bfc4f7b3730b4ca2026f1174ad9074b961413bcbe2", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Peec AI", "category": "AI visibility and AEO", "domain": "peec.ai", "robots_url": "https://peec.ai/robots.txt", "final_url": "https://peec.ai/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 60, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "691a1d761cbc6b850c0ff4e3a9fe9843d794d9ade1570568f1f6c85044c17ab0", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "OtterlyAI", "category": "AI visibility and AEO", "domain": "otterly.ai", "robots_url": "https://otterly.ai/robots.txt", "final_url": "https://otterly.ai/robots.txt", "http_status": 403, "content_type": "text/html", "file_bytes": null, "robots_available": false, "access_basis": "unavailable", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": null, "error": "HTTP 403", "gptbot": "unavailable", "oai_searchbot": "unavailable", "chatgpt_user": "unavailable", "claudebot": "unavailable", "claude_searchbot": "unavailable", "claude_user": "unavailable", "perplexitybot": "unavailable", "perplexity_user": "unavailable", "google_extended": "unavailable", "ccbot": "unavailable", "applebot_extended": "unavailable" }, { "brand": "Scrunch AI", "category": "AI visibility and AEO", "domain": "scrunchai.com", "robots_url": "https://scrunchai.com/robots.txt", "final_url": "https://scrunch.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 71, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "486d29e37672c1dcdc65608af171c74fafc251a2b35c7da71cc91a61e3bc0c1b", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "AEO Engine", "category": "AI visibility and AEO", "domain": "aeoengine.ai", "robots_url": "https://aeoengine.ai/robots.txt", "final_url": "https://aeoengine.ai/robots.txt", "http_status": 403, "content_type": "text/plain; charset=utf-8", "file_bytes": null, "robots_available": false, "access_basis": "unavailable", "has_sitemap": false, "explicit_ai_agents": [], "source_sha256": null, "error": "HTTP 403", "gptbot": "unavailable", "oai_searchbot": "unavailable", "chatgpt_user": "unavailable", "claudebot": "unavailable", "claude_searchbot": "unavailable", "claude_user": "unavailable", "perplexitybot": "unavailable", "perplexity_user": "unavailable", "google_extended": "unavailable", "ccbot": "unavailable", "applebot_extended": "unavailable" }, { "brand": "RadiusRank", "category": "AI visibility and AEO", "domain": "radiusrank.com", "robots_url": "https://radiusrank.com/robots.txt", "final_url": "https://www.radiusrank.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 853, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "PerplexityBot", "Google-Extended", "CCBot", "Applebot-Extended" ], "source_sha256": "7b4967b47cbd8e2bfe2c5e5b6e0a0b1a441462ab0b31ce0c484a93d3214cdb38", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Yext", "category": "Local SEO", "domain": "yext.com", "robots_url": "https://yext.com/robots.txt", "final_url": "https://www.yext.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 544, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "ClaudeBot", "PerplexityBot", "CCBot" ], "source_sha256": "e51a861f324e21901432327846188fa9129ccc308907e8d5cff7fab74a52ecf6", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "BrightLocal", "category": "Local SEO", "domain": "brightlocal.com", "robots_url": "https://brightlocal.com/robots.txt", "final_url": "https://www.brightlocal.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 1293, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-SearchBot", "Claude-User", "PerplexityBot", "Google-Extended", "CCBot", "Applebot-Extended" ], "source_sha256": "0d8d56f2be1b98022b2149db9b78e2bd8dd775f3aa82ec3dd43d0903bc30fed6", "error": null, "gptbot": "blocked", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "blocked", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "blocked", "ccbot": "blocked", "applebot_extended": "blocked" }, { "brand": "Whitespark", "category": "Local SEO", "domain": "whitespark.ca", "robots_url": "https://whitespark.ca/robots.txt", "final_url": "https://whitespark.ca/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 111, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "1fc534d25f89ab07342bb68dca903a88968ca12a09f0eb6fddc4527918f69ef1", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Local Falcon", "category": "Local SEO", "domain": "localfalcon.com", "robots_url": "https://localfalcon.com/robots.txt", "final_url": "https://www.localfalcon.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 2143, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "03849ade50e390404f1de396ba75cdb25fdf86e136ff1f14f380266ab509c556", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Local Viking", "category": "Local SEO", "domain": "localviking.com", "robots_url": "https://localviking.com/robots.txt", "final_url": "https://localviking.com/robots.txt", "http_status": 200, "content_type": "application/octet-stream", "file_bytes": 195, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "70cdcd5d56947d441c4de360090e21a1b993caf60c6bf7f053483e01689e0475", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Yoast", "category": "CMS and optimization", "domain": "yoast.com", "robots_url": "https://yoast.com/robots.txt", "final_url": "https://yoast.com/robots.txt", "http_status": 200, "content_type": "text/plain", "file_bytes": 927, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "7430fe272fd171a2aec5718cc08b6ef90fdaec64617dbb779607643ace91bc06", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Rank Math", "category": "CMS and optimization", "domain": "rankmath.com", "robots_url": "https://rankmath.com/robots.txt", "final_url": "https://rankmath.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 247, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "99bafe1d28f1e3c139329410322c702a18abc8540d29179b0d477147d73a7f03", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "All in One SEO", "category": "CMS and optimization", "domain": "aioseo.com", "robots_url": "https://aioseo.com/robots.txt", "final_url": "https://aioseo.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 610, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "963fe1756588880ea970449b0c88caa31dd5c79c1913aa0a707df47b6c0c2280", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Webflow", "category": "CMS and optimization", "domain": "webflow.com", "robots_url": "https://webflow.com/robots.txt", "final_url": "https://webflow.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 279, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "b04ed9b180086889a2046dc5e952815a6b642497a787f5b54c698cd2549e5d98", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" }, { "brand": "Wix", "category": "CMS and optimization", "domain": "wix.com", "robots_url": "https://wix.com/robots.txt", "final_url": "https://www.wix.com/robots.txt", "http_status": 200, "content_type": "text/plain; charset=utf-8", "file_bytes": 2797, "robots_available": true, "access_basis": "robots.txt rules", "has_sitemap": true, "explicit_ai_agents": [], "source_sha256": "fef1f6ed42a555f0eb1efa45b48d3d00c98d8eb0452bb6d2a1487d6ba8771d6a", "error": null, "gptbot": "allowed", "oai_searchbot": "allowed", "chatgpt_user": "allowed", "claudebot": "allowed", "claude_searchbot": "allowed", "claude_user": "allowed", "perplexitybot": "allowed", "perplexity_user": "allowed", "google_extended": "allowed", "ccbot": "allowed", "applebot_extended": "allowed" } ] }