{
  "study": "AI crawler access across the Tranco top sites",
  "sample": {
    "file": "research/ai-crawler-access/2026-10/sample.csv",
    "size": 1000
  },
  "run": {
    "finishedAt": "2026-10-12T15:15:29.293Z",
    "scoringVersion": 2,
    "parityUserAgents": [
      "GPTBot",
      "OAI-SearchBot",
      "ClaudeBot",
      "PerplexityBot"
    ]
  },
  "reached": {
    "ok": 688,
    "unreachable": 312,
    "error": 0
  },
  "readiness": {
    "scoreMean": 84.5,
    "scoreMedian": 90,
    "grades": {
      "JavaScript-Dependent": 88,
      "AI Ready": 355,
      "Restricted by Design": 106,
      "Partly Checked": 126,
      "Partial Access": 13
    },
    "intentional": 106,
    "intentional_pct": 15.4,
    "jsDependent": 88,
    "jsDependent_pct": 12.8,
    "partlyChecked": 148,
    "partlyChecked_pct": 21.5
  },
  "robots": {
    "withRobotsTxt": 569,
    "withRobotsTxt_pct": 82.7,
    "blocksAnyScored": 143,
    "blocksAnyScored_pct": 25.1,
    "blocksMajorityScored": 58,
    "blocksMajorityScored_pct": 10.2,
    "blocksAnyTraining": 148,
    "blocksAnyTraining_pct": 26,
    "blocksTrainingOnly": 15,
    "blocksTrainingOnly_pct": 2.6,
    "wildcardDisallowAll": 0,
    "contentSignal": 14,
    "contentSignal_pct": 2.5,
    "sitemapInRobots": 355,
    "sitemapInRobots_pct": 62.4
  },
  "adoption": {
    "llmsTxt": {
      "count": 236,
      "pct": 34.3
    },
    "llmsFullTxt": {
      "count": 151,
      "pct": 21.9
    },
    "agentsJson": {
      "count": 2,
      "pct": 0.3
    },
    "markdownServed": {
      "count": 24,
      "pct": 3.5
    },
    "markdownLinked": {
      "count": 10,
      "pct": 1.5
    },
    "jsonLd": {
      "count": 249,
      "pct": 36.2
    },
    "metaDescription": {
      "count": 450,
      "pct": 65.4
    },
    "ogTitle": {
      "count": 374,
      "pct": 54.4
    },
    "feeds": {
      "count": 61,
      "pct": 8.9
    },
    "metaRobotsNoindex": {
      "count": 10,
      "pct": 1.5
    },
    "llmsTxtLoose": {
      "count": 236,
      "pct": 34.3
    },
    "llmsTxtStrict": {
      "count": 104,
      "pct": 15.1,
      "note": "llms.txt that is a text file starting with a Markdown heading; the looser count includes sites that answer 2xx for any path"
    }
  },
  "forms": {
    "withForms": 254,
    "webmcp": 1
  },
  "parity": {
    "base": 549,
    "perCrawler": {
      "GPTBot": {
        "server_refused": 79,
        "server_refused_pct": 14.4,
        "robots_allows_but_server_refuses": 61,
        "robots_allows_but_server_refuses_pct": 11.1,
        "robots_blocks_but_server_serves": 53
      },
      "OAI-SearchBot": {
        "server_refused": 54,
        "server_refused_pct": 9.8,
        "robots_allows_but_server_refuses": 44,
        "robots_allows_but_server_refuses_pct": 8,
        "robots_blocks_but_server_serves": 34
      },
      "ClaudeBot": {
        "server_refused": 81,
        "server_refused_pct": 14.8,
        "robots_allows_but_server_refuses": 60,
        "robots_allows_but_server_refuses_pct": 10.9,
        "robots_blocks_but_server_serves": 49
      },
      "PerplexityBot": {
        "server_refused": 62,
        "server_refused_pct": 11.3,
        "robots_allows_but_server_refuses": 46,
        "robots_allows_but_server_refuses_pct": 8.4,
        "robots_blocks_but_server_serves": 41
      }
    },
    "anyCrawlerRefusedWhileRobotsAllows": 78,
    "anyCrawlerRefusedWhileRobotsAllowsExcluding429": 64
  },
  "crawlers": [
    {
      "token": "GPTBot",
      "group": "OpenAI",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 105,
      "allowed": 464,
      "blocked_pct": 18.5,
      "parity_base": 549,
      "server_refused": 79,
      "server_refused_pct": 14.4,
      "robots_allows_but_server_refuses": 61,
      "robots_allows_but_server_refuses_pct": 11.1,
      "robots_allows_but_server_refuses_429": 6,
      "robots_allows_but_server_refuses_excluding_429": 55,
      "robots_allows_but_server_refuses_excluding_429_pct": 10,
      "robots_blocks_but_server_serves": 53,
      "refused_domains": [
        "adobe.com",
        "endpoints.news",
        "chatgpt.com",
        "openai.com",
        "dropbox.com",
        "unity3d.com",
        "webex.com",
        "xiaomi.com",
        "hosting24.com",
        "cisco.com",
        "ibm.com",
        "pubmatic.com",
        "salesforce.com",
        "autodesk.com",
        "mi.com",
        "wiley.com",
        "gnu.org",
        "slack.com",
        "cdc.gov",
        "un.org",
        "163.com",
        "intel.com",
        "ietf.org",
        "statista.com",
        "trendmicro.com",
        "oup.com",
        "openstreetmap.org",
        "amazonalexa.com",
        "cornell.edu",
        "teamviewer.com",
        "mdpi.com",
        "accuweather.com",
        "nike.com",
        "loc.gov",
        "chatgpt.site",
        "cdnvideo.ru",
        "shopee.com.br",
        "cookielaw.org",
        "itch.io",
        "shopee.co.id",
        "homedepot.com",
        "sedo.com",
        "zillow.com",
        "welt.de",
        "lenovo.com",
        "xvideos.com",
        "zoho.com",
        "att.com",
        "usda.gov",
        "focus.de",
        "readthedocs.io",
        "mayoclinic.org",
        "patreon.com",
        "state.gov",
        "note.com",
        "att.net",
        "eporner.com",
        "hm.com",
        "lowes.com",
        "xnxx.com",
        "shopee.ph"
      ]
    },
    {
      "token": "OAI-SearchBot",
      "group": "OpenAI",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 58,
      "allowed": 511,
      "blocked_pct": 10.2,
      "parity_base": 549,
      "server_refused": 54,
      "server_refused_pct": 9.8,
      "robots_allows_but_server_refuses": 44,
      "robots_allows_but_server_refuses_pct": 8,
      "robots_allows_but_server_refuses_429": 4,
      "robots_allows_but_server_refuses_excluding_429": 40,
      "robots_allows_but_server_refuses_excluding_429_pct": 7.3,
      "robots_blocks_but_server_serves": 34,
      "refused_domains": [
        "yahoo.com",
        "adobe.com",
        "endpoints.news",
        "chatgpt.com",
        "openai.com",
        "dropbox.com",
        "unity3d.com",
        "webex.com",
        "xiaomi.com",
        "medium.com",
        "cisco.com",
        "ibm.com",
        "pubmatic.com",
        "salesforce.com",
        "autodesk.com",
        "mi.com",
        "cdc.gov",
        "intel.com",
        "trendmicro.com",
        "oup.com",
        "amazonalexa.com",
        "cornell.edu",
        "mdpi.com",
        "accuweather.com",
        "nike.com",
        "chatgpt.site",
        "shopee.com.br",
        "airbnb.com",
        "itch.io",
        "shopee.co.id",
        "homedepot.com",
        "zillow.com",
        "welt.de",
        "lenovo.com",
        "att.com",
        "usda.gov",
        "focus.de",
        "mayoclinic.org",
        "state.gov",
        "att.net",
        "eporner.com",
        "hm.com",
        "lowes.com",
        "shopee.ph"
      ]
    },
    {
      "token": "ChatGPT-User",
      "group": "OpenAI",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 69,
      "allowed": 500,
      "blocked_pct": 12.1
    },
    {
      "token": "ClaudeBot",
      "group": "Anthropic",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 104,
      "allowed": 465,
      "blocked_pct": 18.3,
      "parity_base": 549,
      "server_refused": 81,
      "server_refused_pct": 14.8,
      "robots_allows_but_server_refuses": 60,
      "robots_allows_but_server_refuses_pct": 10.9,
      "robots_allows_but_server_refuses_429": 10,
      "robots_allows_but_server_refuses_excluding_429": 50,
      "robots_allows_but_server_refuses_excluding_429_pct": 9.1,
      "robots_blocks_but_server_serves": 49,
      "refused_domains": [
        "wikipedia.org",
        "adobe.com",
        "endpoints.news",
        "chatgpt.com",
        "openai.com",
        "dropbox.com",
        "vk.com",
        "unity3d.com",
        "webex.com",
        "xiaomi.com",
        "hosting24.com",
        "cisco.com",
        "wikimedia.org",
        "rubiconproject.com",
        "ibm.com",
        "pubmatic.com",
        "tinyurl.com",
        "salesforce.com",
        "autodesk.com",
        "harvard.edu",
        "mi.com",
        "wiley.com",
        "weebly.com",
        "slack.com",
        "cdc.gov",
        "nasa.gov",
        "claude.ai",
        "intel.com",
        "trendmicro.com",
        "oup.com",
        "openstreetmap.org",
        "cornell.edu",
        "mdpi.com",
        "accuweather.com",
        "nike.com",
        "loc.gov",
        "2gis.com",
        "chatgpt.site",
        "cookielaw.org",
        "itch.io",
        "homedepot.com",
        "sedo.com",
        "zillow.com",
        "welt.de",
        "lenovo.com",
        "xvideos.com",
        "zoho.com",
        "att.com",
        "usda.gov",
        "focus.de",
        "fwmrm.net",
        "readthedocs.io",
        "mayoclinic.org",
        "patreon.com",
        "state.gov",
        "note.com",
        "att.net",
        "eporner.com",
        "hm.com",
        "xnxx.com"
      ]
    },
    {
      "token": "Claude-User",
      "group": "Anthropic",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 68,
      "allowed": 501,
      "blocked_pct": 12
    },
    {
      "token": "Claude-SearchBot",
      "group": "Anthropic",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 63,
      "allowed": 506,
      "blocked_pct": 11.1
    },
    {
      "token": "Claude-Web",
      "group": "Anthropic",
      "purpose": "retired",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 82,
      "allowed": 487,
      "blocked_pct": 14.4
    },
    {
      "token": "anthropic-ai",
      "group": "Anthropic",
      "purpose": "retired",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 80,
      "allowed": 489,
      "blocked_pct": 14.1
    },
    {
      "token": "Google-Extended",
      "group": "Google AI",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 91,
      "allowed": 478,
      "blocked_pct": 16
    },
    {
      "token": "GoogleOther",
      "group": "Google AI",
      "purpose": "research",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 63,
      "allowed": 506,
      "blocked_pct": 11.1
    },
    {
      "token": "Gemini-Deep-Research",
      "group": "Google AI",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 56,
      "allowed": 513,
      "blocked_pct": 9.8
    },
    {
      "token": "Google-NotebookLM",
      "group": "Google AI",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 54,
      "allowed": 515,
      "blocked_pct": 9.5
    },
    {
      "token": "Google-CloudVertexBot",
      "group": "Google AI",
      "purpose": "owner",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 69,
      "allowed": 500,
      "blocked_pct": 12.1
    },
    {
      "token": "GoogleOther-Image",
      "group": "Google AI",
      "purpose": "research",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 44,
      "allowed": 525,
      "blocked_pct": 7.7
    },
    {
      "token": "GoogleOther-Video",
      "group": "Google AI",
      "purpose": "research",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 44,
      "allowed": 525,
      "blocked_pct": 7.7
    },
    {
      "token": "PerplexityBot",
      "group": "Perplexity",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 84,
      "allowed": 485,
      "blocked_pct": 14.8,
      "parity_base": 549,
      "server_refused": 62,
      "server_refused_pct": 11.3,
      "robots_allows_but_server_refuses": 46,
      "robots_allows_but_server_refuses_pct": 8.4,
      "robots_allows_but_server_refuses_429": 6,
      "robots_allows_but_server_refuses_excluding_429": 40,
      "robots_allows_but_server_refuses_excluding_429_pct": 7.3,
      "robots_blocks_but_server_serves": 41,
      "refused_domains": [
        "adobe.com",
        "endpoints.news",
        "openai.com",
        "dropbox.com",
        "unity3d.com",
        "webex.com",
        "xiaomi.com",
        "vk.ru",
        "medium.com",
        "cisco.com",
        "userapi.com",
        "ibm.com",
        "pubmatic.com",
        "tinyurl.com",
        "salesforce.com",
        "autodesk.com",
        "mi.com",
        "cdc.gov",
        "claude.ai",
        "intel.com",
        "trendmicro.com",
        "oup.com",
        "amazonalexa.com",
        "mdpi.com",
        "accuweather.com",
        "nike.com",
        "chatgpt.site",
        "shopee.com.br",
        "airbnb.com",
        "shopee.co.id",
        "homedepot.com",
        "zillow.com",
        "lenovo.com",
        "xvideos.com",
        "lefigaro.fr",
        "att.com",
        "usda.gov",
        "focus.de",
        "mayoclinic.org",
        "state.gov",
        "att.net",
        "eporner.com",
        "hm.com",
        "lowes.com",
        "xnxx.com",
        "shopee.ph"
      ]
    },
    {
      "token": "Perplexity-User",
      "group": "Perplexity",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 72,
      "allowed": 497,
      "blocked_pct": 12.7
    },
    {
      "token": "GrokBot",
      "group": "xAI / Grok",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 53,
      "allowed": 516,
      "blocked_pct": 9.3
    },
    {
      "token": "Grok-DeepSearch",
      "group": "xAI / Grok",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 52,
      "allowed": 517,
      "blocked_pct": 9.1
    },
    {
      "token": "xAI-Grok",
      "group": "xAI / Grok",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 53,
      "allowed": 516,
      "blocked_pct": 9.3
    },
    {
      "token": "Copilot",
      "group": "Microsoft Copilot",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 52,
      "allowed": 517,
      "blocked_pct": 9.1
    },
    {
      "token": "CopilotNative",
      "group": "Microsoft Copilot",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 52,
      "allowed": 517,
      "blocked_pct": 9.1
    },
    {
      "token": "meta-externalagent",
      "group": "Meta AI",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 96,
      "allowed": 473,
      "blocked_pct": 16.9
    },
    {
      "token": "meta-externalfetcher",
      "group": "Meta AI",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 64,
      "allowed": 505,
      "blocked_pct": 11.2
    },
    {
      "token": "FacebookBot",
      "group": "Meta AI",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 76,
      "allowed": 493,
      "blocked_pct": 13.4
    },
    {
      "token": "Applebot-Extended",
      "group": "Apple Intelligence",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 86,
      "allowed": 483,
      "blocked_pct": 15.1
    },
    {
      "token": "Amazonbot",
      "group": "Amazon",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 78,
      "allowed": 491,
      "blocked_pct": 13.7
    },
    {
      "token": "PetalBot",
      "group": "Huawei",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 89,
      "allowed": 480,
      "blocked_pct": 15.6
    },
    {
      "token": "YandexAdditional",
      "group": "Yandex AI",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 47,
      "allowed": 522,
      "blocked_pct": 8.3
    },
    {
      "token": "YandexAdditionalBot",
      "group": "Yandex AI",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 48,
      "allowed": 521,
      "blocked_pct": 8.4
    },
    {
      "token": "DeepSeekBot",
      "group": "DeepSeek",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 62,
      "allowed": 507,
      "blocked_pct": 10.9
    },
    {
      "token": "MistralAI-User",
      "group": "Mistral",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 67,
      "allowed": 502,
      "blocked_pct": 11.8
    },
    {
      "token": "cohere-ai",
      "group": "Cohere",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 90,
      "allowed": 479,
      "blocked_pct": 15.8
    },
    {
      "token": "cohere-training-data-crawler",
      "group": "Cohere",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 71,
      "allowed": 498,
      "blocked_pct": 12.5
    },
    {
      "token": "CCBot",
      "group": "Common Crawl",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 119,
      "allowed": 450,
      "blocked_pct": 20.9
    },
    {
      "token": "DuckAssistBot",
      "group": "DuckDuckGo AI",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 66,
      "allowed": 503,
      "blocked_pct": 11.6
    },
    {
      "token": "Bytespider",
      "group": "Bytedance",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 120,
      "allowed": 449,
      "blocked_pct": 21.1
    },
    {
      "token": "Diffbot",
      "group": "Diffbot",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 97,
      "allowed": 472,
      "blocked_pct": 17
    },
    {
      "token": "AI2Bot",
      "group": "Allen AI",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 76,
      "allowed": 493,
      "blocked_pct": 13.4
    },
    {
      "token": "AI2Bot-Dolma",
      "group": "Allen AI",
      "purpose": "training",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 69,
      "allowed": 500,
      "blocked_pct": 12.1
    },
    {
      "token": "PhindBot",
      "group": "Phind",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 57,
      "allowed": 512,
      "blocked_pct": 10
    },
    {
      "token": "YouBot",
      "group": "You.com",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 82,
      "allowed": 487,
      "blocked_pct": 14.4
    },
    {
      "token": "Kagibot",
      "group": "Kagi",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 39,
      "allowed": 530,
      "blocked_pct": 6.9
    },
    {
      "token": "AndiBot",
      "group": "Andi",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 55,
      "allowed": 514,
      "blocked_pct": 9.7
    },
    {
      "token": "Timpibot",
      "group": "Timpi",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 80,
      "allowed": 489,
      "blocked_pct": 14.1
    },
    {
      "token": "SiderAI",
      "group": "Sider",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 38,
      "allowed": 531,
      "blocked_pct": 6.7
    },
    {
      "token": "HuggingFace-Bot",
      "group": "HuggingFace",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 38,
      "allowed": 531,
      "blocked_pct": 6.7
    },
    {
      "token": "Quora-Bot",
      "group": "Quora / Poe",
      "purpose": "answers",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 44,
      "allowed": 525,
      "blocked_pct": 7.7
    },
    {
      "token": "GensparkBot",
      "group": "Genspark",
      "purpose": "search",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 39,
      "allowed": 530,
      "blocked_pct": 6.9
    },
    {
      "token": "NeevaBot",
      "group": "Neeva",
      "purpose": "retired",
      "scored": false,
      "sites_with_robots": 569,
      "blocked": 42,
      "allowed": 527,
      "blocked_pct": 7.4
    },
    {
      "token": "Elicit-Bot",
      "group": "Elicit",
      "purpose": "user",
      "scored": true,
      "sites_with_robots": 569,
      "blocked": 38,
      "allowed": 531,
      "blocked_pct": 6.7
    }
  ]
}