{
  "index": "AI Access Index",
  "edition": "2026-11",
  "label": "November 2026",
  "measuredOn": "2026-11-01",
  "previousEdition": null,
  "run": {
    "startedAt": "2026-11-01T13:27:19.282Z",
    "finishedAt": "2026-11-01T13:37:03.435Z",
    "durationSec": 584,
    "runner": "GitHub Actions",
    "concurrency": 6,
    "scoringVersion": 3,
    "node": "v24.22.1",
    "sample": {
      "file": "data/ai-access-index/sample.csv",
      "size": 1000,
      "source": "The AI Access Index sample: Tranco list 8PXYV (https://tranco-list.eu/list/8PXYV), top 1,000, downloaded 2026-10-11 for the October 2026 study and frozen here, so every edition checks the same sites."
    }
  },
  "reached": {
    "sample": 1000,
    "ok": 684,
    "unreachable": 316,
    "error": 0
  },
  "policy": {
    "open": {
      "count": 545,
      "base": 684,
      "pct": 79.7
    },
    "search-only": {
      "count": 60,
      "base": 684,
      "pct": 8.8
    },
    "mixed": {
      "count": 26,
      "base": 684,
      "pct": 3.8
    },
    "closed": {
      "count": 53,
      "base": 684,
      "pct": 7.7
    }
  },
  "policyDetail": {
    "openWithoutRobotsTxt": {
      "count": 119,
      "base": 684,
      "pct": 17.4
    },
    "closedToSearchEnginesToo": {
      "count": 13,
      "base": 684,
      "pct": 1.9
    }
  },
  "integrity": {
    "consistent": {
      "count": 460,
      "base": 684,
      "pct": 67.3
    },
    "overblocks": {
      "count": 60,
      "base": 684,
      "pct": 8.8
    },
    "leaks": {
      "count": 77,
      "base": 684,
      "pct": 11.3
    },
    "both": {
      "count": 2,
      "base": 684,
      "pct": 0.3
    },
    "unknown": {
      "count": 85,
      "base": 684,
      "pct": 12.4
    }
  },
  "server": {
    "tested": {
      "count": 599,
      "base": 684,
      "pct": 87.6
    },
    "refusedWhileAllowed": {
      "count": 62,
      "base": 547,
      "pct": 11.3
    },
    "refusedWhileAllowedExcluding429": {
      "count": 45,
      "base": 547,
      "pct": 8.2
    },
    "servedWhileBlocked": {
      "count": 79,
      "base": 124,
      "pct": 63.7
    }
  },
  "perCrawler": {
    "GPTBot": {
      "robotsBlocks": {
        "count": 107,
        "base": 684,
        "pct": 15.6
      },
      "serverRefuses": {
        "count": 46,
        "base": 525,
        "pct": 8.8
      },
      "refusedWhileAllowed": {
        "count": 31,
        "base": 525,
        "pct": 5.9
      },
      "servedWhileBlocked": {
        "count": 60,
        "base": 80,
        "pct": 75
      }
    },
    "OAI-SearchBot": {
      "robotsBlocks": {
        "count": 56,
        "base": 684,
        "pct": 8.2
      },
      "serverRefuses": {
        "count": 20,
        "base": 527,
        "pct": 3.8
      },
      "refusedWhileAllowed": {
        "count": 12,
        "base": 527,
        "pct": 2.3
      },
      "servedWhileBlocked": {
        "count": 35,
        "base": 46,
        "pct": 76.1
      }
    },
    "ClaudeBot": {
      "robotsBlocks": {
        "count": 106,
        "base": 684,
        "pct": 15.5
      },
      "serverRefuses": {
        "count": 46,
        "base": 527,
        "pct": 8.7
      },
      "refusedWhileAllowed": {
        "count": 29,
        "base": 527,
        "pct": 5.5
      },
      "servedWhileBlocked": {
        "count": 56,
        "base": 75,
        "pct": 74.7
      }
    },
    "PerplexityBot": {
      "robotsBlocks": {
        "count": 84,
        "base": 684,
        "pct": 12.3
      },
      "serverRefuses": {
        "count": 45,
        "base": 527,
        "pct": 8.5
      },
      "refusedWhileAllowed": {
        "count": 33,
        "base": 527,
        "pct": 6.3
      },
      "servedWhileBlocked": {
        "count": 43,
        "base": 60,
        "pct": 71.7
      }
    }
  },
  "crawlers": [
    {
      "token": "Bytespider",
      "group": "Bytedance",
      "purpose": "training",
      "count": 120,
      "base": 684,
      "pct": 17.5
    },
    {
      "token": "CCBot",
      "group": "Common Crawl",
      "purpose": "training",
      "count": 118,
      "base": 684,
      "pct": 17.3
    },
    {
      "token": "GPTBot",
      "group": "OpenAI",
      "purpose": "training",
      "count": 107,
      "base": 684,
      "pct": 15.6
    },
    {
      "token": "ClaudeBot",
      "group": "Anthropic",
      "purpose": "training",
      "count": 106,
      "base": 684,
      "pct": 15.5
    },
    {
      "token": "Diffbot",
      "group": "Diffbot",
      "purpose": "answers",
      "count": 96,
      "base": 684,
      "pct": 14
    },
    {
      "token": "meta-externalagent",
      "group": "Meta AI",
      "purpose": "answers",
      "count": 96,
      "base": 684,
      "pct": 14
    },
    {
      "token": "Google-Extended",
      "group": "Google AI",
      "purpose": "answers",
      "count": 92,
      "base": 684,
      "pct": 13.5
    },
    {
      "token": "cohere-ai",
      "group": "Cohere",
      "purpose": "answers",
      "count": 90,
      "base": 684,
      "pct": 13.2
    },
    {
      "token": "Applebot-Extended",
      "group": "Apple Intelligence",
      "purpose": "training",
      "count": 88,
      "base": 684,
      "pct": 12.9
    },
    {
      "token": "PetalBot",
      "group": "Huawei",
      "purpose": "search",
      "count": 88,
      "base": 684,
      "pct": 12.9
    },
    {
      "token": "Amazonbot",
      "group": "Amazon",
      "purpose": "answers",
      "count": 84,
      "base": 684,
      "pct": 12.3
    },
    {
      "token": "PerplexityBot",
      "group": "Perplexity",
      "purpose": "search",
      "count": 84,
      "base": 684,
      "pct": 12.3
    },
    {
      "token": "Claude-Web",
      "group": "Anthropic",
      "purpose": "retired",
      "count": 81,
      "base": 684,
      "pct": 11.8
    },
    {
      "token": "YouBot",
      "group": "You.com",
      "purpose": "search",
      "count": 80,
      "base": 684,
      "pct": 11.7
    },
    {
      "token": "anthropic-ai",
      "group": "Anthropic",
      "purpose": "retired",
      "count": 79,
      "base": 684,
      "pct": 11.5
    },
    {
      "token": "Timpibot",
      "group": "Timpi",
      "purpose": "search",
      "count": 78,
      "base": 684,
      "pct": 11.4
    },
    {
      "token": "AI2Bot",
      "group": "Allen AI",
      "purpose": "training",
      "count": 73,
      "base": 684,
      "pct": 10.7
    },
    {
      "token": "FacebookBot",
      "group": "Meta AI",
      "purpose": "retired",
      "count": 73,
      "base": 684,
      "pct": 10.7
    },
    {
      "token": "cohere-training-data-crawler",
      "group": "Cohere",
      "purpose": "training",
      "count": 72,
      "base": 684,
      "pct": 10.5
    },
    {
      "token": "Perplexity-User",
      "group": "Perplexity",
      "purpose": "user",
      "count": 72,
      "base": 684,
      "pct": 10.5
    },
    {
      "token": "Google-CloudVertexBot",
      "group": "Google AI",
      "purpose": "owner",
      "count": 70,
      "base": 684,
      "pct": 10.2
    },
    {
      "token": "AI2Bot-Dolma",
      "group": "Allen AI",
      "purpose": "training",
      "count": 68,
      "base": 684,
      "pct": 9.9
    },
    {
      "token": "Claude-User",
      "group": "Anthropic",
      "purpose": "user",
      "count": 67,
      "base": 684,
      "pct": 9.8
    },
    {
      "token": "ChatGPT-User",
      "group": "OpenAI",
      "purpose": "user",
      "count": 66,
      "base": 684,
      "pct": 9.6
    },
    {
      "token": "MistralAI-User",
      "group": "Mistral",
      "purpose": "user",
      "count": 66,
      "base": 684,
      "pct": 9.6
    },
    {
      "token": "DuckAssistBot",
      "group": "DuckDuckGo AI",
      "purpose": "user",
      "count": 65,
      "base": 684,
      "pct": 9.5
    },
    {
      "token": "meta-externalfetcher",
      "group": "Meta AI",
      "purpose": "user",
      "count": 65,
      "base": 684,
      "pct": 9.5
    },
    {
      "token": "Claude-SearchBot",
      "group": "Anthropic",
      "purpose": "search",
      "count": 64,
      "base": 684,
      "pct": 9.4
    },
    {
      "token": "GoogleOther",
      "group": "Google AI",
      "purpose": "research",
      "count": 63,
      "base": 684,
      "pct": 9.2
    },
    {
      "token": "DeepSeekBot",
      "group": "DeepSeek",
      "purpose": "answers",
      "count": 60,
      "base": 684,
      "pct": 8.8
    },
    {
      "token": "meta-webindexer",
      "group": "Meta AI",
      "purpose": "search",
      "count": 59,
      "base": 684,
      "pct": 8.6
    },
    {
      "token": "OAI-SearchBot",
      "group": "OpenAI",
      "purpose": "search",
      "count": 56,
      "base": 684,
      "pct": 8.2
    },
    {
      "token": "PhindBot",
      "group": "Phind",
      "purpose": "search",
      "count": 55,
      "base": 684,
      "pct": 8
    },
    {
      "token": "Gemini-Deep-Research",
      "group": "Google AI",
      "purpose": "user",
      "count": 54,
      "base": 684,
      "pct": 7.9
    },
    {
      "token": "AndiBot",
      "group": "Andi",
      "purpose": "search",
      "count": 53,
      "base": 684,
      "pct": 7.7
    },
    {
      "token": "Google-NotebookLM",
      "group": "Google AI",
      "purpose": "retired",
      "count": 52,
      "base": 684,
      "pct": 7.6
    },
    {
      "token": "GrokBot",
      "group": "xAI / Grok",
      "purpose": "answers",
      "count": 51,
      "base": 684,
      "pct": 7.5
    },
    {
      "token": "xAI-Grok",
      "group": "xAI / Grok",
      "purpose": "answers",
      "count": 51,
      "base": 684,
      "pct": 7.5
    },
    {
      "token": "Copilot",
      "group": "Microsoft Copilot",
      "purpose": "answers",
      "count": 50,
      "base": 684,
      "pct": 7.3
    },
    {
      "token": "CopilotNative",
      "group": "Microsoft Copilot",
      "purpose": "answers",
      "count": 50,
      "base": 684,
      "pct": 7.3
    },
    {
      "token": "Grok-DeepSearch",
      "group": "xAI / Grok",
      "purpose": "user",
      "count": 50,
      "base": 684,
      "pct": 7.3
    },
    {
      "token": "YandexAdditionalBot",
      "group": "Yandex AI",
      "purpose": "answers",
      "count": 46,
      "base": 684,
      "pct": 6.7
    },
    {
      "token": "YandexAdditional",
      "group": "Yandex AI",
      "purpose": "answers",
      "count": 45,
      "base": 684,
      "pct": 6.6
    },
    {
      "token": "Amzn-SearchBot",
      "group": "Amazon",
      "purpose": "search",
      "count": 44,
      "base": 684,
      "pct": 6.4
    },
    {
      "token": "Amzn-User",
      "group": "Amazon",
      "purpose": "user",
      "count": 42,
      "base": 684,
      "pct": 6.1
    },
    {
      "token": "GoogleOther-Image",
      "group": "Google AI",
      "purpose": "research",
      "count": 42,
      "base": 684,
      "pct": 6.1
    },
    {
      "token": "GoogleOther-Video",
      "group": "Google AI",
      "purpose": "research",
      "count": 42,
      "base": 684,
      "pct": 6.1
    },
    {
      "token": "Quora-Bot",
      "group": "Quora / Poe",
      "purpose": "answers",
      "count": 42,
      "base": 684,
      "pct": 6.1
    },
    {
      "token": "MistralAI-Training",
      "group": "Mistral",
      "purpose": "training",
      "count": 40,
      "base": 684,
      "pct": 5.8
    },
    {
      "token": "NeevaBot",
      "group": "Neeva",
      "purpose": "retired",
      "count": 40,
      "base": 684,
      "pct": 5.8
    },
    {
      "token": "MistralAI-Index",
      "group": "Mistral",
      "purpose": "search",
      "count": 38,
      "base": 684,
      "pct": 5.6
    },
    {
      "token": "GensparkBot",
      "group": "Genspark",
      "purpose": "search",
      "count": 37,
      "base": 684,
      "pct": 5.4
    },
    {
      "token": "Google-Agent",
      "group": "Google AI",
      "purpose": "user",
      "count": 37,
      "base": 684,
      "pct": 5.4
    },
    {
      "token": "Kagibot",
      "group": "Kagi",
      "purpose": "search",
      "count": 37,
      "base": 684,
      "pct": 5.4
    },
    {
      "token": "OAI-AdsBot",
      "group": "OpenAI",
      "purpose": "owner",
      "count": 37,
      "base": 684,
      "pct": 5.4
    },
    {
      "token": "Elicit-Bot",
      "group": "Elicit",
      "purpose": "user",
      "count": 36,
      "base": 684,
      "pct": 5.3
    },
    {
      "token": "Google-GeminiNotebook",
      "group": "Google AI",
      "purpose": "user",
      "count": 36,
      "base": 684,
      "pct": 5.3
    },
    {
      "token": "HuggingFace-Bot",
      "group": "HuggingFace",
      "purpose": "answers",
      "count": 36,
      "base": 684,
      "pct": 5.3
    },
    {
      "token": "SiderAI",
      "group": "Sider",
      "purpose": "user",
      "count": 36,
      "base": 684,
      "pct": 5.3
    },
    {
      "token": "meta-externalads",
      "group": "Meta AI",
      "purpose": "ads",
      "count": 32,
      "base": 684,
      "pct": 4.7
    }
  ],
  "adoption": {
    "llmsTxt": {
      "count": 108,
      "base": 684,
      "pct": 15.8
    },
    "contentSignal": {
      "count": 17,
      "base": 684,
      "pct": 2.5
    },
    "tdmReserved": {
      "count": 6,
      "base": 684,
      "pct": 0.9
    },
    "markdownCopy": {
      "count": 25,
      "base": 684,
      "pct": 3.7
    },
    "agentsJson": {
      "count": 2,
      "base": 684,
      "pct": 0.3
    },
    "noai": {
      "count": 0,
      "base": 684,
      "pct": 0
    }
  },
  "changes": null
}