{
  "name": "Three Impact AI crawler registry",
  "source": "https://threeimpact.com/ai-crawler-registry",
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "verified": "2026-11-06",
  "note": "Each entry was checked against the operator's own documentation (docs). observedSharePct is the site-weighted mean share of AI fetches across the 12 small-business sites in the Q4 2026 Observatory; empty where the agent was not seen on 5+ sites.",
  "crawlers": [
    {
      "token": "GPTBot",
      "operator": "OpenAI",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "Crawls pages that may be used to train OpenAI's foundation models.",
      "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot",
      "respectsRobotsTxt": "Yes",
      "verify": "Published IP list",
      "verifyUrl": "https://openai.com/gptbot.json",
      "docs": "https://developers.openai.com/api/docs/bots",
      "blockCost": "Your future pages stay out of OpenAI training data. OpenAI says this does not affect ChatGPT search, which uses OAI-SearchBot.",
      "observedSharePct": 10.1
    },
    {
      "token": "OAI-SearchBot",
      "operator": "OpenAI",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Builds the index that ChatGPT search uses to surface and link to websites.",
      "userAgent": "Mozilla/5.0 … compatible; OAI-SearchBot/1.4; +https://openai.com/searchbot (version may change)",
      "respectsRobotsTxt": "Yes (changes take about 24 hours)",
      "verify": "Published IP list",
      "verifyUrl": "https://openai.com/searchbot.json",
      "docs": "https://developers.openai.com/api/docs/bots",
      "blockCost": "You drop out of ChatGPT search results and their citations. This is the one to keep open if you want ChatGPT to link to you.",
      "observedSharePct": 14.9
    },
    {
      "token": "ChatGPT-User",
      "operator": "OpenAI",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Fetches a page live when a ChatGPT user or a Custom GPT asks about it.",
      "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
      "respectsRobotsTxt": "May not apply: OpenAI says robots.txt rules may not apply to user-initiated visits",
      "verify": "Published IP list",
      "verifyUrl": "https://openai.com/chatgpt-user.json",
      "docs": "https://developers.openai.com/api/docs/bots",
      "blockCost": "ChatGPT can't read your page when someone asks about you directly. OpenAI says it does not decide whether you appear in search.",
      "observedSharePct": 28.1
    },
    {
      "token": "ClaudeBot",
      "operator": "Anthropic",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "Collects public web content that may be used to train Anthropic's models.",
      "userAgent": "Contains “ClaudeBot” (Anthropic does not publish the full string)",
      "respectsRobotsTxt": "Yes, including the non-standard Crawl-delay. Since 30 Oct 2026 it also honours the shared __usage_ai-training group; a ClaudeBot group overrides it",
      "verify": "Published IP list",
      "verifyUrl": "https://claude.com/crawling/bots.json",
      "docs": "https://support.claude.com/en/articles/8896518",
      "blockCost": "Your future pages are excluded from Anthropic training datasets. Claude's search and live fetches use separate agents.",
      "observedSharePct": 14.7
    },
    {
      "token": "Claude-SearchBot",
      "operator": "Anthropic",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Crawls to improve the quality of Claude's search results.",
      "userAgent": "Contains “Claude-SearchBot”",
      "respectsRobotsTxt": "Yes",
      "verify": "Published IP list",
      "verifyUrl": "https://claude.com/crawling/bots.json",
      "docs": "https://support.claude.com/en/articles/8896518",
      "blockCost": "Anthropic says your content won't be indexed for Claude's search, which may reduce your visibility in its answers.",
      "observedSharePct": 1.4
    },
    {
      "token": "Claude-User",
      "operator": "Anthropic",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Fetches a page when a Claude user's question needs it.",
      "userAgent": "Contains “Claude-User”",
      "respectsRobotsTxt": "Yes (Anthropic says its bots honour robots.txt)",
      "verify": "Published IP list",
      "verifyUrl": "https://claude.com/crawling/bots.json",
      "docs": "https://support.claude.com/en/articles/8896518",
      "blockCost": "Claude can't read your page for a user who asks, which Anthropic says may reduce your visibility in user-directed search.",
      "observedSharePct": 4.2
    },
    {
      "token": "PerplexityBot",
      "operator": "Perplexity",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Surfaces and links websites in Perplexity's search results. Perplexity says it is not used to train foundation models.",
      "userAgent": "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
      "respectsRobotsTxt": "Yes (changes take up to 24 hours)",
      "verify": "Published IP list",
      "verifyUrl": "https://www.perplexity.ai/perplexitybot.json",
      "docs": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "blockCost": "You stop appearing as a cited source in Perplexity answers.",
      "observedSharePct": 6.9
    },
    {
      "token": "Perplexity-User",
      "operator": "Perplexity",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Visits a page when a Perplexity user's question needs it, and links to it in the answer.",
      "userAgent": "Mozilla/5.0 (compatible; Perplexity-User/1.0; +https://perplexity.ai/perplexity-user)",
      "respectsRobotsTxt": "Generally ignores robots.txt, because a user asked for the fetch",
      "verify": "Published IP list",
      "verifyUrl": "https://www.perplexity.ai/perplexity-user.json",
      "docs": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "blockCost": "A robots.txt block is unlikely to stop it. Blocking at the firewall stops Perplexity reading you live for a user.",
      "observedSharePct": 2.1
    },
    {
      "token": "Googlebot",
      "operator": "Google",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Google Search's crawler. AI Overviews and AI Mode are built from the same index, so this is also Google's AI-answer crawler.",
      "userAgent": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/W.X.Y.Z Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
      "respectsRobotsTxt": "Yes (Google does not support Crawl-delay)",
      "verify": "Reverse DNS to googlebot.com, or Google's IP list",
      "verifyUrl": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "blockCost": "You disappear from Google Search, AI Overviews and AI Mode. Never block it to avoid AI: use nosnippet or max-snippet instead.",
      "observedSharePct": null
    },
    {
      "token": "Google-Extended",
      "operator": "Google",
      "purpose": "token",
      "purposeLabel": "robots.txt token only",
      "what": "Not a crawler. A robots.txt token that controls whether content Google already crawls may be used for Gemini training and grounding.",
      "userAgent": "None: Google says it has no separate HTTP user agent",
      "respectsRobotsTxt": "It is a robots.txt rule",
      "verify": "Nothing to verify: any request claiming to be Google-Extended is not from Google",
      "verifyUrl": null,
      "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "blockCost": "Google says it does not affect inclusion or ranking in Google Search. It limits use of your content by Gemini and other Google AI systems. Google says it intends to support the shared __usage_ai-training group in early 2027.",
      "observedSharePct": null
    },
    {
      "token": "Google-CloudVertexBot",
      "operator": "Google",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Crawls sites whose owners asked for it while building Vertex AI agents.",
      "userAgent": "Contains “Google-CloudVertexBot”",
      "respectsRobotsTxt": "Yes (also obeys Googlebot rules)",
      "verify": "Google's crawler IP ranges",
      "verifyUrl": "https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
      "docs": "https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers",
      "blockCost": "Google says it has no effect on Google Search or other products.",
      "observedSharePct": 0.3
    },
    {
      "token": "Applebot",
      "operator": "Apple",
      "purpose": "mixed",
      "purposeLabel": "Mixed use",
      "what": "Powers search in Spotlight, Siri and Safari. Apple says crawled data may also help train its foundation models and ground AI answers.",
      "userAgent": "Safari-style string ending (Applebot/0.1; +http://www.apple.com/go/applebot)",
      "respectsRobotsTxt": "Yes. With no Applebot group it follows your Googlebot rules, and it ignores Crawl-delay",
      "verify": "Reverse DNS to applebot.apple.com, or Apple's IP list",
      "verifyUrl": "https://search.developer.apple.com/applebot.json",
      "docs": "https://support.apple.com/en-us/HT6619",
      "blockCost": "You drop out of Siri, Spotlight and Safari suggestions. To opt out of training only, block Applebot-Extended instead.",
      "observedSharePct": null
    },
    {
      "token": "Applebot-Extended",
      "operator": "Apple",
      "purpose": "token",
      "purposeLabel": "robots.txt token only",
      "what": "Not a crawler. A robots.txt token that controls whether Applebot's data trains Apple's foundation models.",
      "userAgent": "None: Apple says Applebot-Extended does not crawl webpages",
      "respectsRobotsTxt": "It is a robots.txt rule",
      "verify": "Nothing to verify",
      "verifyUrl": null,
      "docs": "https://support.apple.com/en-us/HT6619",
      "blockCost": "Apple says it does not affect search ranking. It only opts you out of foundation-model training.",
      "observedSharePct": null
    },
    {
      "token": "Bingbot",
      "operator": "Microsoft",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Bing's crawler. Microsoft Copilot's web answers are grounded in the Bing index, and there is no separate Copilot token.",
      "userAgent": "Contains “bingbot/2.0; +http://www.bing.com/bingbot.htm”",
      "respectsRobotsTxt": "Yes",
      "verify": "Reverse DNS to search.msn.com, or Bing's IP list",
      "verifyUrl": "https://www.bing.com/toolbox/bingbot.json",
      "docs": "https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0",
      "blockCost": "You lose Bing, Copilot, and the engines that use Bing's index. Rarely worth it.",
      "observedSharePct": null
    },
    {
      "token": "meta-externalagent",
      "operator": "Meta",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "Crawls for uses such as training Meta's foundation AI models or indexing content directly.",
      "userAgent": "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
      "respectsRobotsTxt": "Meta tells you to block it with a robots.txt Disallow",
      "verify": "Meta's AS32934 route list (whois), shared by all Meta crawlers",
      "verifyUrl": null,
      "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "blockCost": "Keeps your pages out of Meta AI training. Link previews on Facebook and Instagram use a different agent (facebookexternalhit).",
      "observedSharePct": 7.2
    },
    {
      "token": "meta-externalfetcher",
      "operator": "Meta",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Fetches individual links when a user asks, for Meta's agentic AI features.",
      "userAgent": "meta-externalfetcher/1.1",
      "respectsRobotsTxt": "May bypass robots.txt, per Meta",
      "verify": "Meta's AS32934 route list (whois), shared by all Meta crawlers",
      "verifyUrl": null,
      "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "blockCost": "Robots.txt may not stop it. Low volume in our data.",
      "observedSharePct": null
    },
    {
      "token": "Amazonbot",
      "operator": "Amazon",
      "purpose": "mixed",
      "purposeLabel": "Mixed use",
      "what": "Used to improve Amazon products and services. Amazon says it may be used to train Amazon AI models.",
      "userAgent": "Contains “compatible; Amazonbot/0.1”",
      "respectsRobotsTxt": "Yes (no Crawl-delay; may use a cached robots.txt up to 30 days old)",
      "verify": "Published IP list (on the docs page)",
      "verifyUrl": "https://developer.amazon.com/amazonbot/ip-addresses/",
      "docs": "https://developer.amazon.com/amazonbot",
      "blockCost": "Keeps you out of Amazonbot's crawl, including possible training. Alexa search and live answers use Amzn-SearchBot and Amzn-User.",
      "observedSharePct": null
    },
    {
      "token": "Amzn-SearchBot",
      "operator": "Amazon",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Improves search experiences such as Alexa. Amazon says it does not gather content for generative AI training.",
      "userAgent": "Contains “compatible; Amzn-SearchBot/0.1”",
      "respectsRobotsTxt": "Yes",
      "verify": "Published IP list (on the docs page)",
      "verifyUrl": "https://developer.amazon.com/amazonbot/searchbot-ip-addresses/",
      "docs": "https://developer.amazon.com/amazonbot",
      "blockCost": "You may drop out of Alexa answers that rely on web search.",
      "observedSharePct": null
    },
    {
      "token": "Amzn-User",
      "operator": "Amazon",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Fetches live web information when an Alexa user asks something that needs it.",
      "userAgent": "Contains “compatible; Amzn-User/0.1”",
      "respectsRobotsTxt": "May not follow all directives (user-initiated)",
      "verify": "Published IP list (on the docs page)",
      "verifyUrl": "https://developer.amazon.com/amazonbot/live-ip-addresses/",
      "docs": "https://developer.amazon.com/amazonbot",
      "blockCost": "Alexa can't read your page live for a user.",
      "observedSharePct": null
    },
    {
      "token": "Bytespider",
      "operator": "ByteDance",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "ByteDance's crawler. ByteDance publishes no crawler documentation we could find; it is widely described as collecting data for its AI models.",
      "userAgent": "Contains “Bytespider”",
      "respectsRobotsTxt": "Not documented by ByteDance",
      "verify": "No published method",
      "verifyUrl": null,
      "docs": "",
      "blockCost": "Little known upside to allowing it. If robots.txt isn't respected, a firewall rule is the only reliable block.",
      "observedSharePct": 4.5
    },
    {
      "token": "CCBot",
      "operator": "Common Crawl",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "Builds Common Crawl's open web archive, a dataset widely used to train language models.",
      "userAgent": "CCBot/2.0 (https://commoncrawl.org/faq/)",
      "respectsRobotsTxt": "Yes, including Crawl-delay; backs off on 429 and 5xx",
      "verify": "Reverse DNS to crawl.commoncrawl.org, or the IP list",
      "verifyUrl": "https://index.commoncrawl.org/ccbot.json",
      "docs": "https://commoncrawl.org/faq",
      "blockCost": "Your pages stay out of future Common Crawl snapshots, and so out of the many models trained on them.",
      "observedSharePct": 2.2
    },
    {
      "token": "DuckAssistBot",
      "operator": "DuckDuckGo",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Crawls in real time for DuckDuckGo's AI-assisted answers, which cite sources. DuckDuckGo says the data is not used to train AI models.",
      "userAgent": "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
      "respectsRobotsTxt": "Yes (changes take 72 hours)",
      "verify": "Published IP list",
      "verifyUrl": "https://duckduckgo.com/duckassistbot.json",
      "docs": "https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot",
      "blockCost": "You lose citations in DuckDuckGo's AI answers. DuckDuckGo says organic rankings are unaffected.",
      "observedSharePct": 0.5
    },
    {
      "token": "MistralAI-User",
      "operator": "Mistral",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Visits a page when a user of Mistral's assistant asks a question, and links to the source. Mistral says it is not used for training.",
      "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-User/1.0; +https://docs.mistral.ai/robots)",
      "respectsRobotsTxt": "Mistral documents it as robots.txt-controllable",
      "verify": "Published IP list",
      "verifyUrl": "https://mistral.ai/mistralai-user-ips.json",
      "docs": "https://docs.mistral.ai/robots",
      "blockCost": "Mistral's assistant can't read or cite your page live.",
      "observedSharePct": 0.2
    },
    {
      "token": "MistralAI-Index",
      "operator": "Mistral",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Crawls automatically to build Mistral's search index. Mistral says this content is not used for training.",
      "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Index/1.0; +https://docs.mistral.ai/robots)",
      "respectsRobotsTxt": "Not stated by Mistral",
      "verify": "Published IP list",
      "verifyUrl": "https://mistral.ai/mistralai-index-ips.json",
      "docs": "https://docs.mistral.ai/robots",
      "blockCost": "You drop out of Mistral's search index.",
      "observedSharePct": null
    },
    {
      "token": "MistralAI-Training",
      "operator": "Mistral",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "Crawls to build datasets for training Mistral's models.",
      "userAgent": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MistralAI-Training/1.0; +https://docs.mistral.ai/robots)",
      "respectsRobotsTxt": "Yes",
      "verify": "No IP list published",
      "verifyUrl": null,
      "docs": "https://docs.mistral.ai/robots",
      "blockCost": "Keeps your future pages out of Mistral training data.",
      "observedSharePct": null
    },
    {
      "token": "cohere-ai",
      "operator": "Cohere",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "A user agent seen on many sites (including ours) and attributed to Cohere. We could not find official Cohere documentation for it.",
      "userAgent": "Contains “cohere-ai” (unverified)",
      "respectsRobotsTxt": "Not documented by Cohere",
      "verify": "No published method",
      "verifyUrl": null,
      "docs": "",
      "blockCost": "Unknown. Low volume in our data.",
      "observedSharePct": 0.3
    },
    {
      "token": "__usage_ai-training",
      "operator": "Shared (IETF AIPREF draft)",
      "purpose": "token",
      "purposeLabel": "robots.txt token only",
      "what": "Not a crawler. One robots.txt group that opts your pages out of AI training for every operator that supports it, from the IETF draft draft-kelley-aipref-training-preference-signal.",
      "userAgent": "None: it is a robots.txt group name, not a user agent",
      "respectsRobotsTxt": "It is a robots.txt rule. Anthropic honours it since 30 Oct 2026; Microsoft says from 1 Feb 2027; Google says early 2027; OpenAI has said it will follow, with no date",
      "verify": "Nothing to verify: no request carries this name",
      "verifyUrl": null,
      "docs": "https://datatracker.ietf.org/doc/draft-kelley-aipref-training-preference-signal/",
      "blockCost": "None for search: it only expresses a training preference. Anthropic says a ClaudeBot group still wins over it; other operators haven't said how they combine the two.",
      "observedSharePct": null
    },
    {
      "token": "meta-webindexer",
      "operator": "Meta",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Meta's crawler for Meta AI search results. Meta says allowing it helps Meta AI cite and link to your content.",
      "userAgent": "Contains “meta-webindexer/1.1”",
      "respectsRobotsTxt": "Yes",
      "verify": "Meta's AS32934 route list (whois), shared by all Meta crawlers",
      "verifyUrl": null,
      "docs": "https://developers.facebook.com/docs/sharing/webmasters/web-crawlers",
      "blockCost": "Meta AI is less able to cite and link to your pages. Keep it open if you want Meta AI answers to mention you.",
      "observedSharePct": null
    },
    {
      "token": "YouBot",
      "operator": "You.com",
      "purpose": "search",
      "purposeLabel": "AI search index",
      "what": "Crawler for the You.com search engine.",
      "userAgent": "Contains “compatible; YouBot/1.0; +https://docs.you.com/youbot”",
      "respectsRobotsTxt": "Yes, including Crawl-delay (caches robots.txt for 30 minutes)",
      "verify": "Reverse DNS to youbot-*.search.you.com, or Web Bot Auth signatures",
      "verifyUrl": null,
      "docs": "https://you.com/docs/youbot",
      "blockCost": "You drop out of You.com search.",
      "observedSharePct": 0.7
    },
    {
      "token": "Claude-Agent",
      "operator": "Anthropic",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Browsers Anthropic hosts so Claude can carry out tasks on a website for a user.",
      "userAgent": "Contains “compatible; Claude-Agent/1.0”",
      "respectsRobotsTxt": "Not stated by Anthropic for this agent",
      "verify": "Published IP list",
      "verifyUrl": "https://claude.com/.well-known/sail",
      "docs": "https://support.claude.com/en/articles/18234133",
      "blockCost": "Claude can't complete tasks on your site for a user, such as filling in a form.",
      "observedSharePct": null
    },
    {
      "token": "Google-Agent",
      "operator": "Google",
      "purpose": "user",
      "purposeLabel": "User-triggered fetch",
      "what": "Agents running on Google infrastructure that browse and act on sites at a user's request.",
      "userAgent": "Contains “Google-Agent”",
      "respectsRobotsTxt": "Generally ignores robots.txt, because a user started the request",
      "verify": "Published IP list",
      "verifyUrl": "https://developers.google.com/static/crawling/ipranges/user-triggered-agents.json",
      "docs": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-user-triggered-fetchers",
      "blockCost": "Google's agents can't act on your site for a user.",
      "observedSharePct": null
    },
    {
      "token": "AI2Bot",
      "operator": "Allen Institute for AI",
      "purpose": "training",
      "purposeLabel": "Model training",
      "what": "Collects pages used to train AI2's open language models.",
      "userAgent": "Mozilla/5.0 (compatible) AI2Bot (+https://www.allenai.org/crawler)",
      "respectsRobotsTxt": "Not stated by AI2",
      "verify": "No published method",
      "verifyUrl": null,
      "docs": "https://allenai.org/crawler",
      "blockCost": "Keeps your pages out of AI2's open-model training data.",
      "observedSharePct": null
    }
  ]
}