{
  "schema": "ofb-ai-crawlers-dataset-v1",
  "title": "OpenForBots AI crawler and data-use reference dataset",
  "description": "Provider-documented AI search crawlers, model-training crawlers, user-triggered fetchers, general search crawlers, and data-use controls, each with its purpose, policy token, and primary source.",
  "canonicalUrl": "https://openforbots.com/data/ai-crawlers.json",
  "csvUrl": "https://openforbots.com/data/ai-crawlers.csv",
  "registryUrl": "https://openforbots.com/ai-crawlers/",
  "lastVerifiedAt": "2026-10-04",
  "recordCount": 12,
  "publisher": {
    "name": "OpenForBots",
    "url": "https://openforbots.com/"
  },
  "reuse": "Cite OpenForBots and link the canonical record URL when you reuse this data. Use is subject to https://openforbots.com/terms/. Provider names, crawler names, and linked documentation belong to their respective owners.",
  "limitations": [
    "This is a maintained reference of documented crawler identities, not a complete list of web crawlers.",
    "OpenForBots documents provider-published information. It does not register, authenticate, or certify third-party crawlers.",
    "A user-agent string can be spoofed. Use the provider’s published verification method, where one exists, before trusting a request.",
    "null means the value is not documented by the provider or not yet verified by OpenForBots; it does not mean false.",
    "Provider documentation can change after lastVerifiedAt. Check officialDocumentationUrl for the current source."
  ],
  "fields": {
    "purpose": "search_discovery | model_training | user_triggered_retrieval | data_use_control | general_search | mixed_or_other",
    "relevantToSearchVisibility": "direct | indirect | not_direct | null (unknown)",
    "crawlsPages": "true | false | null (unknown)",
    "obeysRobotsTxt": "documented | true | false | null (unknown)"
  },
  "crawlers": [
    {
      "id": "applebot",
      "slug": "applebot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/applebot/",
      "provider": "Apple",
      "displayName": "Applebot",
      "robotsToken": "Applebot",
      "requestUserAgentExample": null,
      "purpose": [
        "search_discovery"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://support.apple.com/en-us/119829",
      "publishedIpRangesUrl": null,
      "verificationMethod": null,
      "directAnswer": null,
      "doesNotControl": [],
      "commonMisconceptions": [
        "That Applebot-Extended performs the page crawl."
      ],
      "limitations": [],
      "lastVerifiedAt": "2026-08-01",
      "revision": "1.0"
    },
    {
      "id": "applebot-extended",
      "slug": "applebot-extended",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/applebot-extended/",
      "provider": "Apple",
      "displayName": "Applebot-Extended",
      "robotsToken": "Applebot-Extended",
      "requestUserAgentExample": null,
      "purpose": [
        "data_use_control"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "not_direct",
      "crawlsPages": false,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://support.apple.com/en-us/119829",
      "publishedIpRangesUrl": null,
      "verificationMethod": null,
      "directAnswer": null,
      "doesNotControl": [],
      "commonMisconceptions": [
        "That Applebot-Extended is an ordinary crawler or that blocking it removes a site from Apple search experiences."
      ],
      "limitations": [],
      "lastVerifiedAt": "2026-08-01",
      "revision": "1.0"
    },
    {
      "id": "chatgpt-user",
      "slug": "chatgpt-user",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/chatgpt-user/",
      "provider": "OpenAI",
      "displayName": "ChatGPT-User",
      "robotsToken": "ChatGPT-User",
      "requestUserAgentExample": null,
      "purpose": [
        "user_triggered_retrieval"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "indirect",
      "crawlsPages": true,
      "obeysRobotsTxt": null,
      "status": "active",
      "officialDocumentationUrl": "https://developers.openai.com/api/docs/bots",
      "publishedIpRangesUrl": null,
      "verificationMethod": null,
      "directAnswer": "ChatGPT-User identifies certain retrieval performed in response to a user's action in ChatGPT or a Custom GPT. OpenAI documents it separately from OAI-SearchBot and GPTBot and says robots.txt rules may not apply to these user-initiated actions, so a matching robots.txt group should not be presented as a guaranteed enforcement verdict.",
      "doesNotControl": [
        "OAI-SearchBot discovery or GPTBot model-training crawling.",
        "Whether ChatGPT will answer a question, cite the page, or use a third-party search result.",
        "Access to content protected by authentication or other real security controls."
      ],
      "commonMisconceptions": [
        "ChatGPT-User is the same crawler as GPTBot or OAI-SearchBot.",
        "A ChatGPT-User Allow or Disallow rule is guaranteed to control every user-requested fetch."
      ],
      "limitations": [
        "OpenAI states that robots.txt rules may not apply to ChatGPT-User because the actions are initiated by a user.",
        "A robots.txt evaluation therefore cannot establish whether a particular ChatGPT-User request will be allowed or blocked in production.",
        "A user-agent header can be spoofed and does not authenticate a requester.",
        "Network, WAF, authentication, consent, and product-level restrictions may still affect retrieval."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "claude-searchbot",
      "slug": "claude-searchbot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/claude-searchbot/",
      "provider": "Anthropic",
      "displayName": "Claude-SearchBot",
      "robotsToken": "Claude-SearchBot",
      "requestUserAgentExample": null,
      "purpose": [
        "search_discovery"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "publishedIpRangesUrl": "https://claude.com/crawling/bots.json",
      "verificationMethod": "Compare the request source with Anthropic's current published crawler IP list as supporting evidence; keep the robots.txt policy check separate from request authentication.",
      "directAnswer": "Claude-SearchBot is Anthropic's crawler for improving the relevance and accuracy of Claude search results. Anthropic documents it separately from ClaudeBot and Claude-User, says its bots honor robots.txt, and publishes a crawler source-IP list for verification support.",
      "doesNotControl": [
        "ClaudeBot model-training collection.",
        "Claude-User requests made at a user's direction.",
        "Citation, ranking, recommendation, or inclusion in a particular Claude answer."
      ],
      "commonMisconceptions": [
        "A rule for ClaudeBot automatically applies to Claude-SearchBot.",
        "Allowing Claude-SearchBot guarantees that Claude will cite or recommend the site."
      ],
      "limitations": [
        "OpenForBots can inspect the published directive but cannot verify Anthropic's indexing state.",
        "Anthropic publishes a crawler source-IP list, but ranges can change and should be checked live when request authenticity matters.",
        "WAF, CDN, network, or server behaviour may block access independently of robots.txt."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "claude-user",
      "slug": "claude-user",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/claude-user/",
      "provider": "Anthropic",
      "displayName": "Claude-User",
      "robotsToken": "Claude-User",
      "requestUserAgentExample": null,
      "purpose": [
        "user_triggered_retrieval"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "indirect",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "publishedIpRangesUrl": "https://claude.com/crawling/bots.json",
      "verificationMethod": "Use Anthropic's current crawler IP list as supporting verification evidence alongside the user-agent string.",
      "directAnswer": "Claude-User is Anthropic's user-triggered retrieval agent. Anthropic says its named bots honor robots.txt and that disabling Claude-User prevents retrieval in response to a user query, so its control semantics differ from ChatGPT-User and Perplexity-User.",
      "doesNotControl": [],
      "commonMisconceptions": [
        "That Claude-User is a search-index crawler.",
        "That all user-triggered agents across providers have the same robots.txt semantics."
      ],
      "limitations": [
        "A robots.txt rule still does not authenticate a request or prove a provider attempted retrieval.",
        "Anthropic's published crawler IP list is time-sensitive and should be checked live when request authenticity matters."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "claudebot",
      "slug": "claudebot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/claudebot/",
      "provider": "Anthropic",
      "displayName": "ClaudeBot",
      "robotsToken": "ClaudeBot",
      "requestUserAgentExample": null,
      "purpose": [
        "model_training"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "not_direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler",
      "publishedIpRangesUrl": "https://claude.com/crawling/bots.json",
      "verificationMethod": "Use Anthropic's current crawler IP list as supporting verification evidence alongside the user-agent string.",
      "directAnswer": null,
      "doesNotControl": [],
      "commonMisconceptions": [
        "That ClaudeBot controls every Claude product surface."
      ],
      "limitations": [],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "google-extended",
      "slug": "google-extended",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/google-extended/",
      "provider": "Google",
      "displayName": "Google-Extended",
      "robotsToken": "Google-Extended",
      "requestUserAgentExample": null,
      "purpose": [
        "data_use_control"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "not_direct",
      "crawlsPages": false,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers#google-extended",
      "publishedIpRangesUrl": "https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers",
      "verificationMethod": null,
      "directAnswer": "Google-Extended is a standalone robots.txt product token, not a separate page crawler. Google says it controls whether content already crawled by Google may be used for future Gemini model training and certain grounding uses, and that it does not affect inclusion or ranking in Google Search.",
      "doesNotControl": [
        "Googlebot crawling or a site's inclusion in Google Search.",
        "Eligibility for AI Overviews or AI Mode; Google documents Googlebot and standard Search controls for those features.",
        "A separate HTTP request identity, because Google-Extended has no separate user-agent string."
      ],
      "commonMisconceptions": [
        "Google-Extended crawls pages with a separate HTTP user-agent.",
        "Blocking Google-Extended removes a site from Google Search, AI Overviews, or AI Mode."
      ],
      "limitations": [
        "OpenForBots can read the published preference but cannot determine how specific content has been used by Google systems.",
        "Because Google-Extended is a control token rather than a separate crawler identity, it should not be verified by looking for a Google-Extended HTTP user-agent.",
        "Google product documentation can change; this reference requires periodic re-verification."
      ],
      "lastVerifiedAt": "2026-08-01",
      "revision": "1.0"
    },
    {
      "id": "googlebot",
      "slug": "googlebot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/googlebot/",
      "provider": "Google",
      "displayName": "Googlebot",
      "robotsToken": "Googlebot",
      "requestUserAgentExample": null,
      "purpose": [
        "general_search"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://developers.google.com/search/docs/crawling-indexing/googlebot",
      "publishedIpRangesUrl": "https://developers.google.com/search/apis/ipranges/googlebot.json",
      "verificationMethod": null,
      "directAnswer": null,
      "doesNotControl": [],
      "commonMisconceptions": [
        "That Google-Extended replaces Googlebot's crawl."
      ],
      "limitations": [],
      "lastVerifiedAt": "2026-08-01",
      "revision": "1.0"
    },
    {
      "id": "gptbot",
      "slug": "gptbot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/gptbot/",
      "provider": "OpenAI",
      "displayName": "GPTBot",
      "robotsToken": "GPTBot",
      "requestUserAgentExample": null,
      "purpose": [
        "model_training"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "not_direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://developers.openai.com/api/docs/bots",
      "publishedIpRangesUrl": null,
      "verificationMethod": null,
      "directAnswer": "GPTBot is OpenAI's robots.txt token for web content that may be used to train its generative AI foundation models. Disallowing GPTBot indicates that site content should not be used for that training crawl; it is not, by itself, a request to remove the site from ChatGPT search.",
      "doesNotControl": [
        "ChatGPT search discovery; OAI-SearchBot is the separate Search crawler token.",
        "User-triggered retrieval through ChatGPT-User.",
        "Whether a page is already present in an existing model or third-party search index."
      ],
      "commonMisconceptions": [
        "Blocking GPTBot automatically blocks ChatGPT search.",
        "GPTBot activity proves that a page is being cited in ChatGPT answers."
      ],
      "limitations": [
        "robots.txt rules apply prospectively to compliant crawler requests and do not remove information already obtained elsewhere.",
        "OpenForBots cannot determine whether a provider used a particular page in training.",
        "A user-agent string can be spoofed; robots.txt is not proof that a request came from OpenAI."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "oai-searchbot",
      "slug": "oai-searchbot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/oai-searchbot/",
      "provider": "OpenAI",
      "displayName": "OAI-SearchBot",
      "robotsToken": "OAI-SearchBot",
      "requestUserAgentExample": null,
      "purpose": [
        "search_discovery"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://developers.openai.com/api/docs/bots",
      "publishedIpRangesUrl": null,
      "verificationMethod": null,
      "directAnswer": "OAI-SearchBot is OpenAI's robots.txt token for the crawler used to surface websites in ChatGPT search results. OpenAI recommends allowing it for Search visibility and says sites opted out will not be shown in ChatGPT search answers, though they may still appear as navigational links. Allowing access still does not guarantee selection or citation for a particular query.",
      "doesNotControl": [
        "Potential use of content for model training; GPTBot is the separate token for that policy.",
        "Whether a user-triggered fetch succeeds in every ChatGPT workflow.",
        "Selection, ranking, citation, recommendation, or referral traffic for a particular query."
      ],
      "commonMisconceptions": [
        "Blocking GPTBot also blocks OAI-SearchBot.",
        "Allowing OAI-SearchBot guarantees that a page will be indexed, cited, ranked, or recommended."
      ],
      "limitations": [
        "robots.txt expresses crawler preferences; it is not authentication or an access-control system.",
        "OpenForBots can test the published rule but cannot prove that OpenAI has indexed or will cite a page for a particular query.",
        "CDN, WAF, authentication, DNS, or server errors can still prevent retrieval even when robots.txt allows it."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "perplexity-user",
      "slug": "perplexity-user",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/perplexity-user/",
      "provider": "Perplexity",
      "displayName": "Perplexity-User",
      "robotsToken": "Perplexity-User",
      "requestUserAgentExample": null,
      "purpose": [
        "user_triggered_retrieval"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "indirect",
      "crawlsPages": true,
      "obeysRobotsTxt": false,
      "status": "active",
      "officialDocumentationUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "publishedIpRangesUrl": "https://www.perplexity.com/perplexity-user.json",
      "verificationMethod": "Combine the Perplexity-User user-agent string with Perplexity's current published IP ranges when verifying legitimate traffic.",
      "directAnswer": "Perplexity-User supports user-requested retrieval in Perplexity. Perplexity says it is not used for automatic web crawling or foundation-model training and generally ignores robots.txt because the user requested the fetch. The provider publishes a separate IP-range endpoint for verifying legitimate Perplexity-User traffic.",
      "doesNotControl": [],
      "commonMisconceptions": [
        "That Perplexity-User and PerplexityBot have identical policy semantics.",
        "That a matching Perplexity-User robots.txt rule reliably controls every user-requested fetch."
      ],
      "limitations": [
        "A robots.txt parser can show the site's declared Perplexity-User rule, but the provider says this fetcher generally ignores robots.txt.",
        "Published IP ranges are time-sensitive and should be checked live for WAF or request-verification decisions."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    },
    {
      "id": "perplexitybot",
      "slug": "perplexitybot",
      "canonicalUrl": "https://openforbots.com/ai-crawlers/perplexitybot/",
      "provider": "Perplexity",
      "displayName": "PerplexityBot",
      "robotsToken": "PerplexityBot",
      "requestUserAgentExample": "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
      "purpose": [
        "search_discovery"
      ],
      "purposeConfidence": "documented",
      "relevantToSearchVisibility": "direct",
      "crawlsPages": true,
      "obeysRobotsTxt": "documented",
      "status": "active",
      "officialDocumentationUrl": "https://docs.perplexity.ai/docs/resources/perplexity-crawlers",
      "publishedIpRangesUrl": "https://www.perplexity.com/perplexitybot.json",
      "verificationMethod": "Compare the request source with Perplexity's current published IP ranges as well as the user-agent string.",
      "directAnswer": "PerplexityBot is Perplexity's crawler for surfacing and linking websites in Perplexity search results. Perplexity documents that it is not used to crawl content for foundation-model training and publishes IP ranges that can help site owners verify legitimate requests.",
      "doesNotControl": [
        "Perplexity-User requests initiated by a person using the product.",
        "Whether a page will be indexed, selected, quoted, ranked, or cited in an answer.",
        "Requests that only spoof the PerplexityBot user-agent string."
      ],
      "commonMisconceptions": [
        "Every Perplexity-attributed request is PerplexityBot.",
        "Allowing PerplexityBot guarantees that Perplexity will cite the site."
      ],
      "limitations": [
        "Published IP ranges can change and must be treated as time-sensitive source data.",
        "OpenForBots does not verify that Perplexity has indexed a page or will use it in a response.",
        "robots.txt permission does not bypass authentication, WAF challenges, rate limits, or server failures."
      ],
      "lastVerifiedAt": "2026-10-04",
      "revision": "1.1"
    }
  ]
}