{
  "slug": "ai-crawler-market-share",
  "title": "AI Crawler Market Share",
  "description": "Market share of AI crawlers by observed visits, crawler type, brands reached, and unique URLs fetched.",
  "updatedAt": "2026-06-08T13:00:11.373572+00:00",
  "generatedAt": "2026-06-12T12:30:06Z",
  "sourceUrls": [
    {
      "label": "AI crawler behavior research",
      "url": "https://storage.googleapis.com/trakkr-research/003-ai-crawler-behavior/crawler_research_data.json"
    },
    {
      "label": "Trakkr Data: crawlers",
      "url": "https://trakkr.ai/data/crawlers"
    }
  ],
  "coverage": [
    {
      "label": "Observed visits",
      "value": 16390735
    },
    {
      "label": "Brands",
      "value": 144
    },
    {
      "label": "Unique URLs",
      "value": 1838373
    }
  ],
  "heroStats": [
    {
      "label": "Observed visits",
      "value": 16390735,
      "format": "number"
    },
    {
      "label": "Tracked crawlers",
      "value": 41,
      "format": "number"
    },
    {
      "label": "Top crawler share",
      "value": 49.0,
      "format": "percent"
    }
  ],
  "table": {
    "columns": [
      {
        "key": "crawler",
        "label": "Crawler",
        "format": "text"
      },
      {
        "key": "crawler_type",
        "label": "Type",
        "format": "text"
      },
      {
        "key": "visits",
        "label": "Visits",
        "format": "number",
        "align": "right"
      },
      {
        "key": "market_share",
        "label": "Market share",
        "format": "percent",
        "align": "right"
      },
      {
        "key": "brands_seen",
        "label": "Brands seen",
        "format": "number",
        "align": "right"
      },
      {
        "key": "unique_urls",
        "label": "Unique URLs",
        "format": "number",
        "align": "right"
      }
    ]
  },
  "rows": [
    {
      "crawler": "Bytespider",
      "crawler_type": "Training crawler",
      "visits": 8030373,
      "market_share": 49.0,
      "brands_seen": 82,
      "unique_urls": 0
    },
    {
      "crawler": "ClaudeBot",
      "crawler_type": "Training crawler",
      "visits": 1736590,
      "market_share": 10.6,
      "brands_seen": 86,
      "unique_urls": 0
    },
    {
      "crawler": "ChatGPT-User",
      "crawler_type": "Conversation fetch",
      "visits": 1540508,
      "market_share": 9.4,
      "brands_seen": 40,
      "unique_urls": 0
    },
    {
      "crawler": "Applebot",
      "crawler_type": "AI search index",
      "visits": 1504844,
      "market_share": 9.2,
      "brands_seen": 28,
      "unique_urls": 0
    },
    {
      "crawler": "Amazonbot",
      "crawler_type": "Training crawler",
      "visits": 1015896,
      "market_share": 6.2,
      "brands_seen": 38,
      "unique_urls": 0
    },
    {
      "crawler": "OAI-SearchBot",
      "crawler_type": "AI search index",
      "visits": 901386,
      "market_share": 5.5,
      "brands_seen": 101,
      "unique_urls": 0
    },
    {
      "crawler": "GPTBot",
      "crawler_type": "Training crawler",
      "visits": 868367,
      "market_share": 5.3,
      "brands_seen": 122,
      "unique_urls": 0
    },
    {
      "crawler": "Scrapy",
      "crawler_type": "Unknown",
      "visits": 339806,
      "market_share": 2.1,
      "brands_seen": 12,
      "unique_urls": 0
    },
    {
      "crawler": "Meta-ExternalFetcher",
      "crawler_type": "Conversation fetch",
      "visits": 197356,
      "market_share": 1.2,
      "brands_seen": 50,
      "unique_urls": 0
    },
    {
      "crawler": "PerplexityBot",
      "crawler_type": "AI search index",
      "visits": 140159,
      "market_share": 0.9,
      "brands_seen": 36,
      "unique_urls": 0
    },
    {
      "crawler": "GoogleOther",
      "crawler_type": "Unknown",
      "visits": 16606,
      "market_share": 0.1,
      "brands_seen": 1,
      "unique_urls": 0
    },
    {
      "crawler": "Claude-User",
      "crawler_type": "Conversation fetch",
      "visits": 11841,
      "market_share": 0.1,
      "brands_seen": 20,
      "unique_urls": 0
    },
    {
      "crawler": "Meta-ExternalAgent",
      "crawler_type": "Unknown",
      "visits": 11760,
      "market_share": 0.1,
      "brands_seen": 9,
      "unique_urls": 0
    },
    {
      "crawler": "Bingbot",
      "crawler_type": "Unknown",
      "visits": 11375,
      "market_share": 0.1,
      "brands_seen": 9,
      "unique_urls": 0
    },
    {
      "crawler": "Claude-SearchBot",
      "crawler_type": "AI search index",
      "visits": 11204,
      "market_share": 0.1,
      "brands_seen": 12,
      "unique_urls": 0
    },
    {
      "crawler": "CCBot",
      "crawler_type": "Training crawler",
      "visits": 11193,
      "market_share": 0.1,
      "brands_seen": 13,
      "unique_urls": 0
    },
    {
      "crawler": "ChatGPT",
      "crawler_type": "Unknown",
      "visits": 9388,
      "market_share": 0.1,
      "brands_seen": 8,
      "unique_urls": 0
    },
    {
      "crawler": "GrokBot",
      "crawler_type": "Unknown",
      "visits": 8300,
      "market_share": 0.1,
      "brands_seen": 12,
      "unique_urls": 0
    },
    {
      "crawler": "YouBot",
      "crawler_type": "AI search index",
      "visits": 7834,
      "market_share": 0.0,
      "brands_seen": 12,
      "unique_urls": 0
    },
    {
      "crawler": "Perplexity-User",
      "crawler_type": "Conversation fetch",
      "visits": 3492,
      "market_share": 0.0,
      "brands_seen": 13,
      "unique_urls": 0
    },
    {
      "crawler": "bytespider",
      "crawler_type": "Training crawler",
      "visits": 2748,
      "market_share": 0.0,
      "brands_seen": 1,
      "unique_urls": 0
    },
    {
      "crawler": "DeepSeekBot",
      "crawler_type": "Unknown",
      "visits": 2051,
      "market_share": 0.0,
      "brands_seen": 19,
      "unique_urls": 0
    },
    {
      "crawler": "Claude",
      "crawler_type": "Unknown",
      "visits": 1661,
      "market_share": 0.0,
      "brands_seen": 8,
      "unique_urls": 0
    },
    {
      "crawler": "MistralAI-User",
      "crawler_type": "Conversation fetch",
      "visits": 1431,
      "market_share": 0.0,
      "brands_seen": 43,
      "unique_urls": 0
    },
    {
      "crawler": "Googlebot",
      "crawler_type": "Unknown",
      "visits": 911,
      "market_share": 0.0,
      "brands_seen": 1,
      "unique_urls": 0
    },
    {
      "crawler": "Perplexity",
      "crawler_type": "Unknown",
      "visits": 752,
      "market_share": 0.0,
      "brands_seen": 6,
      "unique_urls": 0
    },
    {
      "crawler": "Bing",
      "crawler_type": "Unknown",
      "visits": 682,
      "market_share": 0.0,
      "brands_seen": 3,
      "unique_urls": 0
    },
    {
      "crawler": "Amazon",
      "crawler_type": "Unknown",
      "visits": 645,
      "market_share": 0.0,
      "brands_seen": 3,
      "unique_urls": 0
    },
    {
      "crawler": "Meta AI",
      "crawler_type": "Unknown",
      "visits": 503,
      "market_share": 0.0,
      "brands_seen": 3,
      "unique_urls": 0
    },
    {
      "crawler": "cohere-ai",
      "crawler_type": "Unknown",
      "visits": 225,
      "market_share": 0.0,
      "brands_seen": 13,
      "unique_urls": 0
    },
    {
      "crawler": "Claude-Web",
      "crawler_type": "Unknown",
      "visits": 203,
      "market_share": 0.0,
      "brands_seen": 7,
      "unique_urls": 0
    },
    {
      "crawler": "Diffbot",
      "crawler_type": "Unknown",
      "visits": 177,
      "market_share": 0.0,
      "brands_seen": 10,
      "unique_urls": 0
    },
    {
      "crawler": "facebookexternalhit",
      "crawler_type": "Unknown",
      "visits": 171,
      "market_share": 0.0,
      "brands_seen": 1,
      "unique_urls": 0
    },
    {
      "crawler": "Google-Agent",
      "crawler_type": "Agent",
      "visits": 88,
      "market_share": 0.0,
      "brands_seen": 5,
      "unique_urls": 0
    },
    {
      "crawler": "AI2Bot",
      "crawler_type": "Unknown",
      "visits": 82,
      "market_share": 0.0,
      "brands_seen": 4,
      "unique_urls": 0
    },
    {
      "crawler": "anthropic-ai",
      "crawler_type": "Unknown",
      "visits": 82,
      "market_share": 0.0,
      "brands_seen": 6,
      "unique_urls": 0
    },
    {
      "crawler": "Timpibot",
      "crawler_type": "Unknown",
      "visits": 37,
      "market_share": 0.0,
      "brands_seen": 2,
      "unique_urls": 0
    },
    {
      "crawler": "MistralBot",
      "crawler_type": "Unknown",
      "visits": 21,
      "market_share": 0.0,
      "brands_seen": 7,
      "unique_urls": 0
    },
    {
      "crawler": "Mistral",
      "crawler_type": "Unknown",
      "visits": 4,
      "market_share": 0.0,
      "brands_seen": 1,
      "unique_urls": 0
    },
    {
      "crawler": "Omgili",
      "crawler_type": "Unknown",
      "visits": 3,
      "market_share": 0.0,
      "brands_seen": 2,
      "unique_urls": 0
    },
    {
      "crawler": "Google-Extended",
      "crawler_type": "Unknown",
      "visits": 2,
      "market_share": 0.0,
      "brands_seen": 2,
      "unique_urls": 0
    }
  ],
  "methodology": [
    "Crawler visits are grouped from Trakkr's AI crawler behavior research data.",
    "Crawler type is assigned from known bot patterns for training, indexing, conversation fetches, and agents.",
    "Market share is the share of observed crawler visits within the current research window."
  ],
  "notes": [
    "Some crawlers serve multiple product surfaces; type labels describe the primary observed role."
  ]
}