{ "metadata": { "title": "When AI Comes to Your Website", "subtitle": "A Behavioral Analysis of AI Crawlers in the Wild", "analysis_date": "2026-02-01T08:18:33.313152", "total_visits": 575788, "brands": 84, "unique_urls": 314501, "date_range": "2025-06-11 to 2026-02-01", "exclusions": "Dominant e-commerce brand excluded to ensure generalizable patterns" }, "findings": [ { "id": 1, "title": "OpenAI Dominates", "stat": "72%", "detail": "GPTBot + OAI-SearchBot account for 72% of all AI crawler traffic", "validated": true, "visualization": { "type": "donut", "data": { "GPTBot": 57.2, "OAI-SearchBot": 15.1, "Bytespider": 9.2, "Meta-ExternalFetcher": 7.9, "Amazonbot": 6.7, "ClaudeBot": 3.8, "MistralAI-User": 0.1 } } }, { "id": 2, "title": "Claude Prefers Homepages", "stat": "7x", "detail": "ClaudeBot visits homepages 7x more than GPTBot (controlled for same 38 brands)", "validated": true, "visualization": { "type": "comparison_bar", "data": { "ClaudeBot": 19.2, "GPTBot": 2.8 }, "unit": "% of visits to homepage" } }, { "id": 3, "title": "Weekends Are Peak", "stat": "+29%", "detail": "GPTBot is 29% more active on weekends than weekdays", "validated": true, "visualization": { "type": "grouped_bar", "data": { "GPTBot": { "weekday_avg": 1430, "weekend_avg": 1841, "ratio": 1.29 }, "OAI-SearchBot": { "weekday_avg": 383, "weekend_avg": 540, "ratio": 1.41 }, "ClaudeBot": { "weekday_avg": 99, "weekend_avg": 91, "ratio": 0.92 } }, "note": "Opposite of traditional web traffic patterns" } }, { "id": 4, "title": "Blog Entry Points", "stat": "21%", "detail": "1 in 5 OAI-SearchBot crawl sessions start on blog pages", "validated": true, "visualization": { "type": "pie", "data": { "Blog": 21, "Product": 3, "Homepage": 1, "Other": 75 } } }, { "id": 5, "title": "One-and-Done", "stat": "1 visit", "detail": "Most pages visited exactly once (median = 1)", "validated": true, "visualization": { "type": "histogram", "data": { "1": 88.5, "10+": 0.3, "2": 8.3, "3-5": 2.4, "6-10": 0.4 }, "stats": { "median": 1, "p90": 2, "p99": 5 } } }, { "id": 6, "title": "The 3-Click Rule", "stat": "52%", "detail": "GPTBot visits Depth 3 pages (e.g. /blog/2024/post) 52% of the time", "validated": true, "note": "Depth 0 = homepage (/), Depth 1 = /about, Depth 2 = /blog/post, Depth 3 = /blog/2024/post", "visualization": { "type": "stacked_bar", "data": { "GPTBot": { "Depth 0": 2.7, "Depth 1": 10.3, "Depth 2": 19.6, "Depth 3": 51.7, "Depth 4": 12.0, "Depth 5+": 3.7 }, "OAI-SearchBot": { "Depth 0": 0.1, "Depth 1": 1.3, "Depth 2": 56.3, "Depth 3": 17.0, "Depth 4": 13.7, "Depth 5+": 11.7 }, "ClaudeBot": { "Depth 0": 15.5, "Depth 1": 8.9, "Depth 2": 15.3, "Depth 3": 41.1, "Depth 4": 11.6, "Depth 5+": 7.6 } } } }, { "id": 7, "title": "SearchBot Reaches Everyone", "stat": "76%", "detail": "OAI-SearchBot visits 76% of all brands (widest reach)", "validated": true, "visualization": { "type": "coverage_bar", "data": { "OAI-SearchBot": 76.2, "GPTBot": 70.2, "ClaudeBot": 56.0 } } }, { "id": 8, "title": "Velocity Gap", "stat": "10x", "detail": "GPTBot crawls 10x faster (61 req/hr) than ClaudeBot (6 req/hr)", "validated": true, "visualization": { "type": "horizontal_bar", "data": { "ClaudeBot": 6.2, "GPTBot": 61.4, "OAI-SearchBot": 12.0 }, "unit": "requests per active hour" } } ], "new_insights": [ { "id": 9, "title": "Bytespider: NOT a Training Crawler", "stat": "96%", "detail": "ByteDance's Bytespider visits homepages 96% of the time. User-agent links to Toutiao webmaster tools - it's for content discovery, NOT LLM training.", "visualization": { "type": "pie", "data": { "Homepage": 95.9, "Deep page": 4.1 } }, "note": "Often miscategorized as AI training. Actually brand verification for Toutiao/Douyin recommendation systems." }, { "id": 10, "title": "The Triple Crown", "stat": "47%", "detail": "Only 47% of brands (35/75) get visited by ALL three major AI crawlers", "visualization": { "type": "stacked_bar", "data": { "1 crawler": 20.0, "2 crawlers": 33.3, "3 crawlers": 46.7 } } }, { "id": 11, "title": "The Evening Crawl", "stat": "3.7x", "detail": "OAI-SearchBot is 3.7x more active during US evening hours (4pm-11pm PST) than US morning hours", "visualization": { "type": "line_chart", "data": { "GPTBot": { "peak_hour_utc": 14, "peak_visits": 16999, "peak_pct": 5.2, "trough_hour_utc": 2, "hourly_data": { "0": 11982, "1": 11678, "2": 11250, "3": 12944, "4": 16172, "5": 14057, "6": 12317, "7": 12619, "8": 15373, "9": 12302, "10": 13688, "11": 12696, "12": 13690, "13": 13870, "14": 16999, "15": 14038, "16": 12770, "17": 14584, "18": 14371, "19": 13953, "20": 15705, "21": 12678, "22": 13597, "23": 16239 } }, "OAI-SearchBot": { "peak_hour_utc": 21, "peak_visits": 7770, "peak_pct": 8.9, "trough_hour_utc": 13, "hourly_data": { "0": 4097, "1": 4160, "2": 3617, "3": 6179, "4": 5856, "5": 5305, "6": 6323, "7": 4916, "8": 4920, "9": 3674, "10": 2053, "11": 2336, "12": 1411, "13": 1185, "14": 1492, "15": 2034, "16": 2700, "17": 1587, "18": 1784, "19": 2634, "20": 4137, "21": 7770, "22": 4697, "23": 2288 } }, "ClaudeBot": { "peak_hour_utc": 23, "peak_visits": 1185, "peak_pct": 5.4, "trough_hour_utc": 13, "hourly_data": { "0": 1153, "1": 1155, "2": 1062, "3": 905, "4": 1031, "5": 928, "6": 956, "7": 958, "8": 887, "9": 894, "10": 1036, "11": 784, "12": 736, "13": 675, "14": 701, "15": 692, "16": 749, "17": 712, "18": 759, "19": 933, "20": 985, "21": 1039, "22": 1159, "23": 1185 } } } } }, { "id": 12, "title": "Deep Dive Sessions", "stat": "60 pages", "detail": "GPTBot averages 60 pages per crawl session vs ClaudeBot's 5", "visualization": { "type": "comparison_bar", "data": { "ClaudeBot": 5.1, "GPTBot": 60.5, "OAI-SearchBot": 9.5 } } } ], "visualization_data": { "market_share": { "GPTBot": { "visits": 329572, "share": 57.2, "brands": 59 }, "OAI-SearchBot": { "visits": 87155, "share": 15.1, "brands": 64 }, "Bytespider": { "visits": 52704, "share": 9.2, "brands": 48 }, "Meta-ExternalFetcher": { "visits": 45445, "share": 7.9, "brands": 29 }, "Amazonbot": { "visits": 38335, "share": 6.7, "brands": 11 }, "ClaudeBot": { "visits": 22074, "share": 3.8, "brands": 47 }, "MistralAI-User": { "visits": 336, "share": 0.1, "brands": 30 } }, "homepage_preference": { "GPTBot": 2.753356065493196, "ClaudeBot": 19.17942225802739 }, "weekend_patterns": { "GPTBot": { "weekday_avg": 1430, "weekend_avg": 1841, "ratio": 1.29 }, "OAI-SearchBot": { "weekday_avg": 383, "weekend_avg": 540, "ratio": 1.41 }, "ClaudeBot": { "weekday_avg": 99, "weekend_avg": 91, "ratio": 0.92 } }, "revisit_histogram": { "1": 88.5, "10+": 0.3, "2": 8.3, "3-5": 2.4, "6-10": 0.4 }, "depth_distribution": { "GPTBot": { "Depth 0": 2.7, "Depth 1": 10.3, "Depth 2": 19.6, "Depth 3": 51.7, "Depth 4": 12.0, "Depth 5+": 3.7 }, "OAI-SearchBot": { "Depth 0": 0.1, "Depth 1": 1.3, "Depth 2": 56.3, "Depth 3": 17.0, "Depth 4": 13.7, "Depth 5+": 11.7 }, "ClaudeBot": { "Depth 0": 15.5, "Depth 1": 8.9, "Depth 2": 15.3, "Depth 3": 41.1, "Depth 4": 11.6, "Depth 5+": 7.6 } }, "coverage": { "OAI-SearchBot": { "brands_visited": 64, "total_brands": 84, "coverage": 76.2, "avg_visits_per_brand": 1362 }, "GPTBot": { "brands_visited": 59, "total_brands": 84, "coverage": 70.2, "avg_visits_per_brand": 5586 }, "ClaudeBot": { "brands_visited": 47, "total_brands": 84, "coverage": 56.0, "avg_visits_per_brand": 470 } }, "velocity": { "ClaudeBot": 6.2, "GPTBot": 61.4, "OAI-SearchBot": 12.0 }, "bytespider_behavior": { "Homepage": { "visits": 50535, "pct": 95.9, "brands": 42 }, "Deep page": { "visits": 2169, "pct": 4.1, "brands": 26 } }, "multi_crawler_coverage": { "1 crawler": { "brands": 15, "pct": 20.0 }, "2 crawlers": { "brands": 25, "pct": 33.3 }, "3 crawlers": { "brands": 35, "pct": 46.7 } }, "hour_of_day": { "GPTBot": { "peak_hour_utc": 14, "peak_visits": 16999, "peak_pct": 5.2, "trough_hour_utc": 2, "hourly_data": { "0": 11982, "1": 11678, "2": 11250, "3": 12944, "4": 16172, "5": 14057, "6": 12317, "7": 12619, "8": 15373, "9": 12302, "10": 13688, "11": 12696, "12": 13690, "13": 13870, "14": 16999, "15": 14038, "16": 12770, "17": 14584, "18": 14371, "19": 13953, "20": 15705, "21": 12678, "22": 13597, "23": 16239 } }, "OAI-SearchBot": { "peak_hour_utc": 21, "peak_visits": 7770, "peak_pct": 8.9, "trough_hour_utc": 13, "hourly_data": { "0": 4097, "1": 4160, "2": 3617, "3": 6179, "4": 5856, "5": 5305, "6": 6323, "7": 4916, "8": 4920, "9": 3674, "10": 2053, "11": 2336, "12": 1411, "13": 1185, "14": 1492, "15": 2034, "16": 2700, "17": 1587, "18": 1784, "19": 2634, "20": 4137, "21": 7770, "22": 4697, "23": 2288 } }, "ClaudeBot": { "peak_hour_utc": 23, "peak_visits": 1185, "peak_pct": 5.4, "trough_hour_utc": 13, "hourly_data": { "0": 1153, "1": 1155, "2": 1062, "3": 905, "4": 1031, "5": 928, "6": 956, "7": 958, "8": 887, "9": 894, "10": 1036, "11": 784, "12": 736, "13": 675, "14": 701, "15": 692, "16": 749, "17": 712, "18": 759, "19": 933, "20": 985, "21": 1039, "22": 1159, "23": 1185 } } }, "session_data": { "ClaudeBot": { "total_sessions": 4324, "total_visits": 22074, "avg_pages_per_session": 5.1 }, "GPTBot": { "total_sessions": 5445, "total_visits": 329572, "avg_pages_per_session": 60.5 }, "OAI-SearchBot": { "total_sessions": 9206, "total_visits": 87155, "avg_pages_per_session": 9.5 } }, "day_of_week": { "GPTBot": { "Sun": 18.8, "Mon": 15.9, "Tue": 17.6, "Wed": 12.8, "Thu": 10.0, "Fri": 9.5, "Sat": 15.2 }, "OAI-SearchBot": { "Sun": 7.3, "Mon": 20.1, "Tue": 7.9, "Wed": 11.6, "Thu": 12.9, "Fri": 13.4, "Sat": 26.8 }, "ClaudeBot": { "Sun": 12.6, "Mon": 14.2, "Tue": 14.7, "Wed": 18.2, "Thu": 15.8, "Fri": 10.3, "Sat": 14.3 } } }, "crawler_types": { "training": { "description": "Collect data to train future AI models. No direct traffic benefit.", "crawlers": [ "GPTBot", "ClaudeBot" ] }, "search": { "description": "Index content for real-time AI search. Users find your content via AI.", "crawlers": [ "OAI-SearchBot" ] }, "discovery": { "description": "Brand verification for content platforms (not LLM training).", "crawlers": [ "Bytespider" ], "note": "Bytespider is for Toutiao/Douyin content discovery, not AI training. User-agent links to zhanzhang.toutiao.com" }, "link_preview": { "description": "Generate preview cards when URLs are shared on social platforms.", "crawlers": [ "Meta-ExternalFetcher" ], "note": "User-agent is facebookexternalhit - this is Facebook/Instagram link unfurling" }, "unknown": { "description": "Purpose unclear.", "crawlers": [ "Amazonbot" ], "note": "92% deep pages, low homepage %. Could be Alexa, Amazon search, or Titan AI training" } }, "additional_insights": { "first_visit_pattern": { "description": "Crawlers start at homepage, then dive deep on return visits", "data": { "GPTBot": { "first_visit_homepage_pct": 13.6, "return_visit_homepage_pct": 2.7 }, "OAI-SearchBot": { "first_visit_homepage_pct": 10.9, "return_visit_homepage_pct": 0.1 }, "ClaudeBot": { "first_visit_homepage_pct": 14.9, "return_visit_homepage_pct": 15.5 } } }, "brand_size_threshold": { "description": "Medium brands (100-999 visits) have highest triple-crown rate", "data": { "small_under_100": { "brands": 32, "triple_crown_pct": 22 }, "medium_100_999": { "brands": 37, "triple_crown_pct": 68 } } }, "file_type_preference": { "description": "AI crawlers almost exclusively want HTML, not PDFs or other documents", "data": { "GPTBot": { "pdf_pct": 0.001, "html_pct": 99.99 }, "ClaudeBot": { "pdf_pct": 0.005, "html_pct": 99.99 } } } } }