{
 "generated": "2026-08-30",
 "sample_size": 5000,
 "with_robots_txt": 2776,
 "source": "Tranco top-1M",
 "agents": {
  "gptbot": {
   "vendor": "OpenAI",
   "job": "training",
   "blocked": 577,
   "named_explicitly": 646,
   "blocked_pct": 20.79
  },
  "claudebot": {
   "vendor": "Anthropic",
   "job": "training",
   "blocked": 560,
   "named_explicitly": 568,
   "blocked_pct": 20.17
  },
  "google-extended": {
   "vendor": "Google",
   "job": "training",
   "blocked": 502,
   "named_explicitly": 529,
   "blocked_pct": 18.08
  },
  "ccbot": {
   "vendor": "Common Crawl",
   "job": "training",
   "blocked": 612,
   "named_explicitly": 555,
   "blocked_pct": 22.05
  },
  "meta-externalagent": {
   "vendor": "Meta",
   "job": "training",
   "blocked": 522,
   "named_explicitly": 432,
   "blocked_pct": 18.8
  },
  "bytespider": {
   "vendor": "ByteDance",
   "job": "training",
   "blocked": 579,
   "named_explicitly": 486,
   "blocked_pct": 20.86
  },
  "applebot-extended": {
   "vendor": "Apple",
   "job": "training",
   "blocked": 453,
   "named_explicitly": 462,
   "blocked_pct": 16.32
  },
  "amazonbot": {
   "vendor": "Amazon",
   "job": "training",
   "blocked": 469,
   "named_explicitly": 402,
   "blocked_pct": 16.89
  },
  "cohere-ai": {
   "vendor": "Cohere",
   "job": "training",
   "blocked": 393,
   "named_explicitly": 288,
   "blocked_pct": 14.16
  },
  "diffbot": {
   "vendor": "Diffbot",
   "job": "training",
   "blocked": 394,
   "named_explicitly": 263,
   "blocked_pct": 14.19
  },
  "omgili": {
   "vendor": "Webz.io",
   "job": "training",
   "blocked": 408,
   "named_explicitly": 276,
   "blocked_pct": 14.7
  },
  "timpibot": {
   "vendor": "Timpi",
   "job": "training",
   "blocked": 329,
   "named_explicitly": 183,
   "blocked_pct": 11.85
  },
  "oai-searchbot": {
   "vendor": "OpenAI",
   "job": "search",
   "blocked": 260,
   "named_explicitly": 327,
   "blocked_pct": 9.37
  },
  "perplexitybot": {
   "vendor": "Perplexity",
   "job": "search",
   "blocked": 378,
   "named_explicitly": 418,
   "blocked_pct": 13.62
  },
  "claude-searchbot": {
   "vendor": "Anthropic",
   "job": "search",
   "blocked": 267,
   "named_explicitly": 213,
   "blocked_pct": 9.62
  },
  "chatgpt-user": {
   "vendor": "OpenAI",
   "job": "user-action",
   "blocked": 319,
   "named_explicitly": 383,
   "blocked_pct": 11.49
  },
  "perplexity-user": {
   "vendor": "Perplexity",
   "job": "user-action",
   "blocked": 272,
   "named_explicitly": 197,
   "blocked_pct": 9.8
  },
  "claude-user": {
   "vendor": "Anthropic",
   "job": "user-action",
   "blocked": 268,
   "named_explicitly": 212,
   "blocked_pct": 9.65
  },
  "googlebot": {
   "vendor": "Google",
   "job": "classic-search",
   "blocked": 96,
   "named_explicitly": 386,
   "blocked_pct": 3.46
  },
  "bingbot": {
   "vendor": "Microsoft",
   "job": "classic-search",
   "blocked": 114,
   "named_explicitly": 251,
   "blocked_pct": 4.11
  }
 },
 "by_job": {
  "any_training": 739,
  "any_search": 391,
  "any_user_action": 381,
  "n": 2776
 },
 "misconfiguration": {
  "blocks_ai_search_but_allows_googlebot": 301,
  "blocks_ai_search_and_googlebot": 90,
  "blocks_gptbot_and_oai_searchbot": 256,
  "blocks_gptbot_only_correct": 321
 },
 "examples_blocking_ai_search_allowing_google": [
  {
   "domain": "facebook.com",
   "rank": 3
  },
  {
   "domain": "instagram.com",
   "rank": 11
  },
  {
   "domain": "mail.ru",
   "rank": 12
  },
  {
   "domain": "twitter.com",
   "rank": 17
  },
  {
   "domain": "amazon.com",
   "rank": 26
  },
  {
   "domain": "whatsapp.net",
   "rank": 32
  },
  {
   "domain": "x.com",
   "rank": 52
  },
  {
   "domain": "tiktok.com",
   "rank": 55
  },
  {
   "domain": "whatsapp.com",
   "rank": 56
  },
  {
   "domain": "pinterest.com",
   "rank": 57
  },
  {
   "domain": "yahoo.com",
   "rank": 59
  },
  {
   "domain": "msn.com",
   "rank": 65
  },
  {
   "domain": "baidu.com",
   "rank": 82
  },
  {
   "domain": "amazonvideo.com",
   "rank": 124
  },
  {
   "domain": "soundcloud.com",
   "rank": 173
  },
  {
   "domain": "cnn.com",
   "rank": 194
  },
  {
   "domain": "theguardian.com",
   "rank": 199
  },
  {
   "domain": "forbes.com",
   "rank": 208
  },
  {
   "domain": "bbc.com",
   "rank": 210
  },
  {
   "domain": "t-online.de",
   "rank": 211
  },
  {
   "domain": "bbc.co.uk",
   "rank": 227
  },
  {
   "domain": "t.co",
   "rank": 231
  },
  {
   "domain": "ebay.com",
   "rank": 240
  },
  {
   "domain": "imdb.com",
   "rank": 253
  },
  {
   "domain": "launchpad.net",
   "rank": 261
  },
  {
   "domain": "meraki.com",
   "rank": 263
  },
  {
   "domain": "reuters.com",
   "rank": 272
  },
  {
   "domain": "weibo.com",
   "rank": 282
  },
  {
   "domain": "weather.com",
   "rank": 304
  },
  {
   "domain": "amazon.co.uk",
   "rank": 325
  },
  {
   "domain": "wsj.com",
   "rank": 355
  },
  {
   "domain": "washingtonpost.com",
   "rank": 359
  },
  {
   "domain": "bloomberg.com",
   "rank": 361
  },
  {
   "domain": "amazon.de",
   "rank": 381
  },
  {
   "domain": "unsplash.com",
   "rank": 405
  },
  {
   "domain": "claude.ai",
   "rank": 436
  },
  {
   "domain": "sohu.com",
   "rank": 453
  },
  {
   "domain": "cnbc.com",
   "rank": 465
  },
  {
   "domain": "statista.com",
   "rank": 470
  },
  {
   "domain": "quora.com",
   "rank": 488
  }
 ]
}