{"publisher":"Scovant","dataset":"Scovant — Cloudflare AI-bot defaults, September 2026 edge snapshots","license":"https://creativecommons.org/licenses/by/4.0/","methodology":"https://scovant.com/research#cloudflare-2026-09","methodology_note":"Three independent snapshots of the marketing corpus around Cloudflare's 2026-09-15 AI-bot default change, taken with the same crawler identity (ScovantBot honest UA + a browser-UA baseline). This is not a before/after comparison: each two-day window reached a different set of sites (see `overlap`), and from 2026-09-21 the crawler follows redirects to the final page before judging a block, so the first two snapshots (first hop) and the third (redirects followed) measure differently. Each snapshot's shares describe that snapshot only. Edge provider labelled by a one-off header probe or by the crawler's own headers; 'none' = no CDN marker recognised. Layers under 20 domains report null. Causality is not claimed.","experiment":"cloudflare_2026_09","design":"observational","mismatch_wording":"declared crawler policy and observed edge policy differ","phases":{"before":{"taken_at":"2026-09-14T21:10:27.676247+00:00","window":["2026-09-13T02:00:00+00:00","2026-09-15T00:00:00+00:00"],"sample_size":452,"metrics":{"layers":{"none":{"n":202,"by_reason":{"captcha":3,"http_block":14},"mismatch_pct":null,"declared_any_pct":25.7,"agent_blocked_pct":8.4,"by_declared_search":{"mixed":1,"allowed":193,"blocked":4,"undeclared":4},"anonymous_blocked_pct":9.2},"other":{"n":8,"by_reason":{"http_block":2},"mismatch_pct":null,"declared_any_pct":null,"agent_blocked_pct":null,"by_declared_search":{"allowed":8},"anonymous_blocked_pct":null},"akamai":{"n":4,"by_reason":{"http_block":1},"mismatch_pct":null,"declared_any_pct":null,"agent_blocked_pct":null,"by_declared_search":{"allowed":4},"anonymous_blocked_pct":null},"fastly":{"n":6,"by_reason":{"http_block":1},"mismatch_pct":null,"declared_any_pct":null,"agent_blocked_pct":null,"by_declared_search":{"allowed":6},"anonymous_blocked_pct":null},"cloudflare":{"n":201,"by_reason":{"captcha":22,"cloudflare":13},"mismatch_pct":16.9,"declared_any_pct":42.8,"agent_blocked_pct":17.4,"by_declared_search":{"mixed":1,"allowed":197,"blocked":3},"anonymous_blocked_pct":18.9},"cloudfront":{"n":31,"by_reason":{"http_block":1},"mismatch_pct":null,"declared_any_pct":29.0,"agent_blocked_pct":3.2,"by_declared_search":{"mixed":2,"allowed":28,"blocked":1},"anonymous_blocked_pct":3.2}},"sample":452,"min_layer_n":20,"mismatch_wording":"declared crawler policy and observed edge policy differ"},"instrument":"first_hop"},"after-1":{"taken_at":"2026-09-24T13:49:33.748077+00:00","window":["2026-09-20T02:00:00+00:00","2026-09-22T00:00:00+00:00"],"sample_size":646,"metrics":{"layers":{"none":{"n":324,"by_reason":{"captcha":9,"http_block":15},"mismatch_pct":null,"declared_any_pct":19.1,"agent_blocked_pct":7.4,"by_declared_search":{"mixed":7,"allowed":302,"blocked":8,"undeclared":7},"anonymous_blocked_pct":7.6},"other":{"n":6,"by_reason":{"http_block":1},"mismatch_pct":null,"declared_any_pct":null,"agent_blocked_pct":null,"by_declared_search":{"allowed":6},"anonymous_blocked_pct":null},"akamai":{"n":33,"by_reason":{"http_block":1},"mismatch_pct":null,"declared_any_pct":0.0,"agent_blocked_pct":3.0,"by_declared_search":{"allowed":32,"undeclared":1},"anonymous_blocked_pct":3.1},"fastly":{"n":34,"by_reason":{"http_block":1},"mismatch_pct":null,"declared_any_pct":17.6,"agent_blocked_pct":2.9,"by_declared_search":{"mixed":2,"allowed":30,"blocked":1,"undeclared":1},"anonymous_blocked_pct":3.0},"cloudflare":{"n":173,"by_reason":{"captcha":24,"cloudflare":4},"mismatch_pct":15.6,"declared_any_pct":23.1,"agent_blocked_pct":16.2,"by_declared_search":{"mixed":1,"allowed":166,"blocked":3,"undeclared":3},"anonymous_blocked_pct":17.6},"cloudfront":{"n":76,"by_reason":{"captcha":2,"http_block":3},"mismatch_pct":null,"declared_any_pct":21.1,"agent_blocked_pct":6.6,"by_declared_search":{"mixed":2,"allowed":73,"blocked":1},"anonymous_blocked_pct":5.3}},"sample":646,"min_layer_n":20,"mismatch_wording":"declared crawler policy and observed edge policy differ"},"instrument":"first_hop"},"after-2":{"taken_at":"2026-09-29T12:39:32.235252+00:00","window":["2026-09-27T02:00:00+00:00","2026-09-29T00:00:00+00:00"],"sample_size":7972,"metrics":{"layers":{"none":{"n":3250,"by_reason":{"captcha":179,"http_block":200},"mismatch_pct":null,"declared_any_pct":26.5,"agent_blocked_pct":11.7,"by_declared_search":{"mixed":49,"allowed":3063,"blocked":83,"undeclared":55},"anonymous_blocked_pct":13.6},"other":{"n":158,"by_reason":{"captcha":11,"cloudflare":2,"http_block":9},"mismatch_pct":null,"declared_any_pct":22.8,"agent_blocked_pct":13.9,"by_declared_search":{"mixed":1,"allowed":156,"undeclared":1},"anonymous_blocked_pct":13.4},"akamai":{"n":362,"by_reason":{"captcha":11,"http_block":136},"mismatch_pct":null,"declared_any_pct":3.3,"agent_blocked_pct":40.6,"by_declared_search":{"allowed":360,"undeclared":2},"anonymous_blocked_pct":40.7},"fastly":{"n":295,"by_reason":{"captcha":16,"http_block":42},"mismatch_pct":null,"declared_any_pct":15.3,"agent_blocked_pct":19.7,"by_declared_search":{"mixed":8,"allowed":283,"blocked":3,"undeclared":1},"anonymous_blocked_pct":19.8},"cloudflare":{"n":2961,"by_reason":{"captcha":743,"cloudflare":242},"mismatch_pct":33.3,"declared_any_pct":26.6,"agent_blocked_pct":33.3,"by_declared_search":{"mixed":23,"allowed":2874,"blocked":42,"undeclared":22},"anonymous_blocked_pct":35.1},"cloudfront":{"n":946,"by_reason":{"captcha":46,"http_block":102},"mismatch_pct":null,"declared_any_pct":20.8,"agent_blocked_pct":15.6,"by_declared_search":{"mixed":16,"allowed":910,"blocked":16,"undeclared":4},"anonymous_blocked_pct":12.4}},"sample":7972,"min_layer_n":20,"instrument_mix":{"first_hop":85,"redirects_followed":7887},"mismatch_wording":"declared crawler policy and observed edge policy differ"},"instrument":"redirects_followed"}},"overlap":{"before∩after-1":0,"before∩after-2":439,"after-1∩after-2":550}}