{"api":"v1","generated_at":"2026-09-16T01:58:14.110Z","licence":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/","attribution":"TOLL, with a link to the page cited"},"page":"https://tollindex.com/e/crawl-preflight-postnov01-workers-dev-crawl-check-domain-d03236","pinned_page":"https://tollindex.com/e/crawl-preflight-postnov01-workers-dev-crawl-check-domain-d03236/at/2026-09-15T21-20Z","pinned":false,"method":"https://tollindex.com/ledger/method","snapshot_at":"2026-09-15T21:20:12.737Z","snapshot_stamp":"2026-09-15T21-20Z","endpoint":{"slug":"crawl-preflight-postnov01-workers-dev-crawl-check-domain-d03236","canonical_url":"https://crawl-preflight.postnov01.workers.dev/crawl-check/:domain","resource":"https://crawl-preflight.postnov01.workers.dev/crawl-check/:domain","http_method":"GET","type":"http","x402_version":2,"registries":["cdp"],"primary_registry":"cdp","curated_by_coinbase":false,"description":"Crawl pre-flight for a domain: will you be blocked, is content behind a paywall, and how much is actually there. Returns per-bot robots.txt verdicts for 16 AI crawlers, crawlable page count from sitemaps, detected CDN and paid-access signals (x402, TollBit), plus a fetch/skip recommendation. Call before spending requests on an unknown domain.","service_name":"Crawl Pre-Flight","declared_category":null,"declared_tags":["crawl","robots","web","preflight","agent"],"route_template":"/crawl-check/:domain","registry_updated":"2026-08-31T09:40:26.676Z"},"derived":{"note":"fields no registry supplies; TOLL derives them and marks them derived on the page","title":"crawl check","category":"scraping","first_observed_by_toll":"2026-09-10T19:08:24.178Z","last_observed_by_toll":"2026-09-15T21:20:12.737Z","observation_began":"2026-09-10"},"presence":{"listed_now":true,"first_absent_at":null,"delisting_confirmed_at":null,"events":[{"registry":"cdp","at":"2026-09-10T19:08:24.178Z","event":"listed"}]},"accepts":[{"registry":"cdp","ordinal":0,"network":"eip155:8453","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","asset_name":"USD Coin","symbol":"USDC","decimals":6,"amount_display":"0.02 USDC","amount_units":0.02,"decimals_known":true,"pay_to":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","scheme":"exact"},{"registry":"cdp","ordinal":1,"network":"eip155:137","asset":"0x3c499c542cef5e3811e1192ce70d8cc03d5c3359","asset_name":"USD Coin","symbol":"USDC","decimals":6,"amount_display":"0.02 USDC","amount_units":0.02,"decimals_known":true,"pay_to":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","scheme":"exact"},{"registry":"cdp","ordinal":2,"network":"eip155:42161","asset":"0xaf88d065e77c8cc2239327c5edb3a432268e5831","asset_name":"USD Coin","symbol":"USDC","decimals":6,"amount_display":"0.02 USDC","amount_units":0.02,"decimals_known":true,"pay_to":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","scheme":"exact"}],"counters":{"observed_at":"2026-09-15T21:20:12.737Z","calls_30d":2,"unique_payers_30d":1,"last_called_at":"2026-08-31T09:40:26.331Z"},"validation":{"latest":{"observed_at":"2026-09-15T06:36:34.142Z","state":"fail","failed_checks":["returns_402"],"endpoint_http_status":400,"run_id":"daily-2026-09-15"},"uptime":[{"days":7,"observed":5,"passed":0,"ratio":0},{"days":30,"observed":5,"passed":0,"ratio":0},{"days":90,"observed":5,"passed":0,"ratio":0}],"history_90d":[{"observed_at":"2026-09-15T06:36:34.142Z","state":"fail","failed_checks":["returns_402"],"run_id":"daily-2026-09-15","state_changed":false},{"observed_at":"2026-09-14T06:05:15.050Z","state":"fail","failed_checks":["returns_402"],"run_id":"daily-2026-09-14","state_changed":false},{"observed_at":"2026-09-13T05:57:06.176Z","state":"fail","failed_checks":["returns_402"],"run_id":"daily-2026-09-13","state_changed":false},{"observed_at":"2026-09-12T08:14:18.981Z","state":"fail","failed_checks":["returns_402"],"run_id":"daily-2026-09-12","state_changed":false},{"observed_at":"2026-09-11T12:35:26.059Z","state":"fail","failed_checks":["returns_402"],"run_id":"daily-2026-09-11","state_changed":true}]},"seller":{"wallet":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","page":"https://tollindex.com/seller/0x02d15E7f25d29743c27021d32a7B280d760B2bC2","endpoints":1,"hosts":1},"reports":0,"registry_record":{"cdp":[{"tags":["crawl","robots","web","preflight","agent"],"type":"http","accepts":[{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","amount":"20000","scheme":"exact","network":"eip155:8453","maxTimeoutSeconds":60},{"asset":"0x3c499c542cEF5E3811e1192ce70d8cC03d5c3359","extra":{"name":"USD Coin","version":"2"},"payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","amount":"20000","scheme":"exact","network":"eip155:137","maxTimeoutSeconds":60},{"asset":"0xaf88d065e77c8cC2239327C5EDb3A432268e5831","extra":{"name":"USD Coin","version":"2"},"payTo":"0x02d15E7f25d29743c27021d32a7B280d760B2bC2","amount":"20000","scheme":"exact","network":"eip155:42161","maxTimeoutSeconds":60}],"quality":{"lastCalledAt":"2026-08-31T09:40:26.331Z","l30DaysTotalCalls":2,"l30DaysUniquePayers":1},"resource":"https://crawl-preflight.postnov01.workers.dev/crawl-check/:domain","extensions":{"bazaar":{"info":{"input":{"type":"http","method":"GET","pathParams":{"domain":"sitepoint.com"},"queryParams":{}},"output":{"type":"json","example":{"domain":"example.com","reason":"No AI crawler restrictions declared.","robots":{"present":true,"blocks_all_agents":false,"ai_crawlers_allowed":16,"ai_crawlers_blocked":[]},"crawl_surface":{"pages":28400,"is_estimate":true,"sitemaps_found":2},"infrastructure":{"cdn":["Cloudflare"],"paid_access":[]},"recommendation":"fetch"}}},"schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"pathParams":{"type":"object","required":["domain"],"properties":{"domain":{"type":"string","description":"Bare domain to check, without scheme or path"}}},"queryParams":{"type":"object","properties":{}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object"}}}}},"routeTemplate":"/crawl-check/:domain"}},"description":"Crawl pre-flight for a domain: will you be blocked, is content behind a paywall, and how much is actually there. Returns per-bot robots.txt verdicts for 16 AI crawlers, crawlable page count from sitemaps, detected CDN and paid-access signals (x402, TollBit), plus a fetch/skip recommendation. Call before spending requests on an unknown domain.","lastUpdated":"2026-08-31T09:40:26.676Z","serviceName":"Crawl Pre-Flight","x402Version":2}]}}