{"api":"v1","generated_at":"2026-10-06T04:24:29.410Z","licence":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/","attribution":"TOLL, with a link to the page cited"},"page":"https://tollindex.com/e/api-vextorium-com-v1-robots-txt-e3f838","pinned_page":"https://tollindex.com/e/api-vextorium-com-v1-robots-txt-e3f838/at/2026-10-06T00-02Z","pinned":false,"method":"https://tollindex.com/ledger/method","snapshot_at":"2026-10-06T00:02:10.461Z","snapshot_stamp":"2026-10-06T00-02Z","endpoint":{"slug":"api-vextorium-com-v1-robots-txt-e3f838","canonical_url":"https://api.vextorium.com/v1/robots-txt","resource":"https://api.vextorium.com/v1/robots-txt","http_method":"POST","type":"http","x402_version":2,"registries":["cdp"],"primary_registry":"cdp","curated_by_coinbase":false,"description":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","service_name":"Vextorium","declared_category":null,"declared_tags":["robots","crawler","ai-bots","scraping","compliance"],"route_template":null,"registry_updated":"2026-10-03T20:44:09.560Z"},"derived":{"note":"fields no registry supplies; TOLL derives them and marks them derived on the page","title":"robots txt","category":"scraping","first_observed_by_toll":"2026-10-03T21:12:52.185Z","last_observed_by_toll":"2026-10-06T00:02:10.461Z","observation_began":"2026-09-10"},"presence":{"listed_now":true,"first_absent_at":null,"delisting_confirmed_at":null,"events":[{"registry":"cdp","at":"2026-10-03T21:12:52.186Z","event":"listed"}]},"accepts":[{"registry":"cdp","ordinal":0,"network":"eip155:8453","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","asset_name":"USD Coin","symbol":"USDC","decimals":6,"amount_display":"0.003 USDC","amount_units":0.003,"decimals_known":true,"pay_to":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","scheme":"exact"},{"registry":"cdp","ordinal":1,"network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","asset_name":null,"symbol":"USDC","decimals":6,"amount_display":"0.003 USDC","amount_units":0.003,"decimals_known":true,"pay_to":"HcjfmZLgWRZzwj4deEiS7pewc4KBYSw39mW5nTyt3pz2","scheme":"exact"},{"registry":"cdp","ordinal":2,"network":"eip155:10","asset":"0x0b2c639c533813f4aa9d7837caf62653d097ff85","asset_name":"GatewayWalletBatched","symbol":"USDC","decimals":6,"amount_display":"0.003 USDC","amount_units":0.003,"decimals_known":true,"pay_to":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","scheme":"exact"},{"registry":"cdp","ordinal":3,"network":"eip155:5042","asset":"0x3600000000000000000000000000000000000000","asset_name":"GatewayWalletBatched","symbol":null,"decimals":null,"amount_display":"3000 smallest units, decimals unknown","amount_units":null,"decimals_known":false,"pay_to":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","scheme":"exact"}],"counters":{"observed_at":"2026-10-06T00:02:10.461Z","calls_30d":1,"unique_payers_30d":1,"last_called_at":"2026-10-03T20:44:09.318Z"},"validation":{"latest":{"observed_at":"2026-10-05T12:14:10.903Z","state":"pass","failed_checks":[],"endpoint_http_status":402,"run_id":"daily-2026-10-05"},"uptime":[{"days":7,"observed":3,"passed":3,"ratio":1},{"days":30,"observed":3,"passed":3,"ratio":1},{"days":90,"observed":3,"passed":3,"ratio":1}],"history_90d":[{"observed_at":"2026-10-05T12:14:10.903Z","state":"pass","failed_checks":[],"run_id":"daily-2026-10-05","state_changed":false},{"observed_at":"2026-10-04T14:47:49.662Z","state":"pass","failed_checks":[],"run_id":"daily-2026-10-04","state_changed":false},{"observed_at":"2026-10-03T21:59:01.222Z","state":"pass","failed_checks":[],"run_id":"new-after-daily-2026-10-03","state_changed":true}]},"seller":{"wallet":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","page":"https://tollindex.com/seller/0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","endpoints":957,"hosts":3},"reports":0,"registry_record":{"cdp":[{"tags":["robots","crawler","ai-bots","scraping","compliance"],"type":"http","accepts":[{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","amount":"3000","scheme":"exact","network":"eip155:8453","maxTimeoutSeconds":300},{"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","extra":{"feePayer":"GVJJ7rdGiXr5xaYbRwRbjfaJL7fmwRygFi1H6aGqDveb"},"payTo":"HcjfmZLgWRZzwj4deEiS7pewc4KBYSw39mW5nTyt3pz2","amount":"3000","scheme":"exact","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","maxTimeoutSeconds":300},{"asset":"0x0b2C639c533813f4Aa9D7837CAf62653d097Ff85","extra":{"name":"GatewayWalletBatched","assets":[{"symbol":"USDC","address":"0x0b2c639c533813f4aa9d7837caf62653d097ff85","decimals":6}],"version":"1","verifyingContract":"0x77777777dcc4d5a8b6e418fd04d8997ef11000ee","minValiditySeconds":604800},"payTo":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","amount":"3000","scheme":"exact","network":"eip155:10","maxTimeoutSeconds":604900},{"asset":"0x3600000000000000000000000000000000000000","extra":{"name":"GatewayWalletBatched","assets":[{"symbol":"USDC","address":"0x3600000000000000000000000000000000000000","decimals":6}],"version":"1","verifyingContract":"0x77777777dcc4d5a8b6e418fd04d8997ef11000ee","minValiditySeconds":604800},"payTo":"0xb9f06D2734B7E9D7b21B64C4D8b95bd774b71aCf","amount":"3000","scheme":"exact","network":"eip155:5042","maxTimeoutSeconds":604900}],"quality":{"lastCalledAt":"2026-10-03T20:44:09.318Z","l30DaysTotalCalls":1,"l30DaysUniquePayers":1},"resource":"https://api.vextorium.com/v1/robots-txt","extensions":{"bazaar":{"info":{"input":{"body":{"url":"https://www.nytimes.com","agent":"GPTBot","route":"/section/world"},"type":"http","method":"POST","bodyType":"json"},"output":{"type":"json","example":{"site":"https://www.nytimes.com","query":{"agent":"GPTBot","route":"/section/world","allowed":false,"applied_group":["gptbot"]},"groups":63,"ai_bots":{"CCBot":"bloqueado","GPTBot":"bloqueado","YouBot":"bloqueado","Diffbot":"bloqueado","Amazonbot":"permitido","ClaudeBot":"bloqueado","cohere-ai":"bloqueado","Bytespider":"bloqueado","Claude-User":"bloqueado","ChatGPT-User":"bloqueado","anthropic-ai":"bloqueado","DuckAssistBot":"bloqueado","OAI-SearchBot":"bloqueado","PerplexityBot":"bloqueado","MistralAI-User":"permitido","Google-Extended":"bloqueado","Perplexity-User":"bloqueado","Claude-SearchBot":"bloqueado","Applebot-Extended":"bloqueado","meta-externalagent":"bloqueado"},"sitemaps":["https://www.nytimes.com/sitemaps/new/news.xml.gz","https://www.nytimes.com/sitemaps/new/sitemap.xml.gz"],"robots_txt":true,"blocked_ai_bots":18}}},"schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method","bodyType","body"],"properties":{"body":{"required":["url"],"properties":{"url":{"type":"string"},"agent":{"type":"string","description":"User-agent to test (optional)"},"route":{"type":"string","description":"Path to test (default /)"}}},"type":{"type":"string","const":"http"},"method":{"enum":["POST"],"type":"string"},"bodyType":{"enum":["json","form-data","text"],"type":"string"}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":["object","null"],"properties":{"site":{"type":["string","null"]},"query":{"type":["object","null"],"properties":{"agent":{"type":["string","null"]},"route":{"type":["string","null"]},"allowed":{"type":["boolean","null"]},"applied_group":{"type":["array","null"],"items":{"type":["null","string"]}}}},"groups":{"type":["number","null"]},"ai_bots":{"type":["object","null"],"properties":{"CCBot":{"type":["string","null"]},"GPTBot":{"type":["string","null"]},"YouBot":{"type":["string","null"]},"Diffbot":{"type":["string","null"]},"Amazonbot":{"type":["string","null"]},"ClaudeBot":{"type":["string","null"]},"cohere-ai":{"type":["string","null"]},"Bytespider":{"type":["string","null"]},"Claude-User":{"type":["string","null"]},"ChatGPT-User":{"type":["string","null"]},"anthropic-ai":{"type":["string","null"]},"DuckAssistBot":{"type":["string","null"]},"OAI-SearchBot":{"type":["string","null"]},"PerplexityBot":{"type":["string","null"]},"MistralAI-User":{"type":["string","null"]},"Google-Extended":{"type":["string","null"]},"Perplexity-User":{"type":["string","null"]},"Claude-SearchBot":{"type":["string","null"]},"Applebot-Extended":{"type":["string","null"]},"meta-externalagent":{"type":["string","null"]}}},"sitemaps":{"type":["array","null"],"items":{"type":["null","string"]}},"robots_txt":{"type":["boolean","null"]},"blocked_ai_bots":{"type":["number","null"]}}}}}}}}},"description":"Parse a site's robots.txt: whether a user-agent may crawl a path, which AI crawlers (GPTBot, ClaudeBot, Google-Extended, PerplexityBot, CCBot...) are blocked, and declared sitemaps. Check before your agent scrapes.","lastUpdated":"2026-10-03T20:44:09.56Z","serviceName":"Vextorium","x402Version":2}]}}