{"api":"v1","generated_at":"2026-09-15T19:46:46.505Z","licence":{"name":"CC BY 4.0","url":"https://creativecommons.org/licenses/by/4.0/","attribution":"TOLL, with a link to the page cited"},"page":"https://tollindex.com/e/netintel-dev-robots-txt-analyze-a6d451","pinned_page":"https://tollindex.com/e/netintel-dev-robots-txt-analyze-a6d451/at/2026-09-15T16-57Z","pinned":false,"method":"https://tollindex.com/ledger/method","snapshot_at":"2026-09-15T16:57:44.360Z","snapshot_stamp":"2026-09-15T16-57Z","endpoint":{"slug":"netintel-dev-robots-txt-analyze-a6d451","canonical_url":"https://netintel.dev/robots-txt/analyze","resource":"https://netintel.dev/robots-txt/analyze","http_method":"GET","type":"http","x402_version":2,"registries":["cdp"],"primary_registry":"cdp","curated_by_coinbase":false,"description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","service_name":"NetIntel","declared_category":null,"declared_tags":["robots.txt","crawl rules","seo","web intelligence","scraping"],"route_template":null,"registry_updated":"2026-08-20T15:45:54.601Z"},"derived":{"note":"fields no registry supplies; TOLL derives them and marks them derived on the page","title":"robots txt analyze","category":"scraping","first_observed_by_toll":"2026-09-10T19:08:24.178Z","last_observed_by_toll":"2026-09-15T16:57:44.360Z","observation_began":"2026-09-10"},"presence":{"listed_now":true,"first_absent_at":null,"delisting_confirmed_at":null,"events":[{"registry":"cdp","at":"2026-09-10T19:08:24.178Z","event":"listed"}]},"accepts":[{"registry":"cdp","ordinal":0,"network":"eip155:8453","asset":"0x833589fcd6edb6e08f4c7c32d4f71b54bda02913","asset_name":"USD Coin","symbol":"USDC","decimals":6,"amount_display":"0.01 USDC","amount_units":0.01,"decimals_known":true,"pay_to":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","scheme":"exact"},{"registry":"cdp","ordinal":1,"network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","asset_name":null,"symbol":"USDC","decimals":6,"amount_display":"0.01 USDC","amount_units":0.01,"decimals_known":true,"pay_to":"8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s","scheme":"exact"}],"counters":{"observed_at":"2026-09-15T16:57:44.360Z","calls_30d":1,"unique_payers_30d":1,"last_called_at":"2026-08-20T15:45:54.399Z"},"validation":{"latest":{"observed_at":"2026-09-15T06:07:30.790Z","state":"pass","failed_checks":[],"endpoint_http_status":402,"run_id":"daily-2026-09-15"},"uptime":[{"days":7,"observed":5,"passed":5,"ratio":1},{"days":30,"observed":5,"passed":5,"ratio":1},{"days":90,"observed":5,"passed":5,"ratio":1}],"history_90d":[{"observed_at":"2026-09-15T06:07:30.790Z","state":"pass","failed_checks":[],"run_id":"daily-2026-09-15","state_changed":false},{"observed_at":"2026-09-14T05:38:40.672Z","state":"pass","failed_checks":[],"run_id":"daily-2026-09-14","state_changed":false},{"observed_at":"2026-09-13T05:32:24.712Z","state":"pass","failed_checks":[],"run_id":"daily-2026-09-13","state_changed":false},{"observed_at":"2026-09-12T07:49:17.079Z","state":"pass","failed_checks":[],"run_id":"daily-2026-09-12","state_changed":false},{"observed_at":"2026-09-11T11:41:14.883Z","state":"pass","failed_checks":[],"run_id":"daily-2026-09-11","state_changed":true}]},"seller":{"wallet":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","page":"https://tollindex.com/seller/0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","endpoints":125,"hosts":2},"reports":0,"registry_record":{"cdp":[{"tags":["robots.txt","crawl rules","seo","web intelligence","scraping"],"type":"http","accepts":[{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0xdaDc335482AD545296Fd7b28518A251fFCbEb9Df","amount":"10000","scheme":"exact","network":"eip155:8453","maxTimeoutSeconds":300},{"asset":"EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v","extra":{"feePayer":"BFK9TLC3edb13K6v4YyH3DwPb5DSUpkWvb7XnqCL9b4F"},"payTo":"8FVeybWnB4dYrxTuPK7WzFZNAiEcFVaDLFnKpRwVGr5s","amount":"10000","scheme":"exact","network":"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp","maxTimeoutSeconds":300}],"iconUrl":"https://res.cloudinary.com/bdb-prod/image/upload/c_fill,ar_1:1,w_256,h_256,f_auto,q_auto/v1784214098/bazaar-icons/7d2bf437631b2e4e.png","quality":{"lastCalledAt":"2026-08-20T15:45:54.399Z","l30DaysTotalCalls":1,"l30DaysUniquePayers":1},"resource":"https://netintel.dev/robots-txt/analyze","extensions":{"bazaar":{"info":{"input":{"type":"http","method":"GET","queryParams":{"domain":"example.com"}},"output":{"type":"json","example":{"found":true,"grade":"B","rules":[{"allow":["/public/"],"disallow":["/private/","/admin/"],"user_agent":"*","crawl_delay":null}],"score":85,"domain":"example.com","findings":[],"sitemaps":["https://example.com/sitemap.xml"],"path_check":{"path":"/private/page","allowed":false,"rule_type":"disallow","user_agent":"*","matched_rule":"Disallow: /private/"},"robots_url":"https://example.com/robots.txt","status_code":200,"raw_content_preview":"User-agent: *\nDisallow: /private/\n..."}}},"schema":{"type":"object","$schema":"https://json-schema.org/draft/2020-12/schema","required":["input"],"properties":{"input":{"type":"object","required":["type","method"],"properties":{"type":{"type":"string","const":"http"},"method":{"enum":["GET"],"type":"string"},"queryParams":{"type":"object","required":["domain"],"properties":{"path":{"type":"string","description":"URL path to check permission for (e.g. /api/data)"},"domain":{"type":"string","description":"Domain name to fetch robots.txt from (e.g. example.com)"},"user_agent":{"type":"string","description":"Bot name to check rules for (default: *)"}}}},"additionalProperties":false},"output":{"type":"object","required":["type"],"properties":{"type":{"type":"string"},"example":{"type":"object","properties":{"found":{"type":"boolean"},"grade":{"type":"string"},"rules":{"type":"array","items":{"type":"object"}},"score":{"type":"number"},"domain":{"type":"string"},"findings":{"type":"array","items":{"type":"object"}},"sitemaps":{"type":"array","items":{"type":"string"}},"path_check":{"type":"object"},"robots_url":{"type":"string"},"status_code":{"type":"number"},"raw_content_preview":{"type":"string"}}}}}}}}},"description":"Fetch and parse a domain's robots.txt file — returns all crawl rules by user-agent, sitemap URLs, crawl delay settings, and checks whether a specific path is allowed or blocked for any bot — so agents can respect crawl policies and locate sitemaps before scraping.","lastUpdated":"2026-08-20T15:45:54.601Z","serviceName":"NetIntel","x402Version":2}]}}