canonical robots consistency audit derived

{
  "input": {
    "body": {
      "pages": [
        {
          "url": "https://example.test/widgets/widget-1",
          "html": "<html><head><title>W1</title><link rel=\"canonical\" href=\"https://example.test/widgets/widget-1?ref=newsletter\"></head><body>a <a href=\"/widgets/widget-1/print\">print</a></body></html>"
        },
        {
          "url": "https://example.test/widgets/widget-1/print",
          "html": "<html><head><title>W1 print</title><link rel=\"canonical\" href=\"https://example.test/widgets/widget-1\"><meta name=\"robots\" content=\"noindex\"></head><body>a</body></html>"
        },
        {
          "url": "https://example.test/widgets/widget-2",
          "html": "<html><head><title>W2</title><link rel=\"canonical\" href=\"https://example.test/widgets/widget-2\"><meta name=\"robots\" content=\"noindex, follow\"></head><body>b</body></html>",
          "headers": {
            "X-Robots-Tag": "index, follow"
          }
        },
        {
          "url": "https://example.test/widgets/widget-3",
          "html": "<html><head><title>W3</title></head><body>c</body></html>"
        }
      ],
      "robots_txt": "User-agent: *\nDisallow: /internal/\n"
    },
    "type": "http",
    "method": "POST",
    "bodyType": "json"
  },
  "output": {
    "type": "json",
    "example": {
      "pages": [
        {
          "url": "https://example.test/widgets/widget-1",
          "noindex": false,
          "canonical": "https://example.test/widgets/widget-1?ref=newsletter",
          "meta_robots": null,
          "header_robots": null
        },
        {
          "url": "https://example.test/widgets/widget-1/print",
          "noindex": true,
          "canonical": "https://example.test/widgets/widget-1",
          "meta_robots": "noindex",
          "header_robots": null
        },
        {
          "url": "https://example.test/widgets/widget-2",
          "noindex": true,
          "canonical": "https://example.test/widgets/widget-2",
          "meta_robots": "noindex, follow",
          "header_robots": "index, follow"
        }
      ],
      "limits": {
        "maxPages": 50,
        "maxListed": 200,
        "maxCookies": 100,
        "maxHeaders": 200,
        "maxFindings": 500,
        "maxHtmlBytes": 262144,
        "maxSitemapUrls": 10000,
        "maxSitemapBytes": 1048576
      },
      "passed": false,
      "complete": false,
      "findings": [
        {
          "url": "https://example.test/widgets/widget-1",
          "code": "CANONICAL_POINTS_ELSEWHERE",
          "detail": "https://example.test/widgets/widget-1 names https://example.test/widgets/widget-1?ref=newsletter as canonical, so signals consolidate onto the other URL.",
          "canonical": "https://example.test/widgets/widget-1?ref=newsletter"
        },
        {
          "url": "https://example.test/widgets/widget-1/print",
          "code": "CANONICAL_POINTS_ELSEWHERE",
          "detail": "https://example.test/widgets/widget-1/print names https://example.test/widgets/widget-1 as canonical, so signals consolidate onto the other URL.",
          "canonical": "https://example.test/widgets/widget-1"
        },
        {
          "url": "https://example.test/widgets/widget-2",
          "code": "ROBOTS_HEADER_AND_META_DISAGREE",
          "detail": "https://example.test/widgets/widget-2 declares robots noindex, follow in the markup and index, follow in the header, and which one a crawler honours is not something the page controls."
        }
      ],
      "consistent": false,
      "page_count": 4,
      "request_id": "example-request",
      "service_id": "canonical_robots_consistency_audit",
      "assumptions": [
        "A robots directive may appear in the markup or in an X-Robots-Tag header, and a disagreement between them is reported because which one a crawler honours is not something the page controls.",
        "Two pages naming one canonical is normal consolidation. A canonical pointing at a page that itself points elsewhere is a chain and is reported, because chains are not followed reliably.",
        "The caller supplies every document, so nothing is fetched and no robots.txt applies to this call."
      ],
      "limitations": [
        "Coherence of the directives in the documents supplied. What any particular crawler does with them is not predicted here.",
        "consistent is true only when every page parsed, every canonical resolved sensibly, and no directive contradicted another."
      ],
      "observed_at": "2026-10-02T00:00:00.000Z",
      "not_examined": [
        "ROBOTS_TXT_DECLARES_NO_SITEMAP: robots.txt names no sitemap, which is a missed hint rather than an error."
      ],
      "finding_count": 6,
      "service_version": "1.0.0",
      "findings_by_code": {
        "CANONICAL_CHAIN": 1,
        "CANONICAL_ABSENT": 1,
        "CANONICAL_POINTS_ELSEWHERE": 2,
        "NOINDEX_WITH_SELF_CANONICAL": 1,
        "ROBOTS_HEADER_AND_META_DISAGREE": 1
      },
      "noindex_page_count": 2,
      "distinct_canonical_count": 3
    }
  }
}