{
  "schema_version": 1,
  "title": "Model Evaluation: Test Sets and Data Leakage",
  "review_type": "AI-assisted source-support audit; not independent expert verification",
  "sources": [
    {
      "id": "S1",
      "key": "evaluation-context",
      "title": "Probabilistic Time Series Forecasting with 🤗 Transformers",
      "author": "Hugging Face authors. (2022).",
      "url": "https://huggingface.co/blog/time-series-transformers",
      "requested_url": "https://huggingface.co/blog/time-series-transformers",
      "accessed_at": "2026-11-13T22:38:57.818Z",
      "source_type": "primary",
      "version": null,
      "published_at": "2022-12-01T00:00:00.000Z",
      "source_date_note": "Hugging Face authors. (2022).",
      "retrieved_body_sha256": "d9c0944b11bc155b3b6ef04d10d10be3e6b7f583b997097070cabf07d35e4f4f",
      "library": {
        "id": "c8e031ab6c189d847041872c860e5aa0",
        "revision": "3ab8f467722743350f0c13978e6fbb96",
        "published_at": "2022-12-01T00:00:00.000Z",
        "captured_at": "2026-11-06T20:02:38.014Z",
        "exact_quote_in_saved_original": true,
        "passage_matches": [
          {
            "evidence_id": "S1-E1",
            "exact_match": true
          }
        ]
      },
      "evidence": [
        {
          "id": "S1-E1",
          "location": "Load Dataset; Create DataLoaders",
          "quote": "the dataset contains 3 splits: train, validation and test.",
          "normalized_text_offset": 6315
        }
      ],
      "limitations": "A saved forecasting example provides context for explicit data splits. Its specific temporal setup is not generalized to every task.",
      "checks": [
        "publisher URL and final redirect inspected",
        "actual relevant primary passage read",
        "exact excerpt matched",
        "scope and available version/date inspected"
      ]
    },
    {
      "id": "S2",
      "key": "evaluation-leakage",
      "title": "12. Common pitfalls and recommended practices — scikit-learn 1.9.1 documentation",
      "author": "scikit-learn maintainers. (v1.9.1).",
      "url": "https://scikit-learn.org/stable/common_pitfalls.html",
      "requested_url": "https://scikit-learn.org/stable/common_pitfalls.html",
      "accessed_at": "2026-11-13T22:38:57.748Z",
      "source_type": "primary",
      "version": "scikit-learn 1.9.1",
      "published_at": null,
      "source_date_note": "scikit-learn maintainers. (v1.9.1).",
      "retrieved_body_sha256": "21fac11c181ab35624dcf8673f0475d403d4f6ecf0f9bead01a573d7642b515c",
      "library": null,
      "evidence": [
        {
          "id": "S2-E1",
          "location": "12.2 Data leakage; How to avoid data leakage",
          "quote": "Test data should never be used to make choices about the model.",
          "normalized_text_offset": 2934
        }
      ],
      "limitations": "Training-only preprocessing and separation of test information; examples are not copied.",
      "checks": [
        "publisher URL and final redirect inspected",
        "actual relevant primary passage read",
        "exact excerpt matched",
        "scope and available version/date inspected"
      ]
    },
    {
      "id": "S3",
      "key": "evaluation-splits",
      "title": "3.1. Cross-validation: evaluating estimator performance — scikit-learn 1.9.1 documentation",
      "author": "scikit-learn maintainers. (v1.9.1).",
      "url": "https://scikit-learn.org/stable/modules/cross_validation.html",
      "requested_url": "https://scikit-learn.org/stable/modules/cross_validation.html",
      "accessed_at": "2026-11-13T22:38:57.835Z",
      "source_type": "primary",
      "version": "scikit-learn 1.9.1",
      "published_at": null,
      "source_date_note": "scikit-learn maintainers. (v1.9.1).",
      "retrieved_body_sha256": "78d7587fa92ba3b3e480c9d841b50fedc14ac29d571ccb319d4068235154f0e2",
      "library": null,
      "evidence": [
        {
          "id": "S3-E1",
          "location": "3.1.2.4.1 Group K-fold",
          "quote": "ensures that the same group is not represented in both testing and training sets",
          "normalized_text_offset": 21825
        }
      ],
      "limitations": "Group-aware evaluation for new-group generalization; not a blanket claim that all group overlap is leakage.",
      "checks": [
        "publisher URL and final redirect inspected",
        "actual relevant primary passage read",
        "exact excerpt matched",
        "scope and available version/date inspected"
      ]
    }
  ],
  "claims": [
    {
      "id": "C01",
      "notebook_locations": [
        "workflow"
      ],
      "claim": "Define prediction — New rows or groups — State what will be new when the system is actually used. — This example targets groups absent from training.; Choose split unit — Rows · groups · time — Separate data using the unit appropriate to that prediction question. — Keeping unique row IDs apart does not guarantee groups are separate.; Fit on training — Rules + preprocessing — Learn or choose behavior from training data and designated validation work. — The toy memorizes group labels and defaults to 0 for unseen groups.; Evaluate once — Held-out result — Compute the stated metric on the held-out data and report the split design. — Repeatedly choosing from test results makes those results part of model selection.",
      "source_ids": [
        "S1",
        "S2",
        "S3"
      ],
      "evidence_ids": [
        "S1-E1",
        "S2-E1",
        "S3-E1"
      ],
      "verdict": "editorial_diagram_with_source_supported_roles",
      "reason": "Diagram is a teaching workflow, not an exhaustive protocol or measured system. Each role and arrow is bounded in nearby text."
    },
    {
      "id": "C02",
      "notebook_locations": [
        "experiment"
      ],
      "claim": "Groups A, B, C and D each have two records. Their fixed labels are 0, 1, 0 and 1. The rule memorizes a group’s training label and predicts 0 for an unseen group. Choose the split and inspect overlap alongside accuracy. Row split: A1 B1 C1 D1 train; A2 B2 C2 D2 test. Group split: all A/B train; all C/D test. The label rule is deliberately constructed. Its 100% and 50% scores do not estimate the effect size of leakage in any real dataset. Row split: 4 shared groups; 100% accuracy: Every test group has a memorized label. Held-out-group split: 0 shared groups; 50% accuracy: Unseen C and D receive the fallback label 0. Actual target: New groups: The held-out-group result matches this example’s stated prediction question.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_synthetic_experiment",
      "reason": "Original fixed inputs and outputs; Python assertions and independent browser arithmetic checked in verification artifacts."
    },
    {
      "id": "C03",
      "notebook_locations": [
        "workflow"
      ],
      "claim": "An evaluation split encodes a prediction question. If the intended use is a new group, testing on another row from a training group can answer an easier question. scikit-learn documents group-aware splitting to keep a group out of both training and test simultaneously. The relevant unit might be a person, source document or other correlated group in an actual task.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C04",
      "notebook_locations": [
        "workflow"
      ],
      "claim": "Data leakage uses information during model building that would not be available at prediction time. Test results should not drive model choices, and fitted preprocessing belongs on the training data. The saved forecasting tutorial also makes its train, validation and test windows explicit; its temporal design is context rather than a universal split recipe.",
      "source_ids": [
        "S1",
        "S2"
      ],
      "evidence_ids": [
        "S1-E1",
        "S2-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C05",
      "notebook_locations": [
        "interpretation"
      ],
      "claim": "Both splits use distinct row IDs in training and test. That fact alone misses the group relationship. The overlap matrix therefore shows group membership directly, while the accuracy chart shows what the particular memorizing rule achieves. Report both the score and the unit of separation.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C06",
      "notebook_locations": [
        "interpretation"
      ],
      "claim": "A row-level split is not universally wrong. It would test a different use case, such as new records from already-known groups, if that matched the intended deployment. Here it is misleading only relative to the declared goal of unseen-group prediction. The notebook keeps that goal visible rather than calling every overlap a violation.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C07",
      "notebook_locations": [
        "interpretation"
      ],
      "claim": "Editorial evaluation checklist: freeze the prediction target, split before fitting transformations, use designated validation data for choices and preserve a final test for assessment. Time-dependent tasks need a suitable temporal design; a random shuffle is not automatically appropriate.",
      "source_ids": [
        "S1",
        "S2",
        "S3"
      ],
      "evidence_ids": [
        "S1-E1",
        "S2-E1",
        "S3-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C08",
      "notebook_locations": [
        "reproduce"
      ],
      "claim": "The original standard-library Python example passed its assertions and 2 input fixtures. Its browser implementation is separately compared against those results.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "tested_original_example",
      "reason": "No publisher code executed; no model, external network call or privileged action in the examples."
    },
    {
      "id": "C09",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "Does this row split keep groups out of both train and test? Correct answer: No. Each group contributes one record to training and another to test.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    },
    {
      "id": "C10",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "Why is the group split the relevant one for this toy’s declared target? Correct answer: It evaluates unseen groups. The target is prediction for groups absent from training.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "source_supported_and_worked_example",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    },
    {
      "id": "C11",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "Should final test results guide model choices? Correct answer: No; that uses test information for selection. The test should remain separate from model choices; use validation work for those decisions.",
      "source_ids": [
        "S2"
      ],
      "evidence_ids": [
        "S2-E1"
      ],
      "verdict": "source_supported_and_worked_example",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    },
    {
      "id": "C12",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "What does the 50% result establish? Correct answer: The actual result of this synthetic rule and split. It is a constructed fixture with a declared fallback rule.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    }
  ],
  "original_experiment": {
    "scope": "A synthetic grouped dataset and a memorizing rule. No real model is trained and no production accuracy is estimated.",
    "assumptions": "Row split: A1 B1 C1 D1 train; A2 B2 C2 D2 test. Group split: all A/B train; all C/D test. The label rule is deliberately constructed. Its 100% and 50% scores do not estimate the effect size of leakage in any real dataset.",
    "results_file": "verification/python-results.json"
  },
  "library_gap": null,
  "excluded_evidence": [],
  "limitations": [
    "No real model or GPU measured.",
    "Sources and version/date labels checked as recorded; this does not prove absence of errors."
  ],
  "separate_audit_file": "verification/source-audit.json",
  "publication": {
    "state": "published",
    "approved_on": "2026-11-14",
    "published_on": "2026-11-14",
    "source_checks_on": "2026-11-13"
  }
}
