{
  "schema_version": 1,
  "title": "Retrieval-Augmented Generation (RAG): Retrieval Evaluation",
  "created_on": "2026-11-13",
  "review_type": "AI-assisted source-support audit; not independent expert verification",
  "sources": [
    {
      "id": "S1",
      "title": "Retrieval Augmented Generation with Huggingface Transformers and Ray",
      "publisher": "Hugging Face · Amog Kamsetty (Anyscale guest author)",
      "url": "https://huggingface.co/blog/ray-rag",
      "accessed_at": "2026-11-13T21:48:59.053Z",
      "source_type": "primary",
      "published_at": "2021-02-10",
      "saved_document_id": "46f3ffc4bbd079e05a8ded5ad882e391",
      "saved_revision_id": "e53ba23d35dfe41db4dce1ab971ee509",
      "saved_captured_at": "2026-11-06T20:15:05.499Z",
      "evidence": [
        {
          "id": "S1-E1",
          "location": "What is Retrieval Augmented Generation (RAG)?",
          "quote": "RAG has an intermediate component that retrieves contextual documents from an external knowledge base",
          "normalized_text_offset": 1201
        }
      ],
      "retrieved_body_sha256": "c2c7302114e022e9d1b90118643d4390e6734e48f15826bcfacb98d666d20937",
      "used_for": "Basic retrieved-document plus generator arrangement.",
      "limitations": "Dated architecture explanation; no old installation instructions or performance claims are reused.",
      "verification": "identity, source passage, scope and available dates/version checked"
    },
    {
      "id": "S2",
      "title": "precision_score — scikit-learn 1.9.1 documentation",
      "publisher": "scikit-learn maintainers",
      "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_score.html",
      "accessed_at": "2026-11-13T21:50:37.517Z",
      "source_type": "primary",
      "version": "1.9.1",
      "published_at": null,
      "evidence": [
        {
          "id": "S2-E1",
          "location": "API description, first definition paragraph",
          "quote": "The precision is the ratio tp / (tp + fp)",
          "normalized_text_offset": 186
        }
      ],
      "retrieved_body_sha256": "00fa905fa700f968ce61ef066a6dee219e55f402ebf75a9950fe607f09d69f4d",
      "used_for": "Binary precision ratio; the lab explicitly maps retrieved top-k items to predicted positives.",
      "limitations": "A classification definition, not a prescribed RAG cutoff or deployment policy.",
      "verification": "identity, source passage, scope and available dates/version checked"
    },
    {
      "id": "S3",
      "title": "recall_score — scikit-learn 1.9.1 documentation",
      "publisher": "scikit-learn maintainers",
      "url": "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html",
      "accessed_at": "2026-11-13T21:50:37.504Z",
      "source_type": "primary",
      "version": "1.9.1",
      "published_at": null,
      "evidence": [
        {
          "id": "S3-E1",
          "location": "API description, first definition paragraph",
          "quote": "The recall is the ratio tp / (tp + fn)",
          "normalized_text_offset": 177
        }
      ],
      "retrieved_body_sha256": "859c3fd295a6e64f9aa3816c138c030c9d9295b616bf321b7edb31a2b0c34f68",
      "used_for": "Binary recall ratio, including relevant items that were not retrieved.",
      "limitations": "The labelled set is complete only in this synthetic fixture.",
      "verification": "identity, source passage, scope and available dates/version checked"
    },
    {
      "id": "S4",
      "title": "BEIR custom_metrics.py · mrr()",
      "publisher": "BEIR maintainers",
      "url": "https://raw.githubusercontent.com/beir-cellar/beir/6d811867c38b01f1afc9b5a81b1dd609eb4f897c/beir/retrieval/custom_metrics.py",
      "accessed_at": "2026-11-13T21:50:37.651Z",
      "source_type": "primary",
      "commit": "6d811867c38b01f1afc9b5a81b1dd609eb4f897c",
      "file_commit_date": "2025-02-03T22:36:33Z",
      "published_at": null,
      "evidence": [
        {
          "id": "S4-E1",
          "location": "mrr function, lines 21–30",
          "quote": "MRR[f\"MRR@{k}\"] += 1.0 / (rank + 1)",
          "normalized_text_offset": 658
        }
      ],
      "retrieved_body_sha256": "3b315c1473115dc4e41d3038423fe5f1963868c3b9c92d5386839bde33472157",
      "used_for": "First positive relevance hit, zero for a miss inside k, and query-count averaging.",
      "limitations": "Source implementation inspected, never executed. Our fixture has a ranking for every judged query.",
      "verification": "identity, source passage, scope and available dates/version checked"
    },
    {
      "id": "S5",
      "title": "Retrieval-Augmented Generation (RAG) evaluators",
      "publisher": "Microsoft Learn",
      "url": "https://learn.microsoft.com/en-us/azure/foundry/concepts/evaluation-evaluators/rag-evaluators?view=foundry",
      "accessed_at": "2026-11-13T21:48:59.263Z",
      "source_type": "primary",
      "published_at": null,
      "document_ms_date": "2026-06-02",
      "updated_at": "2026-08-26T22:14:00Z",
      "evidence": [
        {
          "id": "S5-E1",
          "location": "Process evaluation and System evaluation sections",
          "quote": "Groundedness focuses on the precision aspect of the response.",
          "normalized_text_offset": 2657
        }
      ],
      "retrieved_body_sha256": "bff713e64e1c585606aba5ab41f40514f020711fbd857630b1ac8e7d759584f6",
      "used_for": "Separate document-retrieval evaluation from final-response groundedness and relevance.",
      "limitations": "Product-specific evaluator scores are not used here. The public body was readable despite a generic authorization notice in the page template.",
      "verification": "identity, source passage, scope and available dates/version checked"
    },
    {
      "id": "S6",
      "title": "Ragas: Automated Evaluation of Retrieval Augmented Generation",
      "publisher": "Es, James, Espinosa-Anke and Schockaert · arXiv",
      "url": "https://arxiv.org/html/2309.15217v2",
      "accessed_at": "2026-11-13T21:50:37.395Z",
      "source_type": "primary",
      "published_at": "2023-09-26",
      "revised_at": "2025-04-28",
      "version": "arXiv:2309.15217v2",
      "evidence": [
        {
          "id": "S6-E1",
          "location": "Section 3, Evaluation Strategies",
          "quote": "Faithfulness refers to the idea that the answer should be grounded in the given context.",
          "normalized_text_offset": 9414
        }
      ],
      "retrieved_body_sha256": "a62a2216fd33c7964c77d74622a8363608d4f0d863fe174daff37b346027f182",
      "used_for": "Distinguish faithfulness, answer relevance and context relevance.",
      "limitations": "Full Section 3 read. No claim that its LLM judges independently verify facts or match current package behavior.",
      "verification": "identity, source passage, scope and available dates/version checked"
    }
  ],
  "claims": [
    {
      "id": "C01",
      "notebook_locations": [
        "two-checks"
      ],
      "claim": "RAG supplies retrieved documents and a question to a generator.",
      "source_ids": [
        "S1"
      ],
      "evidence_ids": [
        "S1-E1"
      ],
      "verdict": "supported",
      "reason": "Saved public original and current page agree on this bounded architecture description."
    },
    {
      "id": "C02",
      "notebook_locations": [
        "two-checks",
        "evidence-flow"
      ],
      "claim": "Retrieval quality, context use and final-answer behavior can be checked separately.",
      "source_ids": [
        "S5",
        "S6"
      ],
      "evidence_ids": [
        "S5-E1",
        "S6-E1"
      ],
      "verdict": "supported",
      "reason": "Both primary sources distinguish these dimensions; diagram is an explicit teaching sketch."
    },
    {
      "id": "C03",
      "notebook_locations": [
        "ranking-lab",
        "metric-definitions",
        "two-checks"
      ],
      "claim": "This experiment treats the first k unique results as retrieved positives, with binary relevance.",
      "source_ids": [
        "S2",
        "S3",
        "S4"
      ],
      "evidence_ids": [
        "S2-E1",
        "S3-E1",
        "S4-E1"
      ],
      "verdict": "author_defined",
      "reason": "Operational setup is stated, not attributed as a universal convention."
    },
    {
      "id": "C04",
      "notebook_locations": [
        "metric-definitions",
        "self-check"
      ],
      "claim": "Precision is relevant retrieved items divided by retrieved items.",
      "source_ids": [
        "S2"
      ],
      "evidence_ids": [
        "S2-E1"
      ],
      "verdict": "supported",
      "reason": "tp/(tp+fp), mapped to top-k fixture."
    },
    {
      "id": "C05",
      "notebook_locations": [
        "metric-definitions",
        "self-check"
      ],
      "claim": "Recall is relevant retrieved items divided by all relevant items.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "supported",
      "reason": "tp/(tp+fn), with a nonempty declared relevant set."
    },
    {
      "id": "C06",
      "notebook_locations": [
        "metric-definitions",
        "self-check",
        "two-checks"
      ],
      "claim": "RR@k is reciprocal first relevant rank or zero for no hit; MRR@k averages over queries.",
      "source_ids": [
        "S4"
      ],
      "evidence_ids": [
        "S4-E1"
      ],
      "verdict": "supported",
      "reason": "Pinned source initializes zero, adds reciprocal rank once, then divides by qrels count; all fixture queries have rankings."
    },
    {
      "id": "C07",
      "notebook_locations": [
        "metric-definitions"
      ],
      "claim": "Later relevant hits do not change reciprocal rank.",
      "source_ids": [
        "S4"
      ],
      "evidence_ids": [
        "S4-E1"
      ],
      "verdict": "supported",
      "reason": "The mrr() inner loop breaks after the first relevant hit."
    },
    {
      "id": "C08",
      "notebook_locations": [
        "ranking-lab",
        "compare-rankings"
      ],
      "claim": "Corpus passages, relevance labels, queries and rankings are fictional. ",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "author_defined",
      "reason": "Exact fixture is in metrics_lab.py and lab-data.js."
    },
    {
      "id": "C09",
      "notebook_locations": [
        "ranking-lab",
        "reproduce",
        "self-check"
      ],
      "claim": "Export query A at k=3 has P=1/3, R=1/3 and RR=1.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "verified_calculation",
      "reason": "Exact Fraction assertions in metrics_lab.py."
    },
    {
      "id": "C10",
      "notebook_locations": [
        "compare-rankings",
        "reproduce"
      ],
      "claim": "At k=3, macro A=(2/9,4/9,4/9), B=(5/9,1,5/6).",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "verified_calculation",
      "reason": "Three-query exact Fraction calculation; browser comparison checked against these fixtures."
    },
    {
      "id": "C11",
      "notebook_locations": [
        "ranking-lab"
      ],
      "claim": "Recall cannot decrease as k increases on one fixed ranking and relevant set.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "verified_derivation",
      "reason": "Set inclusion proof plus all 720 six-ID permutations and six cutoffs (4320 checks)."
    },
    {
      "id": "C12",
      "notebook_locations": [
        "ranking-lab"
      ],
      "claim": "Precision can rise or fall as k changes on this ranking.",
      "source_ids": [
        "S2"
      ],
      "evidence_ids": [
        "S2-E1"
      ],
      "verdict": "verified_calculation",
      "reason": "A/export: k3=1/3, k4=1/2, k5=2/5."
    },
    {
      "id": "C13",
      "notebook_locations": [
        "metric-definitions",
        "reproduce"
      ],
      "claim": "The lab rejects duplicate IDs, short rankings and empty relevant sets.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "author_defined",
      "reason": "Code policy and rejection checks; not a universal evaluation rule."
    },
    {
      "id": "C14",
      "notebook_locations": [
        "metric-definitions",
        "two-checks"
      ],
      "claim": "Incomplete relevance labels limit recall interpretation.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "reasoned_limitation",
      "reason": "A ratio over labelled positives cannot establish coverage of unknown positives."
    },
    {
      "id": "C15",
      "notebook_locations": [
        "two-checks",
        "trace-diagnosis",
        "self-check"
      ],
      "claim": "Support relative to supplied context does not itself establish source truth.",
      "source_ids": [
        "S5",
        "S6"
      ],
      "evidence_ids": [
        "S5-E1",
        "S6-E1"
      ],
      "verdict": "reasoned_limitation",
      "reason": "Sources define context-relative groundedness/faithfulness; source reliability remains outside that check."
    },
    {
      "id": "C16",
      "notebook_locations": [
        "trace-diagnosis",
        "two-checks"
      ],
      "claim": "A passage can appear in the retrieved list but be absent from supplied context.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "author_defined",
      "reason": "Hand-authored trace records D3 in retrieval and omits it from supplied context; no frequency claim."
    },
    {
      "id": "C17",
      "notebook_locations": [
        "trace-diagnosis"
      ],
      "claim": "An answer may be context-supported while not addressing the question.",
      "source_ids": [
        "S6"
      ],
      "evidence_ids": [
        "S6-E1"
      ],
      "verdict": "supported_and_illustrated",
      "reason": "Ragas separates answer relevance and faithfulness; fictional hours answer illustrates the distinction."
    },
    {
      "id": "C18",
      "notebook_locations": [
        "compare-rankings",
        "two-checks"
      ],
      "claim": "The proposed experiment protocol is guidance, not evidence of deployment improvement.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "editorial_recommendation",
      "reason": "Explicitly labelled editorial; no real-system benchmark is claimed."
    },
    {
      "id": "C19",
      "notebook_locations": [
        "reproduce"
      ],
      "claim": "The original Python arithmetic and browser values agree on the synthetic fixture.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "verified_calculation",
      "reason": "Exact Python, JavaScript arithmetic and all 36 browser UI states matched; verification files record the actual checks."
    },
    {
      "id": "C21",
      "notebook_locations": [
        "metric-definitions",
        "self-check"
      ],
      "claim": "Same top-k set can preserve precision/recall while changing first-hit reciprocal rank.",
      "source_ids": [
        "S2",
        "S3",
        "S4"
      ],
      "evidence_ids": [
        "S2-E1",
        "S3-E1",
        "S4-E1"
      ],
      "verdict": "verified_calculation",
      "reason": "Two three-ID orderings checked with exact arithmetic."
    }
  ],
  "original_experiment": {
    "corpus": {
      "D1": "Choose Export in the notebook menu.",
      "D2": "Select JSON as the output format.",
      "D3": "Download the file when its status says Ready.",
      "D4": "The demo help desk opens from 09:00 to 17:00 UTC.",
      "D5": "Deleted demo drafts stay in Trash for seven days.",
      "D6": "Choose a theme under Appearance."
    },
    "cases": {
      "export": {
        "question": "How do I export my notes?",
        "relevant": [
          "D1",
          "D2",
          "D3"
        ],
        "A": [
          "D1",
          "D4",
          "D6",
          "D2",
          "D5",
          "D3"
        ],
        "B": [
          "D1",
          "D2",
          "D3",
          "D4",
          "D5",
          "D6"
        ]
      },
      "hours": {
        "question": "When is the demo help desk open?",
        "relevant": [
          "D4"
        ],
        "A": [
          "D1",
          "D6",
          "D4",
          "D2",
          "D3",
          "D5"
        ],
        "B": [
          "D6",
          "D4",
          "D1",
          "D2",
          "D3",
          "D5"
        ]
      },
      "trash": {
        "question": "How long are deleted demo drafts retained?",
        "relevant": [
          "D5"
        ],
        "A": [
          "D6",
          "D1",
          "D2",
          "D3",
          "D4",
          "D5"
        ],
        "B": [
          "D5",
          "D6",
          "D1",
          "D2",
          "D3",
          "D4"
        ]
      }
    },
    "conventions": [
      "binary relevance",
      "unique passage IDs",
      "at least k returned results",
      "nonempty relevant sets",
      "equal query weight",
      "misses contribute RR=0"
    ],
    "results_file": "verification/metrics-results.json"
  },
  "excluded_evidence": [
    {
      "source": "Stanford Introduction to Information Retrieval online textbook",
      "reason": "HTML and PDF retrieval attempts did not respond. Search snippets were not treated as verified evidence."
    },
    {
      "source": "Other search hits in ResearchNode",
      "reason": "Most were abstract-only or not directly relevant; not used for metric or implementation claims."
    },
    {
      "source": "work:note-rag",
      "reason": "Read as a generated reading path, not used as primary corroboration. Its linked public original S1 was read directly."
    }
  ],
  "limitations": [
    "Source support is not independent fact verification.",
    "Synthetic labels and rankings do not predict production performance.",
    "No LLM judge, real retriever, benchmark dataset, latency or GPU experiment was run."
  ],
  "publication": {
    "state": "published",
    "approved_on": "2026-11-14",
    "published_on": "2026-11-14",
    "source_checks_on": "2026-11-13"
  }
}
