{
  "schema_version": 1,
  "title": "Model Serving: Batching and Request Queues",
  "review_type": "AI-assisted source-support audit; not independent expert verification",
  "sources": [
    {
      "id": "S1",
      "key": "batching-principles",
      "title": "Continuous batching from first principles",
      "author": "Ouazan Reboul, R., Zucker, A., & Georges, L. (2025).",
      "url": "https://huggingface.co/blog/continuous_batching",
      "requested_url": "https://huggingface.co/blog/continuous_batching",
      "accessed_at": "2026-11-13T22:38:56.756Z",
      "source_type": "primary",
      "version": null,
      "published_at": "2025-11-25T00:00:00.000Z",
      "source_date_note": "Ouazan Reboul, R., Zucker, A., & Georges, L. (2025).",
      "retrieved_body_sha256": "27ce3637da046ad6faf31ccceef537ce5c3e1be57a2ad167c4df01ed071743a5",
      "library": {
        "id": "7b44225dc8d09a3a6793eaef130699be",
        "revision": "93ad81780057f3e35e2e0c92235deb0e",
        "published_at": "2025-11-25T00:00:00.000Z",
        "captured_at": "2026-11-06T18:29:47.038Z",
        "exact_quote_in_saved_original": true,
        "passage_matches": [
          {
            "evidence_id": "S1-E1",
            "exact_match": true
          }
        ]
      },
      "evidence": [
        {
          "id": "S1-E1",
          "location": "Continuous batching section",
          "quote": "we remove finished prompts from the batch as soon as they are done",
          "normalized_text_offset": 17695
        }
      ],
      "limitations": "Conceptual introduction read in the saved public original and current publisher page. No benchmark result is reused.",
      "checks": [
        "publisher URL and final redirect inspected",
        "actual relevant primary passage read",
        "exact excerpt matched",
        "scope and available version/date inspected"
      ]
    },
    {
      "id": "S2",
      "key": "batching-docs",
      "title": "Continuous batching · Hugging Face",
      "author": "Hugging Face maintainers. (Main documentation; accessed 2026).",
      "url": "https://huggingface.co/docs/transformers/main/continuous_batching",
      "requested_url": "https://huggingface.co/docs/transformers/main/continuous_batching",
      "accessed_at": "2026-11-13T22:38:56.793Z",
      "source_type": "primary",
      "version": null,
      "published_at": null,
      "source_date_note": "Hugging Face maintainers. (Main documentation; accessed 2026).",
      "retrieved_body_sha256": "6425304abd00172036c61831330f9c3a459c78d44bbf2f7c53d574bdb198a021",
      "library": null,
      "evidence": [
        {
          "id": "S2-E1",
          "location": "Continuous batching introduction",
          "quote": "As requests finish, new ones join immediately instead of waiting for the whole batch to complete.",
          "normalized_text_offset": 1754
        }
      ],
      "limitations": "Current manager and request lifecycle documentation; the toy scheduler is not this implementation.",
      "checks": [
        "publisher URL and final redirect inspected",
        "actual relevant primary passage read",
        "exact excerpt matched",
        "scope and available version/date inspected"
      ]
    },
    {
      "id": "S3",
      "key": "batching-queues",
      "title": "Efficient Request Queueing – Optimizing LLM Performance",
      "author": "TNG Technology Consulting authors. (2025).",
      "url": "https://huggingface.co/blog/tngtech/llm-performance-request-queueing",
      "requested_url": "https://huggingface.co/blog/tngtech/llm-performance-request-queueing",
      "accessed_at": "2026-11-13T22:38:56.742Z",
      "source_type": "primary",
      "version": null,
      "published_at": "2025-04-02T13:33:53.000Z",
      "source_date_note": "TNG Technology Consulting authors. (2025).",
      "retrieved_body_sha256": "e028ddeecef2a6e359ff7e30b3ab313dcb6257a67b530ac1311edd2214afddaa",
      "library": {
        "id": "8f6d7dcacc39d11598a0bbeb01177459",
        "revision": "9edc40898805d54aacf6d862a3bea560",
        "published_at": "2025-04-02T13:33:53.000Z",
        "captured_at": "2026-11-06T19:00:52.370Z",
        "exact_quote_in_saved_original": true,
        "passage_matches": [
          {
            "evidence_id": "S3-E1",
            "exact_match": true
          }
        ]
      },
      "evidence": [
        {
          "id": "S3-E1",
          "location": "Solution: Fair Scheduling; No Backpressure by Backend Queue",
          "quote": "separate queues for each user (and model)",
          "normalized_text_offset": 1862
        }
      ],
      "limitations": "A first-party operational report, not a universal scheduling guarantee.",
      "checks": [
        "publisher URL and final redirect inspected",
        "actual relevant primary passage read",
        "exact excerpt matched",
        "scope and available version/date inspected"
      ]
    }
  ],
  "claims": [
    {
      "id": "C01",
      "notebook_locations": [
        "workflow"
      ],
      "claim": "Accept work — Request queue — Keep pending requests until execution capacity is available. — The experiment gives every request arrival time zero.; Admit requests — Available slots — Choose which queued request enters each free slot. — The toy uses FIFO; real fairness and admission policies are separate design choices.; Advance work — One unit per tick — Each active request consumes one unit of its declared duration. — This deliberately excludes prefill costs, memory limits and unequal per-step work.; Release slots — Finish or refill — A static batch waits for its longest member; refilling admits new work as slots free. — Compare idle cells and completion time under the same input durations.",
      "source_ids": [
        "S1",
        "S2",
        "S3"
      ],
      "evidence_ids": [
        "S1-E1",
        "S2-E1",
        "S3-E1"
      ],
      "verdict": "editorial_diagram_with_source_supported_roles",
      "reason": "Diagram is a teaching workflow, not an exhaustive protocol or measured system. Each role and arrow is bounded in nearby text."
    },
    {
      "id": "C02",
      "notebook_locations": [
        "experiment"
      ],
      "claim": "Four requests arrive together: A needs 2 ticks, B 6, C 1 and D 4. Both policies use the same slot count. Static batches have a barrier between successive groups; refilling replaces a finished request on the next tick. Every active request advances by one unit per tick. There is no memory pressure, prefill stage, network overhead or cancellation. Slot occupancy is busy slot-ticks divided by capacity slot-ticks, not GPU utilization. Static policy at 2 slots: 10 ticks: A/B form the first batch (6 ticks); C/D form the second (4 ticks). Refill policy at 2 slots: 7 ticks: C starts after A, then D uses that slot while B continues. Total useful work: 13 slot-ticks: Both policies perform the same declared work; the idle capacity differs.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_synthetic_experiment",
      "reason": "Original fixed inputs and outputs; Python assertions and independent browser arithmetic checked in verification artifacts."
    },
    {
      "id": "C03",
      "notebook_locations": [
        "workflow"
      ],
      "claim": "A fixed batch can leave capacity unused when one sequence finishes before its batch peers. Continuous batching changes active membership as requests finish and new work enters. The Hugging Face explanation develops that idea from inference mechanics; the current manager documentation shows request-oriented control.",
      "source_ids": [
        "S1",
        "S2"
      ],
      "evidence_ids": [
        "S1-E1",
        "S2-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C04",
      "notebook_locations": [
        "workflow"
      ],
      "claim": "Admission policy is a separate question from batch membership. A queue can preserve arrival order, prioritize classes or distribute turns among users. TNG’s operational report describes per-user queues and a separate scheduling layer. Its example also shows why filling a downstream queue can undermine a fairness policy implemented upstream.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C05",
      "notebook_locations": [
        "interpretation"
      ],
      "claim": "At two slots, the static timeline contains 20 capacity slot-ticks and 13 busy slot-ticks, so occupancy is 65%. Refilling finishes in 7 ticks, giving 13/14 ≈ 92.9%. This synthetic difference is caused by the barrier; it is not a measured speedup for a serving product.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C06",
      "notebook_locations": [
        "interpretation"
      ],
      "claim": "Changing to four slots lets all four requests start together, so both policies finish at tick 6. This counterexample matters: refilling creates no benefit in this fixture when there is no pending request to admit. Real batch construction also interacts with sequence length, memory and scheduling overhead.",
      "source_ids": [
        "S1",
        "S2"
      ],
      "evidence_ids": [
        "S1-E1",
        "S2-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C07",
      "notebook_locations": [
        "interpretation"
      ],
      "claim": "Editorial operations exercise: record queue delay separately from execution time, state the admission policy and watch where requests accumulate. Do not infer fairness from good occupancy. A scheduler can keep every slot busy while making one user wait disproportionately.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "source_supported_with_stated_scope",
      "reason": "Read against the stated source sections and notebook assumptions; see separate source audit. "
    },
    {
      "id": "C08",
      "notebook_locations": [
        "reproduce"
      ],
      "claim": "The original standard-library Python example passed its assertions and 4 input fixtures. Its browser implementation is separately compared against those results.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "tested_original_example",
      "reason": "No publisher code executed; no model, external network call or privileged action in the examples."
    },
    {
      "id": "C09",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "Why can the static policy show idle cells? Correct answer: It waits for the current batch’s longest request. Finished slots remain unused until that batch finishes in this declared static policy.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    },
    {
      "id": "C10",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "What does 92.9% mean in the two-slot refill example? Correct answer: Busy slot-ticks / available slot-ticks. It is a synthetic capacity occupancy ratio, not a hardware metric.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    },
    {
      "id": "C11",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "Does continuous membership itself specify per-user fairness? Correct answer: No. Batch membership and request admission policy answer different questions.",
      "source_ids": [
        "S3"
      ],
      "evidence_ids": [
        "S3-E1"
      ],
      "verdict": "source_supported_and_worked_example",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    },
    {
      "id": "C12",
      "notebook_locations": [
        "self-check"
      ],
      "claim": "At four slots with four initially queued requests, what happens here? Correct answer: Both policies finish at tick 6. All requests start together; there is no waiting work to refill a slot.",
      "source_ids": [],
      "evidence_ids": [],
      "verdict": "original_example_or_reasoned_limitation",
      "reason": "Answer rechecked against the declared fixture and cited source scope."
    }
  ],
  "original_experiment": {
    "scope": "A discrete scheduling example. Ticks are abstract units, not token latency or measured GPU time.",
    "assumptions": "Every active request advances by one unit per tick. There is no memory pressure, prefill stage, network overhead or cancellation. Slot occupancy is busy slot-ticks divided by capacity slot-ticks, not GPU utilization.",
    "results_file": "verification/python-results.json"
  },
  "library_gap": null,
  "excluded_evidence": [],
  "limitations": [
    "No real model or GPU measured.",
    "Sources and version/date labels checked as recorded; this does not prove absence of errors."
  ],
  "separate_audit_file": "verification/source-audit.json",
  "publication": {
    "state": "published",
    "approved_on": "2026-11-14",
    "published_on": "2026-11-14",
    "source_checks_on": "2026-11-13"
  }
}
