"""Illustrative synthetic cache bookkeeping; no model calls.""" from math import prod def payload_bytes(positions, batch=1, kv_heads=8): dimensions = (24, batch, kv_heads, 64, 2) for value in dimensions: if type(value) is not int or value <= 0: raise ValueError("Dimensions must be positive ints") if type(positions) is not int or positions < 0: raise ValueError("Positions must be a nonnegative int") return 2 * positions * prod(dimensions) def trace(prompt, selections): if not prompt or not selections: raise ValueError("Provide prompt and selections") cached = [] current_input = tuple(prompt) rows = [] for selected in selections: # Represent completion of a forward with new inputs. cached.extend(current_input) rows.append((current_input, tuple(cached), selected)) # The selection is processed only on a later iteration. current_input = (selected,) return rows def main(): prompt = ("P0", "P1", "P2", "P3") selections = ("A", "B", "C") rows = trace(prompt, selections) assert [len(row[1]) for row in rows] == [4, 5, 6] assert rows[-1][1] == prompt + ("A", "B") assert len(rows) == len(selections) expected = len(prompt) + len(selections) - 1 assert len(rows[-1][1]) == expected assert trace(prompt, ("A",))[0][1] == prompt assert payload_bytes(0) == 0 assert payload_bytes(1) == 49152 assert payload_bytes(2048) == 100663296 assert payload_bytes(2048, batch=3) == 288 * 2**20 assert payload_bytes(2048, kv_heads=32) == 384 * 2**20 for new_input, cached, selected in rows: print("Input:", new_input) print("Cached:", cached) print("Selected next:", selected) print("Payload KiB:", payload_bytes(len(cached)) // 1024) if __name__ == "__main__": main()