"""Illustrative synthetic PEFT/NF4 lab; Python standard library only. No model execution or performance measurements. Claims C4, C6, C10, C16-C18 describe the arithmetic and limitations. """ from math import isfinite, sqrt # Rounded reconstruction values from QLoRA Appendix E (S3). # This teaching approximation is not a bitsandbytes implementation. LEVELS = ( -1.0, -0.696193, -0.525073, -0.394917, -0.284441, -0.184773, -0.091050, 0.0, 0.079580, 0.160930, 0.246112, 0.337915, 0.440710, 0.562617, 0.722957, 1.0, ) def shape(matrix): if not matrix or not matrix[0]: raise ValueError('matrix must be nonempty') width = len(matrix[0]) if any(len(row) != width for row in matrix): raise ValueError('ragged matrix') return len(matrix), width def matvec(matrix, vector): _, width = shape(matrix) if width != len(vector): raise ValueError('incompatible vector') return [sum(a * b for a, b in zip(row, vector)) for row in matrix] def matmul(left, right): rows, inner = shape(left) other_inner, cols = shape(right) if inner != other_inner: raise ValueError('incompatible matrices') return [[sum(left[i][k] * right[k][j] for k in range(inner)) for j in range(cols)] for i in range(rows)] def merge(w0, a, b, scale): update = matmul(b, a) if shape(update) != shape(w0): raise ValueError('incompatible update') return [[w + scale * d for w, d in zip(wrow, drow)] for wrow, drow in zip(w0, update)] def dora_columns(direction, magnitudes): rows, cols = shape(direction) if len(magnitudes) != cols: raise ValueError('one magnitude per column required') norms = [sqrt(sum(direction[i][j] ** 2 for i in range(rows))) for j in range(cols)] if any(norm == 0.0 for norm in norms): raise ValueError('zero column has no defined unit direction') return [[magnitudes[j] * direction[i][j] / norms[j] for j in range(cols)] for i in range(rows)] def parameter_counts(d_in, d_out, rank, targets): if any(type(n) is not int or n <= 0 for n in (d_in, d_out, rank, targets)): raise ValueError('positive integer dimensions required') return (targets * d_in * d_out, targets * rank * (d_in + d_out)) def quantize_block(values): if not values or any(not isfinite(v) for v in values): raise ValueError('finite nonempty block required') scale = max(abs(v) for v in values) if scale == 0.0: return [7] * len(values), 0.0 # Nearest rounded reconstruction value; ties use the lower index. indices = [min(range(16), key=lambda k: (abs(v / scale - LEVELS[k]), k)) for v in values] return indices, scale def reconstruct(indices, scale): if not isfinite(scale) or scale < 0: raise ValueError('invalid scale') if any(type(i) is not int or not 0 <= i < 16 for i in indices): raise ValueError('invalid index') return [scale * LEVELS[i] for i in indices] def pack_indices(indices): if any(type(i) is not int or not 0 <= i < 16 for i in indices): raise ValueError('four-bit indices required') # Our format puts the first index in the low nibble. packed = bytearray() for offset in range(0, len(indices), 2): low = indices[offset] high = indices[offset + 1] if offset + 1 < len(indices) else 0 packed.append(low | (high << 4)) return bytes(packed) def unpack_indices(payload, count): if type(count) is not int or count < 0 or len(payload) != (count + 1) // 2: raise ValueError('invalid count or payload length') indices = [] for byte in payload: indices.extend((byte & 15, byte >> 4)) return indices[:count] def quantize_blocks(values, block_size): if type(block_size) is not int or block_size <= 0: raise ValueError('positive integer block size required') records = [] for start in range(0, len(values), block_size): indices, scale = quantize_block(values[start:start + block_size]) records.append((pack_indices(indices), scale, len(indices))) return records def restore_blocks(records): restored = [] for payload, scale, count in records: restored.extend(reconstruct(unpack_indices(payload, count), scale)) return restored def main(): full, adapter = parameter_counts(4096, 4096, 16, 4) assert (full, adapter) == (67108864, 524288) assert full == 128 * adapter print('selected projection counts:', full, adapter) w0 = [[1.0, 2.0], [3.0, 4.0]] a = [[1.0, -1.0]] b = [[0.5], [-0.25]] x = [2.0, 1.0] scale = 2.0 base = matvec(w0, x) delta = matvec(b, matvec(a, x)) unmerged = [u + scale * v for u, v in zip(base, delta)] merged = matvec(merge(w0, a, b, scale), x) assert all(abs(u - v) < 1e-12 for u, v in zip(unmerged, merged)) assert merged == [5.0, 9.5] print('synthetic merged output:', merged) initial = merge(w0, a, [[0.0], [0.0]], scale) assert initial == w0 assert dora_columns([[3.0], [4.0]], [10.0]) == [[6.0], [8.0]] # Explicit zero-block and odd-count behavior. zero_indices, zero_scale = quantize_block([0.0] * 5) assert reconstruct(zero_indices, zero_scale) == [0.0] * 5 for indices in ([], [7], [0, 15, 7], list(range(16))): payload = pack_indices(indices) assert len(payload) == (len(indices) + 1) // 2 assert unpack_indices(payload, len(indices)) == indices # Binary fractions keep the input specification easy to inspect. weights = [-1.0, -0.5, -0.125, 0.0, 0.125, 0.25, 0.5, 1.0, 0.0] for block_size in (4, 8): records = quantize_blocks(weights, block_size) restored = restore_blocks(records) assert len(restored) == len(weights) rmse = sqrt(sum((u - v) ** 2 for u, v in zip(weights, restored)) / len(weights)) index_bytes = sum(len(record[0]) for record in records) print('synthetic block size, RMSE, index bytes, scale count:', block_size, rmse, index_bytes, len(records)) # Ideal payload arithmetic only: double quantization is not implemented. first_level_overhead = 32 / 64 nested_overhead = 8 / 64 + 32 / (64 * 256) assert first_level_overhead == 0.5 assert nested_overhead == 0.126953125 print('ideal bits per weight with nested scales:', 4 + nested_overhead) if __name__ == '__main__': main()