"""Illustrative synthetic MoE arithmetic; standard library only. No model, file, shell or network operations. No measured output is claimed. Source premises: S1 noisy gating, S2 top-1 balance, S3 expert choice, S6 bias selection. All inputs and implementation are original teaching examples. """ import math import random def softmax(values): peak = max(values) terms = [math.exp(value - peak) for value in values] total = sum(terms) return [term / total for term in terms] def softplus(value): return max(value, 0.0) + math.log1p(math.exp(-abs(value))) def sparse_gate(logits, k): assert 1 <= k <= len(logits) ids = sorted(range(len(logits)), key=lambda i: (-logits[i], i))[:k] weights = softmax([logits[i] for i in ids]) return list(zip(ids, weights)) def noisy_gate(logits, noise_projection, k, rng): assert len(logits) == len(noise_projection) scores = [value + rng.gauss(0.0, 1.0) * softplus(scale) for value, scale in zip(logits, noise_projection)] return sparse_gate(scores, k) def assignment_counts(routes, experts): counts = [0] * experts for route in routes: for expert, _ in route: counts[expert] += 1 return counts def admit(routes, experts, capacity): # Synthetic policy: input order, no rerouting or renormalization. counts = [0] * experts dropped = [] for token, route in enumerate(routes): for expert, _ in route: if counts[expert] < capacity: counts[expert] += 1 else: dropped.append((token, expert)) return counts, dropped def switch_top1(probabilities, alpha=0.01): # Full probabilities are separate from sparse top-2 gates. tokens = len(probabilities) experts = len(probabilities[0]) assert all(len(row) == experts for row in probabilities) assert all(all(value >= 0.0 for value in row) and math.isclose(sum(row), 1.0) for row in probabilities) counts = [0] * experts for row in probabilities: winner = max(range(experts), key=lambda i: row[i]) counts[winner] += 1 fractions = [count / tokens for count in counts] means = [sum(row[i] for row in probabilities) / tokens for i in range(experts)] loss = alpha * experts * sum(f * p for f, p in zip(fractions, means)) return fractions, means, loss def expert_choice(scores, slots): tokens, experts = len(scores), len(scores[0]) assert 0 <= slots <= tokens participation = [0] * tokens selected = [] for expert in range(experts): ids = sorted(range(tokens), key=lambda t: (-scores[t][expert], t))[:slots] selected.append(ids) for token in ids: participation[token] += 1 return selected, participation def biased_gate(affinities, biases, k): # Synthetic arithmetic only; no node limits or training loop. ids = sorted(range(len(affinities)), key=lambda i: (-(affinities[i] + biases[i]), i))[:k] total = sum(affinities[i] for i in ids) return [(i, affinities[i] / total) for i in ids] def update_bias(biases, counts, gamma): target = sum(counts) / len(counts) return [bias - gamma if count > target else bias + gamma if count < target else bias for bias, count in zip(biases, counts)] def communication(routes, origins, placement, width, element_bytes): remote = [(token, expert) for token, route in enumerate(routes) for expert, _ in route if origins[token] != placement[expert]] # One input and one output vector per remote assignment. payload = 2 * len(remote) * width * element_bytes return remote, payload def parameters(layers, shared, expert, embedding, experts, selected): assert 1 <= selected <= experts total = embedding + layers * (shared + experts * expert) path = embedding + layers * (shared + selected * expert) return total, path def main(): logits = [[math.log(3.0), -2.0, -3.0, 0.0], [2.0, 1.0, -2.0, -3.0], [2.0, -2.0, 1.0, -3.0], [-2.0, 2.0, 1.0, -3.0]] routes = [sparse_gate(row, 2) for row in logits] assert [[i for i, _ in route] for route in routes] == [ [0, 3], [0, 1], [0, 2], [1, 2]] assert all(math.isclose(sum(w for _, w in route), 1.0) for route in routes) assert math.isclose(routes[0][0][1], 0.75) outputs = {0: [2.0, -1.0], 3: [-2.0, 3.0]} combined = [sum(weight * outputs[expert][j] for expert, weight in routes[0]) for j in range(2)] assert all(math.isclose(a, b, abs_tol=1e-12) for a, b in zip(combined, [1.0, 0.0])) counts = assignment_counts(routes, 4) assert counts == [3, 2, 2, 1] and sum(counts) == 8 admitted, rejected = admit(routes, 4, 2) assert admitted == [2, 2, 2, 1] and rejected == [(2, 0)] assert admit(routes, 4, 3) == (counts, []) print('Synthetic top-2 counts:', counts) print('Selection rates:', [n / 4 for n in counts]) print('Assignment shares:', [n / 8 for n in counts]) print('Capacity-two admission:', admitted, rejected) rng = random.Random(2026) noisy = [noisy_gate(row, [0.0] * 4, 2, rng) for row in logits] assert sum(assignment_counts(noisy, 4)) == 8 print('Illustrative noisy counts:', assignment_counts(noisy, 4)) rows = [[0.51, 0.49] for _ in range(9)] + [[0.0, 1.0]] f, p, loss = switch_top1(rows) assert f == [0.9, 0.1] assert math.isclose(p[0], 0.459) assert math.isclose(loss, 0.009344) and loss < 0.01 uniform_rows = [[0.6, 0.4], [0.4, 0.6]] assert math.isclose(switch_top1(uniform_rows)[2], 0.01) print('Separate top-1 counterexample:', f, p, loss) scores = [[0.5, 0.5], [0.6, 0.4], [0.4, 0.6]] assert expert_choice(scores, 2)[1] == [2, 1, 1] assert expert_choice(scores, 1)[1] == [0, 1, 1] print('Expert-choice participation:', expert_choice(scores, 1)[1]) biased = biased_gate([0.8, 0.7, 0.6], [-0.3, 0.0, 0.2], 2) assert [i for i, _ in biased] == [2, 1] assert math.isclose(biased[0][1], 0.6 / 1.3) assert update_bias([0.0] * 3, [6, 1, 1], 0.01) == [-0.01, 0.01, 0.01] print('Unbiased weights after biased selection:', biased) remote, payload = communication(routes, [0, 0, 1, 1], [0, 0, 1, 1], 4096, 2) assert remote == [(0, 3), (2, 0), (3, 1)] assert payload == 49152 print('Remote assignments and payload bytes:', remote, payload) total, path = parameters(32, 33554432, 176160768, 262144000, 8, 2) assert total == 46433042432 and path == 12610174976 assert math.comb(8, 2) == 28 and math.comb(16, 4) == 1820 print('Synthetic stored / selected-path parameters:', total, path) print('Selected-path fraction:', path / total) if __name__ == '__main__': main()