"""Illustrative synthetic lab; Python 3.10+ standard library only. No shell, file, network, or model calls. Not a production security boundary. C22 covers fixtures, arithmetic, validation behavior, and assertions. C21 covers the original host gate design. """ import json import math from fractions import Fraction def rrf(rankings, constant=60): if type(constant) is not int or constant < 0: raise ValueError('constant must be a nonnegative integer') scores = {} for ranking in rankings: if any(type(doc) is not str or not doc for doc in ranking): raise ValueError('IDs must be nonempty strings') if len(ranking) != len(set(ranking)): raise ValueError('duplicate ID within a ranking') for position, doc_id in enumerate(ranking, 1): contribution = Fraction(1, constant + position) scores[doc_id] = scores.get(doc_id, Fraction(0)) + contribution return sorted(scores.items(), key=lambda item: (-item[1], item[0])) def dcg(grades): return sum((2 ** grade - 1) / math.log2(position + 1) for position, grade in enumerate(grades, 1)) def ndcg(ranking, judgments, cutoff): if type(cutoff) is not int or cutoff < 1: raise ValueError('cutoff must be a positive integer') if len(ranking) != len(set(ranking)): raise ValueError('duplicate returned ID') if any(doc not in judgments for doc in ranking): raise ValueError('unjudged returned ID') if any(type(g) is not int or not 0 <= g <= 4 for g in judgments.values()): raise ValueError('this fixture requires integer grades from 0 to 4') actual = dcg([judgments[doc] for doc in ranking[:cutoff]]) ideal = dcg(sorted(judgments.values(), reverse=True)[:cutoff]) return actual / ideal if ideal else 0.0 TOOL_NAME = 'query_knowledge_base' TOOL_SCHEMA = { 'type': 'object', 'properties': { 'query': {'type': 'string', 'minLength': 1, 'maxLength': 200}, 'top_k': {'type': 'integer', 'minimum': 1, 'maximum': 20}, }, 'required': ['query', 'top_k'], 'additionalProperties': False, } def unique_object(pairs): obj = {} for key, value in pairs: if key in obj: raise ValueError('duplicate JSON key') obj[key] = value return obj def reject_constant(value): raise ValueError('nonstandard JSON constant') def validate_call(name, raw): # A narrow manual validator, with additional local size/whitespace rules. if name != TOOL_NAME: raise ValueError('unknown tool') if type(raw) is not str or len(raw) > 2048: raise ValueError('invalid or oversized request') try: args = json.loads(raw, object_pairs_hook=unique_object, parse_constant=reject_constant) except (ValueError, RecursionError) as exc: raise ValueError('invalid JSON') from exc if type(args) is not dict: raise ValueError('arguments must be an object') if set(args) != set(TOOL_SCHEMA['required']): raise ValueError('missing or extra fields') query = args['query'] count = args['top_k'] if type(query) is not str or not query.strip() or len(query) > 200: raise ValueError('query must contain 1 to 200 characters') if type(count) is not int or not 1 <= count <= 20: raise ValueError('top_k must be an integer from 1 to 20') return args PERMISSIONS = {'reader': frozenset({TOOL_NAME}), 'guest': frozenset()} SYNTHETIC_IDS = ('A', 'B', 'C', 'D', 'E') def guarded_call(principal, name, raw, attempts_left): # principal and attempts_left are trusted host inputs, not model arguments. if type(attempts_left) is not int or attempts_left <= 0: return {'status': 'stopped', 'reason': 'budget', 'remaining': 0} remaining = attempts_left - 1 try: args = validate_call(name, raw) except ValueError as exc: return {'status': 'rejected', 'reason': str(exc), 'remaining': remaining} if name not in PERMISSIONS.get(principal, frozenset()): return {'status': 'denied', 'reason': 'permission', 'remaining': remaining} # Synthetic local execution: no search engine or external operation. return {'status': 'ok', 'remaining': remaining, 'observation': {'trust': 'untrusted', 'ids': list(SYNTHETIC_IDS[:args['top_k']])}} def expect_value_error(function, *args): try: function(*args) except ValueError: return raise AssertionError('expected ValueError') def main(): dense = ['C', 'A', 'D', 'B'] lexical = ['B', 'A', 'E', 'C'] judgments = {'A': 3, 'B': 2, 'C': 2, 'D': 0, 'E': 1} fused = rrf([dense, lexical]) fused_ids = [doc for doc, score in fused] assert fused_ids == ['A', 'B', 'C', 'D', 'E'] assert fused[0][1] == Fraction(1, 31) assert fused[1][1] == fused[2][1] assert ndcg(fused_ids, judgments, 3) == 1.0 assert math.isclose(ndcg(dense, judgments, 3), 0.7136205775898136) assert ndcg(['D'], judgments, 3) == 0.0 assert ndcg([], judgments, 3) == 0.0 assert ndcg(['Z'], {'Z': 0}, 1) == 0.0 expect_value_error(rrf, [['A', 'A']]) expect_value_error(ndcg, ['A', 'A'], judgments, 3) expect_value_error(ndcg, ['unknown'], judgments, 3) imported_grades = [2, 3, 0, 1, 2] imported_ratio = dcg(imported_grades) / dcg(sorted(imported_grades, reverse=True)) assert math.isclose(imported_ratio, 0.8322420383257692) good = json.dumps({'query': 'AX-17 mode Z', 'top_k': 3}) allowed = guarded_call('reader', TOOL_NAME, good, 2) denied = guarded_call('guest', TOOL_NAME, good, 2) stopped = guarded_call('reader', TOOL_NAME, good, 0) assert allowed['observation']['ids'] == ['A', 'B', 'C'] assert denied['status'] == 'denied' assert stopped['status'] == 'stopped' invalid_inputs = [ '[]', '{"query": "x", "top_k": true}', '{"query": "x", "top_k": 99}', '{"query": "x", "top_k": 2, "admin": true}', '{"query": "x", "query": "y", "top_k": 2}', '{"query": " ", "top_k": 2}', ] for raw in invalid_inputs: result = guarded_call('reader', TOOL_NAME, raw, 2) assert result['status'] == 'rejected' assert result['remaining'] == 1 for doc, score in fused: print(doc, f'{float(score):.6f}') print('Synthetic dense nDCG@3:', f'{ndcg(dense, judgments, 3):.4f}') print('Synthetic fused nDCG@3:', f'{ndcg(fused_ids, judgments, 3):.4f}') print('Separate five-grade nDCG@5:', f'{imported_ratio:.4f}') print('Tool gates:', allowed['status'], denied['status'], stopped['status']) if __name__ == '__main__': main()