"""Original, dependency-free retrieval experiment using fictional documents.

Convention: binary relevance, unique document IDs, at least k ranked results,
nonempty relevant set, and zero reciprocal rank when no relevant result is in k.
No model, retriever, external service or source-supplied code is executed.
"""
from fractions import Fraction
from itertools import permutations
import json

CORPUS = {
    "D1": "Choose Export in the notebook menu.",
    "D2": "Select JSON as the output format.",
    "D3": "Download the file when its status says Ready.",
    "D4": "The demo help desk opens from 09:00 to 17:00 UTC.",
    "D5": "Deleted demo drafts stay in Trash for seven days.",
    "D6": "Choose a theme under Appearance.",
}
CASES = {
    "export": {
        "question": "How do I export my notes?",
        "relevant": {"D1", "D2", "D3"},
        "A": ["D1", "D4", "D6", "D2", "D5", "D3"],
        "B": ["D1", "D2", "D3", "D4", "D5", "D6"],
    },
    "hours": {
        "question": "When is the demo help desk open?",
        "relevant": {"D4"},
        "A": ["D1", "D6", "D4", "D2", "D3", "D5"],
        "B": ["D6", "D4", "D1", "D2", "D3", "D5"],
    },
    "trash": {
        "question": "How long are deleted demo drafts retained?",
        "relevant": {"D5"},
        "A": ["D6", "D1", "D2", "D3", "D4", "D5"],
        "B": ["D5", "D6", "D1", "D2", "D3", "D4"],
    },
}


def metrics(ranking, relevant, k):
    if not 1 <= k <= len(ranking):
        raise ValueError("Require 1 <= k <= number of returned results")
    if len(ranking) != len(set(ranking)):
        raise ValueError("Duplicate document IDs must be resolved first")
    if not relevant:
        raise ValueError("Recall needs a declared zero-relevance convention")
    top = ranking[:k]
    hits = sum(doc in relevant for doc in top)
    first = next((rank for rank, doc in enumerate(top, 1)
                  if doc in relevant), None)
    return {
        "precision": Fraction(hits, k),
        "recall": Fraction(hits, len(relevant)),
        "rr": Fraction(1, first) if first is not None else Fraction(0),
    }


def macro_scores(mode, k):
    rows = [metrics(case[mode], case["relevant"], k)
            for case in CASES.values()]
    return {name: sum(row[name] for row in rows) / len(rows)
            for name in rows[0]}


def verify():
    # Independently specified exact expectations for the worked k=3 example.
    assert metrics(CASES["export"]["A"], {"D1", "D2", "D3"}, 3) == {
        "precision": Fraction(1, 3), "recall": Fraction(1, 3), "rr": 1}
    assert macro_scores("A", 3) == {
        "precision": Fraction(2, 9), "recall": Fraction(4, 9),
        "rr": Fraction(4, 9)}
    assert macro_scores("B", 3) == {
        "precision": Fraction(5, 9), "recall": 1,
        "rr": Fraction(5, 6)}
    # Same top-k membership, different first-hit position.
    assert metrics(["D1", "D4", "D2"], {"D1", "D2"}, 3)["rr"] == 1
    assert metrics(["D4", "D1", "D2"], {"D1", "D2"}, 3)["rr"] == Fraction(1, 2)
    checked = 0
    for ranking in permutations(CORPUS):
        previous_recall = Fraction(0)
        for k in range(1, 7):
            score = metrics(ranking, {"D1", "D2", "D3"}, k)
            assert all(0 <= value <= 1 for value in score.values())
            assert score["recall"] >= previous_recall
            previous_recall = score["recall"]
            # Set intersection independently checks the positive-hit count.
            assert score["precision"] * k == len(set(ranking[:k]) & {"D1", "D2", "D3"})
            checked += 1
    for ranking, relevant, k in [(["D1", "D1"], {"D1"}, 1),
                                 (["D1"], set(), 1),
                                 (["D1"], {"D1"}, 2)]:
        try:
            metrics(ranking, relevant, k)
        except ValueError:
            pass
        else:
            raise AssertionError("Invalid input was accepted")
    return checked


if __name__ == "__main__":
    checked = verify()
    fixtures = []
    for key, case in CASES.items():
        for mode in ("A", "B"):
            for k in range(1, 7):
                score = metrics(case[mode], case["relevant"], k)
                fixtures.append({"query": key, "mode": mode, "k": k,
                                 **{name: float(value) for name, value in score.items()}})
    print(json.dumps({"permutation_cutoff_checks": checked, "corpus": CORPUS,
                      "cases": {key: {**case, "relevant": sorted(case["relevant"])}
                                for key, case in CASES.items()}, "fixtures": fixtures,
                      "macro_at_3": {mode: {name: str(value) for name, value
                                            in macro_scores(mode, 3).items()}
                                     for mode in ("A", "B")}}, indent=2))
