"""Greedy-output agreement between benchmark runs (temperature 0): identical responses and mean shared-prefix
fraction (characters) of each run against a reference run.
usage: compare_outputs.py REF.json RUN.json [RUN.json ...]"""
import json
import os
import sys


def load(p):
    return {r["id"]: r["text"] for r in json.load(open(p))["per_request"]}


ref = load(sys.argv[1])
for p in sys.argv[2:]:
    run = load(p)
    same, frac = 0, []
    for k, a in ref.items():
        b = run.get(k, "")
        same += a == b
        n = next((i for i, (x, y) in enumerate(zip(a, b)) if x != y), min(len(a), len(b)))
        frac.append(n / max(len(a), 1))
    print(f"{os.path.basename(p):28s} identical {same}/{len(ref)}  shared prefix {100 * sum(frac) / len(frac):5.1f}%")