{"model": "bench", "per_request": [{"id": "HumanEval/40", "cat": "code", "n": 729, "s": 2.727701382013038}, {"id": "HumanEval/41", "cat": "code", "n": 414, "s": 1.7476835920242593}, {"id": "HumanEval/42", "cat": "code", "n": 179, "s": 0.7658775619929656}, {"id": "HumanEval/43", "cat": "code", "n": 861, "s": 3.1240575670381077}, {"id": "HumanEval/44", "cat": "code", "n": 714, "s": 2.5527657009661198}, {"id": "HumanEval/45", "cat": "code", "n": 168, "s": 0.8923461069935001}, {"id": "gsm8k/40", "cat": "math", "n": 306, "s": 1.2758885299554095}, {"id": "gsm8k/41", "cat": "math", "n": 602, "s": 2.479146101977676}, {"id": "gsm8k/42", "cat": "math", "n": 163, "s": 0.7538518089568242}, {"id": "gsm8k/43", "cat": "math", "n": 484, "s": 1.839121469995007}, {"id": "gsm8k/44", "cat": "math", "n": 401, "s": 1.5070016390527599}, {"id": "gsm8k/45", "cat": "math", "n": 346, "s": 1.3740820209495723}, {"id": "mtbench/56561594", "cat": "chat", "n": 1024, "s": 3.974660051986575}, {"id": "mtbench/3583619", "cat": "chat", "n": 988, "s": 3.468062022002414}, {"id": "mtbench/34377376", "cat": "chat", "n": 853, "s": 3.326899648993276}, {"id": "mtbench/46306085", "cat": "chat", "n": 1024, "s": 3.90042637498118}, {"id": "mtbench/50530142", "cat": "chat", "n": 1024, "s": 4.4263171890052035}, {"id": "mtbench/65906068", "cat": "chat", "n": 1024, "s": 4.085645671002567}, {"id": "glaive/40", "cat": "tool", "n": 71, "s": 0.4537901899893768}, {"id": "glaive/41", "cat": "tool", "n": 155, "s": 0.9037347360281274}, {"id": "glaive/42", "cat": "tool", "n": 98, "s": 0.5890757929882966}, {"id": "glaive/43", "cat": "tool", "n": 75, "s": 0.5211616730084643}, {"id": "glaive/44", "cat": "tool", "n": 71, "s": 0.4933487140224315}, {"id": "glaive/45", "cat": "tool", "n": 71, "s": 0.4984891830245033}], "code": 259.51633463447314, "math": 249.42866611721078, "chat": 256.10375263663406, "tool": 156.37644664054707, "all": 248.42110128744045, "tau": 3.307949790794979, "accept_per_pos": [0.8797768479776849, 0.7668061366806137, 0.6613668061366806]}