{"version":1,"generatedAt":"2026-11-10T01:56:32.791Z","attribution":"Compiled by HARBAN AI LABS from the linked sources. Cite the original source for any number you reuse.","documentation":"https://harban-ai-labs.vercel.app/benchmarks/methodology","count":14,"data":[{"slug":"aime","url":"https://harban-ai-labs.vercel.app/benchmarks/aime","name":"American Invitational Mathematics Examination (AIME), as used in LLM evals","shortName":"AIME","category":"math","year":null,"paperUrl":null,"siteUrl":"https://maa.org/maa-invitational-competitions/","measures":"Solving the 15 integer-answer (0–999) problems of a given year's AIME exam(s), typically averaged over repeated samples.","metric":"Accuracy (%), usually mean over k samples per problem","higherIsBetter":true,"health":{"status":"saturated","note":"The leading models on Artificial Analysis’s AIME 2025 leaderboard score at or near 100%, and Artificial Analysis now lists it as a legacy evaluation.","asOf":"2026-09-28","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025"},"caveats":[{"text":"Contamination: MathArena reported strong signs of contamination in AIME 2024.","sourceUrl":"https://arxiv.org/abs/2505.23281"},{"text":"Artificial Analysis removed AIME 2025 from its Intelligence Index in v4.0 (January 2026), without stating a reason, and has retired it from active reporting.","sourceUrl":"https://artificialanalysis.ai/methodology/intelligence-benchmarking"},{"text":"Small set: AIME 2025 has 30 problems in total, so one problem is worth about 3.3 percentage points and a single run is noisy.","sourceUrl":"https://artificialanalysis.ai/evaluations/aime-2025"}],"referenceVerified":false,"verifiedOn":"2026-09-28","results":[{"id":"muse-glimmer-30b-aime-2026","benchmark":"aime","benchmarkVariant":"2026","model":"muse-glimmer-30b","modelDetail":"High Reasoning","value":94.7,"unit":"%","settings":"High Reasoning","ci":null,"sourceType":"vendor-reported","evaluator":"Meta","sourceUrl":"https://huggingface.co/meta-models/Muse-Glimmer-30B","sourceTitle":null,"evidence":"| AIME 2026 | **94.7** | 89.2 | 94.1 |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gpt-oss-120b-aime-2025","benchmark":"aime","benchmarkVariant":"2025","model":"gpt-oss-120b","modelDetail":"reasoning level high","value":92.5,"unit":"%","settings":"no tools; reasoning level high (Table 3)","ci":null,"sourceType":"vendor-reported","evaluator":"OpenAI","sourceUrl":"https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf","sourceTitle":null,"evidence":"AIME 2025 (no tools) 50.4 80.0 92.5","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemma-4-31b-aime-2026","benchmark":"aime","benchmarkVariant":"2026","model":"gemma-4-31b","modelDetail":null,"value":89.2,"unit":"%","settings":"no tools; instruction-tuned model","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://huggingface.co/google/gemma-4-31B-it","sourceTitle":null,"evidence":"| AIME 2026 no tools | 89.2% |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]},{"slug":"arc-agi-1","url":"https://harban-ai-labs.vercel.app/benchmarks/arc-agi-1","name":"ARC-AGI-1 (Abstraction and Reasoning Corpus)","shortName":"ARC-AGI-1","category":"reasoning","year":2019,"paperUrl":"https://arxiv.org/abs/1911.01547","siteUrl":"https://arcprize.org/arc-agi/1/","measures":"Few-shot abstract pattern induction on grid puzzles from a handful of input-output examples.","metric":"Task accuracy (%) on the semi-private or private set (pass@2), reported with cost per task","higherIsBetter":true,"health":{"status":"saturated","note":"High-compute results on the semi-private set reached 87.5% in December 2024.","asOf":"2026-09-28","sourceUrl":"https://arcprize.org/blog/oai-o3-pub-breakthrough"},"caveats":[{"text":"o3-preview scored 75.7% (low compute) and 87.5% (high compute) on the semi-private set in December 2024; results depend heavily on compute spent.","sourceUrl":"https://arcprize.org/blog/oai-o3-pub-breakthrough"},{"text":"Only ARC Prize-run semi-private results are verified; public-set scores are self-reported.","sourceUrl":"https://arcprize.org/policy"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"arc-agi-2","url":"https://harban-ai-labs.vercel.app/benchmarks/arc-agi-2","name":"ARC-AGI-2","shortName":"ARC-AGI-2","category":"reasoning","year":2025,"paperUrl":"https://arxiv.org/abs/2505.11831","siteUrl":"https://arcprize.org/arc-agi/2","measures":"Harder, human-calibrated abstract reasoning grid tasks (120 each in public/semi-private/private eval sets) solvable by at least two humans in two attempts.","metric":"pass@2 accuracy (%) with cost per task","higherIsBetter":true,"health":{"status":"active","note":"The ARC-AGI-2 page still poses reaching 85% accuracy as an open challenge and links an ARC-AGI-2 competition; no saturation or retirement notice found as of the review date.","asOf":"2026-09-28","sourceUrl":"https://arcprize.org/arc-agi/2"},"caveats":[{"text":"Only scores ARC Prize runs itself on the Semi-Private set appear on its Verified Leaderboard; Community Leaderboard entries are not verified by default. Say which set and leaderboard a score comes from.","sourceUrl":"https://arcprize.org/policy"},{"text":"ARC Prize treats efficiency as part of the result: starting with ARC-AGI-2, all its reporting comes with a cost metric, so show cost alongside accuracy.","sourceUrl":"https://arcprize.org/arc-agi/2"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"gpqa-diamond","url":"https://harban-ai-labs.vercel.app/benchmarks/gpqa-diamond","name":"GPQA: A Graduate-Level Google-Proof Q&A Benchmark (Diamond subset)","shortName":"GPQA Diamond","category":"reasoning","year":2023,"paperUrl":"https://arxiv.org/abs/2311.12022","siteUrl":"https://github.com/idavidrein/gpqa","measures":"Graduate-level biology, physics and chemistry multiple-choice questions written to resist web lookup; Diamond is the 198-question highest-agreement subset.","metric":"Accuracy (%)","higherIsBetter":true,"health":{"status":"active","note":"An Epoch AI post (Greg Burnham, May 2025) estimated roughly 8% of questions may be invalid and judged that saturation had not yet been reached. Artificial Analysis later removed GPQA Diamond from its Intelligence Index (v4.2) without stating a reason.","asOf":"2026-09-28","sourceUrl":"https://epoch.ai/gradient-updates/gpqa-diamond-whats-left"},"caveats":[{"text":"Small set: with 198 questions, error bars are wide. An Epoch AI post by Greg Burnham estimated about 8% of questions may be invalid and reported top models clustering around 83% (May 2025).","sourceUrl":"https://epoch.ai/gradient-updates/gpqa-diamond-whats-left"},{"text":"Artificial Analysis removed GPQA Diamond from its Intelligence Index in v4.2.","sourceUrl":"https://artificialanalysis.ai/methodology/intelligence-benchmarking"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[{"id":"gemini-3-1-pro-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"gemini-3-1-pro","modelDetail":"Thinking (High)","value":94.3,"unit":"%","settings":"No tools; Thinking (High); all Gemini scores pass@1 unless noted (methodology PDF)","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","sourceTitle":null,"evidence":"GPQA Diamond Scientific knowledge No tools 94.3%","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"qwen3-8-max-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"qwen3-8-max","modelDetail":null,"value":92.6,"unit":"%","settings":"conditions not stated; column 'Qwen3.8-Max' on the Qwen3.8-2.4T-A95B card","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","sourceTitle":null,"evidence":"GPQA Diamond | 92.0 | 92.6 | 94.1 | 92.4 | 92.6 (last col = Qwen3.8-Max)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"deepseek-v4-1-flash-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"deepseek-v4-1-flash","modelDetail":"max reasoning effort","value":90.9,"unit":"%","settings":"Pass@1; max reasoning effort (reasoning_effort=100), temperature=1.0, top_p=0.95; last column = DS-V4.1-Flash","ci":null,"sourceType":"vendor-reported","evaluator":"DeepSeek","sourceUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash","sourceTitle":null,"evidence":"GPQA Diamond (Pass@1) | 93.4 | **94.1** | 92.9 | 88.1 | 92.4 | 89.9 | 90.9","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"qwen3-8-27b-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"qwen3-8-27b","modelDetail":null,"value":89.2,"unit":"%","settings":"conditions not stated","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-27B","sourceTitle":null,"evidence":"GPQA Diamond | 89.2 | 87.8 | 90.3 | 83.5 | 91.3 (first col = Qwen3.8-27B)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemma-4-31b-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"gemma-4-31b","modelDetail":null,"value":84.3,"unit":"%","settings":"instruction-tuned model; no other conditions stated","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://huggingface.co/google/gemma-4-31B-it","sourceTitle":null,"evidence":"| GPQA Diamond | 84.3% |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"muse-glimmer-30b-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"muse-glimmer-30b","modelDetail":"High Reasoning","value":83.5,"unit":"%","settings":"Full 198-question set; high reasoning; temperature 1.0, top_p 0.95, top_k 64. The model card marks the row “(AA)”; Meta’s linked methodology report states GPQA Diamond results are sourced from Artificial Analysis.","ci":null,"sourceType":"third-party","evaluator":"Artificial Analysis (republished by Meta)","sourceUrl":"https://huggingface.co/meta-models/Muse-Glimmer-30B","sourceTitle":"Muse Glimmer model card (Meta, Hugging Face)","evidence":"| GPQA Diamond (AA) | 83.5 | **85.7** | 84.2 |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gpt-oss-120b-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"gpt-oss-120b","modelDetail":"reasoning level high","value":80.1,"unit":"%","settings":"no tools; reasoning level high (Table 3; columns low/medium/high)","ci":null,"sourceType":"vendor-reported","evaluator":"OpenAI","sourceUrl":"https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf","sourceTitle":null,"evidence":"GPQA Diamond (no tools) 67.1 73.1 80.1","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"llama-4-maverick-gpqa-diamond-main","benchmark":"gpqa-diamond","benchmarkVariant":null,"model":"llama-4-maverick","modelDetail":null,"value":69.8,"unit":"%","settings":"0-shot; instruction-tuned model","ci":null,"sourceType":"vendor-reported","evaluator":"Meta","sourceUrl":"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct","sourceTitle":null,"evidence":"GPQA Diamond 0 accuracy 50.5 49.0 57.2 69.8","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]},{"slug":"humaneval","url":"https://harban-ai-labs.vercel.app/benchmarks/humaneval","name":"HumanEval","shortName":"HumanEval","category":"coding","year":2021,"paperUrl":"https://arxiv.org/abs/2107.03374","siteUrl":"https://github.com/openai/human-eval","measures":"Functional correctness of Python functions generated from docstrings, checked by unit tests.","metric":"pass@k (usually pass@1)","higherIsBetter":true,"health":{"status":"flawed","note":"Adding ~80× more tests (HumanEval+) lowered pass@k substantially, showing the original tests are weak.","asOf":"2026-09-28","sourceUrl":"https://arxiv.org/abs/2305.01210"},"caveats":[{"text":"Weak tests: adding roughly 80× more tests (HumanEval+) reduced pass@k by up to 19.3–28.9%.","sourceUrl":"https://arxiv.org/abs/2305.01210"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"humanitys-last-exam","url":"https://harban-ai-labs.vercel.app/benchmarks/humanitys-last-exam","name":"Humanity's Last Exam (HLE)","shortName":"HLE","category":"reasoning","year":2025,"paperUrl":"https://arxiv.org/abs/2501.14249","siteUrl":"https://lastexam.ai/","measures":"Expert-written closed-ended questions across many academic subjects intended to sit at the frontier of human knowledge (2,500 public questions).","metric":"Accuracy (%) (calibration error also reported)","higherIsBetter":true,"health":{"status":"active","note":"No saturation or retirement notice found as of the review date; see the label-error caveat.","asOf":"2026-09-28","sourceUrl":"https://lastexam.ai/"},"caveats":[{"text":"Label errors: FutureHouse found 29 ± 3.7% of text-only chemistry and biology answers conflicted with peer-reviewed literature (July 2025). Question selection deliberately targeted questions frontier models could not answer.","sourceUrl":"https://www.futurehouse.org/research/hle-exam"},{"text":"Versions matter: the maintainers released HLE-Rolling (Oct 2025) and HLE-Diamond, a refined 1,000-question subset (22 Sep 2026). State which set a score uses.","sourceUrl":"https://lastexam.ai/"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[{"id":"claude-opus-5-5-humanitys-last-exam-with-tools","benchmark":"humanitys-last-exam","benchmarkVariant":"With tools","model":"claude-opus-5-5","modelDetail":"adaptive thinking at max effort","value":67.7,"unit":"%","settings":"with tools; adaptive thinking at max effort (page default for all Opus 5.5 results unless otherwise noted). Anthropic evaluated Opus 5.5 with production safeguards enabled; when they intervened, biology and frontier LLM development tasks were completed by Claude Opus 5 and cybersecurity tasks by Claude Opus 4.8 (page note).","ci":null,"sourceType":"vendor-reported","evaluator":"Anthropic","sourceUrl":"https://www.anthropic.com/claude-opus-5-5","sourceTitle":null,"evidence":"Humanity's Last Exam 67.7% with tools","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemini-3-1-pro-humanitys-last-exam-with-tools","benchmark":"humanitys-last-exam","benchmarkVariant":"With tools","model":"gemini-3-1-pro","modelDetail":"Thinking (High)","value":51.4,"unit":"%","settings":"Search (blocklist) + Code; Thinking (High)","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","sourceTitle":null,"evidence":"Search (blocklist) + Code 51.4%","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemini-3-1-pro-humanitys-last-exam-no-tools","benchmark":"humanitys-last-exam","benchmarkVariant":"No tools","model":"gemini-3-1-pro","modelDetail":"Thinking (High)","value":44.4,"unit":"%","settings":"No tools; full set, text + MM; column 'Gemini 3.1 Pro Thinking (High)'","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","sourceTitle":null,"evidence":"Humanity's Last Exam Academic reasoning (full set, text + MM) No tools 44.4%","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"qwen3-8-max-humanitys-last-exam-tools-not-stated","benchmark":"humanitys-last-exam","benchmarkVariant":"Tools not stated","model":"qwen3-8-max","modelDetail":null,"value":43.6,"unit":"%","settings":"conditions not stated (a separate 'HLE w/ tools' row gives 56.2)","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","sourceTitle":null,"evidence":"HLE | 45.7 | 53.3 | 47.2 | 41.4 | 43.6 (last col = Qwen3.8-Max)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"deepseek-v4-1-flash-humanitys-last-exam-tools-not-stated","benchmark":"humanitys-last-exam","benchmarkVariant":"Tools not stated","model":"deepseek-v4-1-flash","modelDetail":"max reasoning effort","value":36.8,"unit":"%","settings":"Pass@1; max reasoning effort; tool setting not stated on this row (a separate 'HLE w/ tools' row gives 63.9); 39.1 on the text-only subset","ci":null,"sourceType":"vendor-reported","evaluator":"DeepSeek","sourceUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash","sourceTitle":null,"evidence":"HLE (Pass@1) | **56.3** | 44.5 | 43.5 | 42.0† | 42.7† | 37.8† | 36.8 (39.1†)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"qwen3-8-27b-humanitys-last-exam-tools-not-stated","benchmark":"humanitys-last-exam","benchmarkVariant":"Tools not stated","model":"qwen3-8-27b","modelDetail":null,"value":30.8,"unit":"%","settings":"HLE: Judged by GPT-4o (card footnote)","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-27B","sourceTitle":null,"evidence":"HLE | 30.8 | 24.0 | 34.7 | 22.0 | 40.0 (first col = Qwen3.8-27B)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemma-4-31b-humanitys-last-exam-no-tools","benchmark":"humanitys-last-exam","benchmarkVariant":"No tools","model":"gemma-4-31b","modelDetail":null,"value":19.5,"unit":"%","settings":"no tools; instruction-tuned model","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://huggingface.co/google/gemma-4-31B-it","sourceTitle":null,"evidence":"| HLE no tools | 19.5% |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gpt-oss-120b-humanitys-last-exam-no-tools","benchmark":"humanitys-last-exam","benchmarkVariant":"No tools","model":"gpt-oss-120b","modelDetail":"reasoning level high","value":14.9,"unit":"%","settings":"no tools; reasoning level high (Table 3)","ci":null,"sourceType":"vendor-reported","evaluator":"OpenAI","sourceUrl":"https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf","sourceTitle":null,"evidence":"HLE (no tools) 5.2 8.6 14.9","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]},{"slug":"livecodebench","url":"https://harban-ai-labs.vercel.app/benchmarks/livecodebench","name":"LiveCodeBench","shortName":"LiveCodeBench","category":"coding","year":2024,"paperUrl":"https://arxiv.org/abs/2403.07974","siteUrl":"https://livecodebench.github.io/","measures":"Code generation (plus self-repair, execution and test prediction) on contest problems from LeetCode, AtCoder and Codeforces, tagged by release date.","metric":"pass@1 (%) within a stated problem date window","higherIsBetter":true,"health":{"status":"saturated","note":"A 2026 study of saturation across 60 benchmarks rated LiveCodeBench highly saturated (saturation index 0.77): its top models’ scores are compressed relative to evaluation uncertainty.","asOf":"2026-09-28","sourceUrl":"https://arxiv.org/abs/2602.16763"},"caveats":[{"text":"Problems carry release dates and scores are computed within a date window, so results are only comparable within the same window and release version.","sourceUrl":"https://livecodebench.github.io/"},{"text":"Artificial Analysis removed LiveCodeBench from its Intelligence Index in v4.0 (January 2026), without stating a reason, and has retired it from active reporting.","sourceUrl":"https://artificialanalysis.ai/methodology/intelligence-benchmarking"},{"text":"On Artificial Analysis’s LiveCodeBench leaderboard, which it now lists as a legacy evaluation, the three highest scores are 89.6–91.7%.","sourceUrl":"https://artificialanalysis.ai/evaluations/livecodebench"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[{"id":"qwen3-8-27b-livecodebench-v6","benchmark":"livecodebench","benchmarkVariant":"v6","model":"qwen3-8-27b","modelDetail":null,"value":90.3,"unit":"%","settings":"conditions not stated","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-27B","sourceTitle":null,"evidence":"LiveCodeBench v6 | 90.3 | 83.9 | 89.6 | -- | 88.8 (first col = Qwen3.8-27B)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemma-4-31b-livecodebench-v6","benchmark":"livecodebench","benchmarkVariant":"v6","model":"gemma-4-31b","modelDetail":null,"value":80,"unit":"%","settings":"instruction-tuned model; no other conditions stated","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://huggingface.co/google/gemma-4-31B-it","sourceTitle":null,"evidence":"| LiveCodeBench v6 | 80.0% |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"llama-4-maverick-livecodebench-problems-2024-10-01-to-2025-02-01","benchmark":"livecodebench","benchmarkVariant":"Problems 2024-10-01 to 2025-02-01","model":"llama-4-maverick","modelDetail":null,"value":43.4,"unit":"%","settings":"0-shot; problems dated 10/01/2024-02/01/2025; instruction-tuned model","ci":null,"sourceType":"vendor-reported","evaluator":"Meta","sourceUrl":"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct","sourceTitle":null,"evidence":"LiveCodeBench (10/01/2024-02/01/2025) 0 pass@1 33.3 27.7 32.8 43.4","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]},{"slug":"math","url":"https://harban-ai-labs.vercel.app/benchmarks/math","name":"MATH (Measuring Mathematical Problem Solving With the MATH Dataset)","shortName":"MATH","category":"math","year":2021,"paperUrl":"https://arxiv.org/abs/2103.03874","siteUrl":"https://github.com/hendrycks/math","measures":"Competition-style math problem solving with exact final answers across 7 subjects and 5 difficulty levels (12,500 problems).","metric":"Exact-match accuracy (%) of final answer","higherIsBetter":true,"health":{"status":"flawed","note":"Functionalised variants of MATH problems showed large accuracy gaps versus the static set under simple few-shot prompting, suggesting memorisation.","asOf":"2026-09-28","sourceUrl":"https://arxiv.org/abs/2402.19450"},"caveats":[{"text":"Static-set inflation: functionalised variants of MATH problems showed \"reasoning gaps\" of 58.35%–80.31% versus static scores, suggesting memorisation. The authors measured these gaps with a simple few-shot prompting strategy and expect them to be smaller with more sophisticated prompting.","sourceUrl":"https://arxiv.org/abs/2402.19450"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"mmlu","url":"https://harban-ai-labs.vercel.app/benchmarks/mmlu","name":"Measuring Massive Multitask Language Understanding (MMLU)","shortName":"MMLU","category":"knowledge","year":2020,"paperUrl":"https://arxiv.org/abs/2009.03300","siteUrl":"https://github.com/hendrycks/test","measures":"Broad academic and professional knowledge across 57 subjects via four-option multiple-choice questions.","metric":"Accuracy (%)","higherIsBetter":true,"health":{"status":"flawed","note":"An audit estimated that 6.49% of MMLU questions contain errors, concentrated in some subjects.","asOf":"2026-09-28","sourceUrl":"https://arxiv.org/abs/2406.04127"},"caveats":[{"text":"Label errors: an audit estimated that 6.49% of MMLU questions contain errors, with some subjects far worse (57% of analysed Virology questions).","sourceUrl":"https://arxiv.org/abs/2406.04127"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"mmlu-pro","url":"https://harban-ai-labs.vercel.app/benchmarks/mmlu-pro","name":"MMLU-Pro","shortName":"MMLU-Pro","category":"knowledge","year":2024,"paperUrl":"https://arxiv.org/abs/2406.01574","siteUrl":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","measures":"Reasoning-heavy knowledge questions across 14 disciplines with ten answer options per question (~12K items).","metric":"Accuracy (%)","higherIsBetter":true,"health":{"status":"saturated","note":"Top models cluster: the three highest scores on Artificial Analysis’s MMLU-Pro leaderboard are 89.5–89.8%, and Artificial Analysis now lists it as a legacy evaluation.","asOf":"2026-09-28","sourceUrl":"https://artificialanalysis.ai/evaluations/mmlu-pro"},"caveats":[{"text":"Artificial Analysis removed MMLU-Pro from its Intelligence Index in v4.0 (January 2026), without stating a reason, and has retired it from active reporting.","sourceUrl":"https://artificialanalysis.ai/methodology/intelligence-benchmarking"},{"text":"Dataset revisions matter: the maintainers corrected 15 answers in the medical domain in April 2025.","sourceUrl":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[{"id":"gemma-4-31b-mmlu-pro-main","benchmark":"mmlu-pro","benchmarkVariant":null,"model":"gemma-4-31b","modelDetail":null,"value":85.2,"unit":"%","settings":"instruction-tuned model; no other conditions stated","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://huggingface.co/google/gemma-4-31B-it","sourceTitle":null,"evidence":"| MMLU Pro | 85.2% |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"llama-4-maverick-mmlu-pro-main","benchmark":"mmlu-pro","benchmarkVariant":null,"model":"llama-4-maverick","modelDetail":null,"value":80.5,"unit":"%","settings":"0-shot; instruction-tuned model","ci":null,"sourceType":"vendor-reported","evaluator":"Meta","sourceUrl":"https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct","sourceTitle":null,"evidence":"MMLU Pro 0 macro_avg/acc 68.9 73.4 74.3 80.5","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]},{"slug":"swe-bench","url":"https://harban-ai-labs.vercel.app/benchmarks/swe-bench","name":"SWE-bench","shortName":"SWE-bench","category":"coding","year":2023,"paperUrl":"https://arxiv.org/abs/2310.06770","siteUrl":"https://www.swebench.com/","measures":"Whether a system can resolve real GitHub issues in 12 Python repositories by producing a patch that passes the repository's tests (2,294 tasks).","metric":"% resolved","higherIsBetter":true,"health":{"status":"flawed","note":"SWE-Bench+ found solution leakage in issue text and weak tests behind many \"successful\" patches.","asOf":"2026-09-28","sourceUrl":"https://arxiv.org/abs/2410.06992"},"caveats":[{"text":"SWE-Bench+ found 32.67% of successful patches had solutions leaked in the issue text and 31.08% passed due to weak tests; >94% of issues predate model cutoffs.","sourceUrl":"https://arxiv.org/abs/2410.06992"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"swe-bench-verified","url":"https://harban-ai-labs.vercel.app/benchmarks/swe-bench-verified","name":"SWE-bench Verified","shortName":"SWE-bench Verified","category":"coding","year":2024,"paperUrl":null,"siteUrl":"https://www.swebench.com/verified.html","measures":"A 500-task human-screened subset of SWE-bench (built with OpenAI) where issues are judged clear and tests valid.","metric":"% resolved (of 500)","higherIsBetter":true,"health":{"status":"flawed","note":"Epoch AI’s benchmark review gives SWE-bench Verified a \"Flawed\" verdict (September 2026).","asOf":"2026-09-28","sourceUrl":"https://epoch.ai/benchmarks/swe-bench-verified/review"},"caveats":[{"text":"Epoch AI’s review verdict is \"Flawed\" (reviewed 3 Sep 2026). It summarises an OpenAI audit (Feb 2026) that examined 27.6% of tasks and found 59.4% of those had flawed test cases.","sourceUrl":"https://epoch.ai/benchmarks/swe-bench-verified/review"},{"text":"Since 18 Nov 2025, the maintainers accept new Verified submissions only from academic teams and research institutions that publish their methods.","sourceUrl":"https://github.com/SWE-bench/experiments"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[{"id":"gemini-3-1-pro-swe-bench-verified-main","benchmark":"swe-bench-verified","benchmarkVariant":null,"model":"gemini-3-1-pro","modelDetail":"Thinking (High)","value":80.6,"unit":"%","settings":"Single attempt; methodology PDF says averaged over 10 runs with a bash + file-edit + submit-tool scaffold, and that the score includes a +0.6-point adjustment after Google fixed 3 official-harness items (astropy-7606, sphinx-8595, sphinx-9711) the model passed internally","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","sourceTitle":null,"evidence":"SWE-Bench Verified Agentic coding Single attempt 80.6%","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"mistral-medium-3-5-swe-bench-verified-main","benchmark":"swe-bench-verified","benchmarkVariant":null,"model":"mistral-medium-3-5","modelDetail":null,"value":77.6,"unit":"%","settings":"conditions not stated","ci":null,"sourceType":"vendor-reported","evaluator":"Mistral AI","sourceUrl":"https://huggingface.co/mistralai/Mistral-Medium-3.5-128B","sourceTitle":null,"evidence":"It scores **91.4%** on τ³-Telecom and **77.6%** on SWE-Bench Verified.","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"muse-glimmer-30b-swe-bench-verified-main","benchmark":"swe-bench-verified","benchmarkVariant":null,"model":"muse-glimmer-30b","modelDetail":"High Reasoning","value":76,"unit":"%","settings":"High reasoning; scaffold with a bash tool and a file-operation tool; average pass rate over 500 tasks, averaged across 4 runs (Meta methodology report).","ci":null,"sourceType":"vendor-reported","evaluator":"Meta","sourceUrl":"https://huggingface.co/meta-models/Muse-Glimmer-30B","sourceTitle":null,"evidence":"| SWE-Bench Verified | 76.0 | 66.6 | **77.2** |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gpt-oss-120b-swe-bench-verified-main","benchmark":"swe-bench-verified","benchmarkVariant":null,"model":"gpt-oss-120b","modelDetail":"reasoning level high","value":62.4,"unit":"%","settings":"reasoning level high (Table 3)","ci":null,"sourceType":"vendor-reported","evaluator":"OpenAI","sourceUrl":"https://cdn.openai.com/pdf/419b6906-9da6-406c-a19d-1bb078ac7637/oai_gpt-oss_model_card.pdf","sourceTitle":null,"evidence":"SWE-Bench Verified 47.9 52.6 62.4","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]},{"slug":"tau-bench","url":"https://harban-ai-labs.vercel.app/benchmarks/tau-bench","name":"τ-bench (Tool-Agent-User Interaction benchmark)","shortName":"τ-bench","category":"agents","year":2024,"paperUrl":"https://arxiv.org/abs/2406.12045","siteUrl":"https://github.com/sierra-research/tau-bench","measures":"Whether a tool-using agent completes customer-service tasks (retail, airline) while following domain policy in conversation with an LLM-simulated user.","metric":"pass^k (pass^1 = average task success; higher k = reliability across k trials)","higherIsBetter":true,"health":{"status":"flawed","note":"An audit found τ-bench can count empty responses as successful; the maintainers’ README says the original tasks are outdated and points to the successor τ³-bench.","asOf":"2026-09-28","sourceUrl":"https://arxiv.org/abs/2507.02825"},"caveats":[{"text":"Grading flaw: an audit of agentic benchmarks found τ-bench counts empty responses as successful.","sourceUrl":"https://arxiv.org/abs/2507.02825"},{"text":"Superseded: the maintainers describe the original tasks as outdated and recommend τ³-bench.","sourceUrl":"https://github.com/sierra-research/tau-bench"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[]},{"slug":"terminal-bench","url":"https://harban-ai-labs.vercel.app/benchmarks/terminal-bench","name":"Terminal-Bench","shortName":"Terminal-Bench","category":"agents","year":2025,"paperUrl":"https://arxiv.org/abs/2601.11868","siteUrl":"https://www.tbench.ai/","measures":"Whether an agent can complete realistic, test-verified tasks in a sandboxed command-line environment.","metric":"Task resolution rate (%) with 95% CI, per benchmark version","higherIsBetter":true,"health":{"status":"active","note":"Actively maintained: version 4.0 (dated 28 Aug 2026 on the Terminal-Bench blog) removed eight tasks — two of them for saturation — and fixed 19 others. Compare scores only within one version.","asOf":"2026-09-28","sourceUrl":"https://www.tbench.ai/news/terminal-bench-4-0"},"caveats":[{"text":"Version-sensitive: 2.1 fixed issues in 28 of the 89 tasks in 2.0, and 4.0 introduced further breaking changes; scores are not comparable across versions.","sourceUrl":"https://www.tbench.ai/news/terminal-bench-2-1"},{"text":"Harness matters: the same model can score very differently depending on the agent harness used.","sourceUrl":"https://www.tbench.ai/news/terminal-bench-2-1"},{"text":"Version 1.0 included unstable tasks (for example download-youtube), which 2.0 addressed.","sourceUrl":"https://www.tbench.ai/news/announcement-2-0"}],"referenceVerified":true,"verifiedOn":"2026-09-28","results":[{"id":"deepseek-v4-1-flash-terminal-bench-2-1","benchmark":"terminal-bench","benchmarkVariant":"2.1","model":"deepseek-v4-1-flash","modelDetail":"max reasoning effort","value":90.6,"unit":"%","settings":"Pass@1; max reasoning effort; Minimal mode of DeepSeek Harness, 1M-token context window","ci":null,"sourceType":"vendor-reported","evaluator":"DeepSeek","sourceUrl":"https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash","sourceTitle":null,"evidence":"Terminal-Bench 2.1 (Pass@1) | 89.1 | 88.8 | 88.3 | 88.2 | 87.9 | 82.7 | **90.6**","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"qwen3-8-max-terminal-bench-2-1","benchmark":"terminal-bench","benchmarkVariant":"2.1","model":"qwen3-8-max","modelDetail":null,"value":86.6,"unit":"%","settings":"Evaluated with Claude Code (avg@10), 5-hour timeout, max_tokens=131,072 (card footnote 2)","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B","sourceTitle":null,"evidence":"Terminal Bench 2.1 | 84.6 | 84.6 | 88.8 | 74.5 | 86.6 (last col = Qwen3.8-Max)","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"qwen3-8-27b-terminal-bench-2-1","benchmark":"terminal-bench","benchmarkVariant":"2.1","model":"qwen3-8-27b","modelDetail":null,"value":73,"unit":"%","settings":"Terminus harness (row label 'Terminal Bench 2.1 (Terminus)')","ci":null,"sourceType":"vendor-reported","evaluator":"Alibaba Cloud (Qwen)","sourceUrl":"https://huggingface.co/Qwen/Qwen3.8-27B","sourceTitle":null,"evidence":"Terminal Bench 2.1 (Terminus) | 73.0 | 63.4 | 64.0 | 51.7 | 78.2","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"gemini-3-1-pro-terminal-bench-2-0","benchmark":"terminal-bench","benchmarkVariant":"2.0","model":"gemini-3-1-pro","modelDetail":"Thinking (High)","value":68.5,"unit":"%","settings":"Terminus-2 harness; self-computed","ci":null,"sourceType":"vendor-reported","evaluator":"Google DeepMind","sourceUrl":"https://deepmind.google/models/model-cards/gemini-3-1-pro/","sourceTitle":null,"evidence":"Terminal-Bench 2.0 Agentic terminal coding Terminus-2 harness 68.5%","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"claude-opus-5-5-terminal-bench-4-0","benchmark":"terminal-bench","benchmarkVariant":"4.0","model":"claude-opus-5-5","modelDetail":"xhigh effort","value":66.4,"unit":"%","settings":"xhigh effort (per page footnote); stated standard error ±2.6 pts. Anthropic evaluated Opus 5.5 with production safeguards enabled; when they intervened, biology and frontier LLM development tasks were completed by Claude Opus 5 and cybersecurity tasks by Claude Opus 4.8 (page note).","ci":null,"sourceType":"vendor-reported","evaluator":"Anthropic","sourceUrl":"https://www.anthropic.com/claude-opus-5-5","sourceTitle":null,"evidence":"Agentic coding Terminal-Bench 4.0¹ 66.4%","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"muse-glimmer-30b-terminal-bench-2-1","benchmark":"terminal-bench","benchmarkVariant":"2.1","model":"muse-glimmer-30b","modelDetail":"High Reasoning","value":51.7,"unit":"%","settings":"with terminus2; High Reasoning. Meta’s linked methodology report states Terminal-Bench 2.1 results are sourced from Artificial Analysis (Terminus 2 harness in an E2B sandbox; 89 tasks).","ci":null,"sourceType":"third-party","evaluator":"Artificial Analysis (republished by Meta)","sourceUrl":"https://huggingface.co/meta-models/Muse-Glimmer-30B","sourceTitle":"Muse Glimmer model card (Meta, Hugging Face)","evidence":"| TerminalBench 2.1 (with terminus2) | 51.7 | 43.4 | **60.7** |","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false},{"id":"grok-4-7-terminal-bench-4-0","benchmark":"terminal-bench","benchmarkVariant":"4.0","model":"grok-4-7","modelDetail":"xHigh","value":38,"unit":"%","settings":"xhigh reasoning; Grok Build harness (model card §2.3). xAI’s announcement page reports 37.6% for the same model — the two official sources disagree.","ci":null,"sourceType":"vendor-reported","evaluator":"xAI","sourceUrl":"https://media.x.ai/v1/website/4p7card-5eccc980.pdf","sourceTitle":"Model Card: Grok 4.7 (xAI)","evidence":"Grok 4.7 scores 38.0% at xhigh, evaluated with the Grok Build harness.","publishedDate":null,"retrievedOn":"2026-09-28","logsUrl":null,"preliminary":false}]}]}