{"name":"OpenNash AI Evaluation Benchmark Atlas","version":"2026.07.16","datePublished":"2026-06-07","dateModified":"2026-07-16","lastVerified":"2026-07-15","updateCadence":"The catalog is reviewed monthly, the official score snapshot is refreshed at least monthly, and source links are checked weekly.","editor":{"name":"OpenNash Research","url":"https://opennash.com/"},"licenseNote":"Benchmark names and descriptions are provided for reference. Each linked benchmark retains its own license and terms; verify them at the source before use.","modelHighlights":[{"title":"Intelligence","subtitle":"Artificial Analysis Intelligence Index · higher is better","rows":[["Claude Opus 4.8 (max)",61],["GPT-5.5 (xhigh)",60],["Gemini 3.1 Pro Preview",57],["MiniMax-M3",55],["Kimi K2.6",54],["MiMo-V2.5-Pro",54],["Grok 4.3 (high)",53],["Muse Spark",52],["DeepSeek V4 Pro (Max)",52],["Nemotron 3 Ultra",48],["gpt-oss-120b (high)",33]]},{"title":"Speed","subtitle":"Output tokens per second · higher is better","rows":[["gpt-oss-120b (high)",336],["Grok 4.3 (high)",198],["Nemotron 3 Ultra",143],["Gemini 3.1 Pro Preview",139],["GPT-5.5 (xhigh)",63],["Claude Opus 4.8 (max)",61],["DeepSeek V4 Pro (Max)",53],["Kimi K2.6",44],["MiMo-V2.5-Pro",43],["MiniMax-M3",40]]},{"title":"Price","subtitle":"USD per 1M tokens blended · lower is better","rows":[["DeepSeek V4 Pro (Max)",0.2],["MiMo-V2.5-Pro",0.2],["gpt-oss-120b (high)",0.2],["MiniMax-M3",0.2],["Nemotron 3 Ultra",0.5],["Grok 4.3 (high)",0.6],["Kimi K2.6",0.7],["Gemini 3.1 Pro Preview",1.7],["Claude Opus 4.8 (max)",4.1],["GPT-5.5 (xhigh)",4.3]]}],"evalRecipes":[{"domain":"CX","title":"CX agent eval stack","copy":"For support agents that must follow policy, use tools, resolve customer state, and hand off cleanly.","steps":[["Model shortlist","Artificial Analysis","Shortlist models by quality, coding/tool aptitude, speed, latency, and cost."],["Domain benchmark","tau-bench / tau2 / tau-voice","Test policy following, tool use, user simulation, voice readiness, and final state changes."],["Workflow layer","CRMArena / WorkArena","Exercise CRM and enterprise service workflows where process state matters."],["Private gate","Top support traces","Turn refunds, cancellations, escalations, missing data, and policy conflicts into pass/fail tests."],["Online monitor","Handoffs + outcomes","Track resolution, handoff timing, context transfer, retries, cost, latency, and complaints."]]},{"domain":"Legal","title":"Legal agent eval stack","copy":"For research, drafting, diligence, intake, and legal RAG where citations and jurisdiction matter.","steps":[["Model shortlist","Artificial Analysis + LMArena","Screen model quality and cost before running expensive legal tasks."],["Agent benchmark","Harvey LAB","Use long-horizon legal-agent tasks and expert rubrics as the primary legal-agent signal."],["RAG benchmark","LegalBench-RAG","Check retrieval, grounded generation, citation behavior, and legal document use."],["Private gate","Matter-specific traces","Use real intake, diligence, research, and drafting examples with binary rubrics."],["Online monitor","Citation + handoff review","Track unsupported claims, wrong jurisdiction, stale law, escalation quality, and reviewer corrections."]]},{"domain":"Healthcare","title":"Healthcare eval stack","copy":"For clinical conversations, EHR workflows, medical QA, safety, and escalation to licensed professionals.","steps":[["Model shortlist","HealthBench + AA","Combine general model screening with physician-rubric health conversations."],["Domain benchmark","MedHELM / MedQA","Use medical QA and holistic medical task coverage for baseline knowledge."],["Workflow layer","MedAgentBench","Test EHR and FHIR-like workflows with tools, records, orders, and clinical state."],["Private gate","Clinical scenario set","Create pass/fail cases for uncertainty, escalation, PHI handling, dosage, and contraindications."],["Online monitor","Safety + escalation","Watch unsafe advice, missing escalation, overconfidence, wait time, and clinician corrections."]]},{"domain":"Coding","title":"Coding agent eval stack","copy":"For IDE agents, repo repair, shell execution, and long-running development workflows.","steps":[["Model shortlist","AA coding + speed","Screen coding quality, latency, and cost before running repo-scale tasks."],["Repo benchmark","SWE-bench Verified / Pro","Use real GitHub issue repair and tests for software engineering work."],["Terminal layer","Terminal-Bench","Exercise shell-native task completion, environment use, and command-line reliability."],["Private gate","Your repo tasks","Save migrations, bug fixes, build failures, and flaky tool cases as executable tests."],["Online monitor","PR and CI outcomes","Track test pass rate, review edits, build failures, cycle time, and reverted changes."]]},{"domain":"RAG","title":"RAG eval stack","copy":"For document QA, knowledge bases, filings, policy retrieval, and long-context answer systems.","steps":[["Retriever screen","MTEB / BEIR","Pick embedding and retrieval candidates with standard search metrics."],["RAG benchmark","RAGBench / RGB / KILT","Stress grounded QA, context relevance, faithfulness, and answer quality."],["Document layer","LongBench / OmniDocBench","Check long context, OCR, layout, tables, formulas, and parsing quality."],["Private gate","Query-document pairs","Build Recall@k, Precision@k, MRR, faithfulness, citation, and refusal checks."],["Online monitor","Grounding drift","Track no-answer cases, stale docs, broken citations, missed retrieval, and user corrections."]]},{"domain":"Agents","title":"Agent and tool-use eval stack","copy":"For browser, API, MCP, computer-use, and multi-step agents that act in external systems.","steps":[["Model shortlist","AA value + BFCL","Screen model value and function-calling reliability before workflow tests."],["Tool benchmark","BFCL / ToolSandbox / MCP-Atlas","Test tool selection, argument extraction, multi-turn calls, and stateful execution."],["World benchmark","WebArena / OSWorld / WorkArena","Exercise browser, desktop, and enterprise workflow environments."],["Private gate","End-to-end task set","Define exact task success, allowed actions, checkpoints, and state changes."],["Online monitor","Trace transition matrix","Track first upstream failure by workflow step: plan, retrieve, act, recover, hand off, complete."]]}],"domainStacks":[{"domain":"CX","title":"Customer support and voice agents","copy":"Highest ROI for support builders: use live model quality/value first, then benchmark policy-following, tool use, customer state changes, CRM workflows, and knowledge retrieval.","picks":[["Live model board","Artificial Analysis","Quality, speed, coding, and price shortlist before expensive support simulations.","https://artificialanalysis.ai/leaderboards/models"],["Support benchmark","tau-bench","Retail and airline customer-service agents with policies, tools, users, and state checks.","https://github.com/sierra-research/tau-bench"],["Support benchmark","tau2 / tau3-bench","Expanded tau domains, task fixes, knowledge, and voice-ready customer-service evaluation.","https://github.com/sierra-research/tau2-bench"],["Knowledge support","tau-knowledge","Support agents that must retrieve and use unstructured policy knowledge.","https://taubench.com/blog/tau-knowledge.html"],["CRM workflow","CRMArena","CRM-style service-agent, analyst, and operations workflows.","https://arxiv.org/abs/2411.02305"],["Enterprise workflow","WorkArena / WorkArena++","ServiceNow-style enterprise workflows for business-process agents.","https://github.com/ServiceNow/WorkArena"],["Tool reliability","BFCL / ToolSandbox","Function calling, multi-turn tool use, and stateful tool execution.","https://gorilla.cs.berkeley.edu/leaderboard"]]},{"domain":"Voice","title":"Voice, speech, and realtime agents","copy":"Voice benchmarks should separate speech understanding, latency, interruption behavior, tool calling, safety, and end-state task success.","picks":[["Voice CX","tau-voice","Full-duplex customer-service voice agents scored against final task/database state.","https://sierra.ai/blog/tau-voice-benchmarking-real-time-voice-agents-on-real-world-tasks"],["Voice agents","VoiceAgentBench","Speech-based agentic tasks with tool/function specs, multi-turn spoken queries, and safety cases.","https://huggingface.co/datasets/krutrim-ai-labs/VoiceAgentBench"],["Voice assistants","VoiceBench","LLM-based voice assistants across QA, reasoning, instruction following, safety, and robustness.","https://github.com/MatthewCYM/VoiceBench"],["Vocal interaction","VocalBench","Speech-interaction models across vocal communication and multi-round tasks.","https://github.com/SJTU-OmniAgent/VocalBench"],["Emerging","EVA-Bench / SOVA-Bench","Newer end-to-end and conversational speech benchmarks; verify maturity before relying on them.","https://arxiv.org/abs/2605.13841"],["Clinical voice","VoxClinBench","Cross-lingual clinical-voice benchmark for medical speech settings.","https://huggingface.co/datasets/voice-bench-submission/voxclinbench"]]},{"domain":"Legal","title":"Legal research, drafting, and retrieval","copy":"Do not rely on general MMLU-style scores for legal work. Pair legal reasoning benchmarks with legal RAG, jurisdiction-specific task suites, and citation-grounding checks.","picks":[["Legal agents","Harvey Legal Agent Benchmark","Long-horizon legal-agent tasks across practice areas with expert rubrics and an open harness.","https://github.com/harveyai/harvey-labs"],["Legal reasoning","LegalBench","Broad legal reasoning tasks from the legal benchmark community.","https://github.com/HazyResearch/legalbench"],["Legal RAG","LegalBench-RAG","Grounded legal retrieval and generation tests.","https://github.com/zeroentropy-ai/legalbenchrag"],["Legal NLP","LexGLUE","Classic legal NLP classification and understanding tasks.","https://github.com/coastalcph/lex-glue"],["Legal suite","LawBench","Chinese legal task suite across consultation, reading comprehension, and legal reasoning.","https://github.com/open-compass/LawBench"],["Hosted legal","Vals AI LegalBench","Commercial legal-model comparison signal; inspect methodology before treating as ground truth.","https://www.vals.ai/benchmarks/legal_bench"],["General model screen","Artificial Analysis value board","Use quality, speed, and cost before paying for legal-specific runs.","https://artificialanalysis.ai/leaderboards/models"]]},{"domain":"Finance","title":"Finance, filings, and analyst workflows","copy":"Finance evals need evidence, arithmetic, and source discipline. Start with filings QA, then add numeric reasoning and workflow traces.","picks":[["Filings QA","FinanceBench","Open-book financial QA over public company filings with evidence strings.","https://github.com/patronus-ai/financebench"],["Finance agents","FinanceAgent / FAB v2","Hosted financial-agent benchmark for realistic analyst and workflow tasks.","https://www.vals.ai/benchmarks/fabv2"],["Financial reasoning","FinQA / ConvFinQA","Numerical reasoning over financial reports and conversational finance QA.","https://finqasite.github.io/index.html"],["Finance suite","FinBen / PIXIU","Broad financial benchmark suite across extraction, QA, generation, forecasting, and decisions.","https://github.com/The-FinAI/PIXIU"],["Quant agents","QFBench","Quantitative-finance agent tasks involving code, markets, and stateful reasoning.","https://github.com/QF-Bench/QuantitativeFinance-Bench"],["Finance scoreboard","Open FinLLM Leaderboard","Financial LLM comparison signal for open and finance-tuned models.","https://huggingface.co/spaces/TheFinAI/Open-Financial-LLM-Leaderboard"],["Economic tasks","GDPval","Economically valuable professional tasks across finance, insurance, and operations.","https://openai.com/index/gdpval/"],["Document RAG","RAGBench / LongBench","Check retrieval, citations, and long-document stability for filings-heavy systems.","https://huggingface.co/datasets/rungalileo/ragbench"]]},{"domain":"Coding","title":"Coding agents and software engineering","copy":"For coding agents, toy function benchmarks are not enough. Use real repos, terminal execution, and task completion.","picks":[["Live coding rank","Artificial Analysis coding + value","Screen coding quality, throughput, and price together.","https://artificialanalysis.ai/leaderboards/models"],["SWE agents","SWE-bench / Verified / Pro","Real GitHub issues, repository edits, and test-passing outcomes.","https://www.swebench.com/"],["Terminal agents","Terminal-Bench 2.0","Command-line task execution for shell-native coding agents.","https://www.tbench.ai/leaderboard/terminal-bench/2.0"],["Fresh coding","LiveCodeBench","Recent contest-style coding with contamination-aware releases.","https://github.com/LiveCodeBench/LiveCodeBench"],["Practical editing","Aider Polyglot","Multi-language code editing and test-passing benchmark.","https://aider.chat/docs/leaderboards/"],["Modern code gen","BigCodeBench","Software-engineering-oriented code-generation tasks beyond HumanEval/MBPP.","https://github.com/bigcode-project/bigcodebench"],["ML agents","MLE-bench / ML-Bench","Machine-learning and repo-scale coding tasks for executable agents.","https://github.com/openai/mle-bench"],["Legacy baselines","HumanEval / MBPP","Classic baselines for regression checks, not frontier differentiation.","https://github.com/openai/human-eval"]]},{"domain":"Science","title":"Science, biology, and research agents","copy":"Scientific agents need expert reasoning plus tool/code grounding. Use science QA only as a screen, not as proof of research usefulness.","picks":[["Expert science","GPQA / GPQA Diamond","Graduate-level expert science questions designed to be hard for non-experts.","https://github.com/idavidrein/gpqa"],["Frontier academic","Humanity's Last Exam","Expert-level multimodal academic questions for frontier model differentiation.","https://lastexam.ai/"],["Biology research","LAB-Bench / LABBench2","Biology tasks across literature, protocols, databases, sequences, and lab reasoning.","https://huggingface.co/datasets/futurehouse/labbench2"],["Scientific coding","SciCode","Scientist-curated coding problems from natural-science contexts.","https://github.com/scicode-bench/SciCode"],["Scientific agents","AstaBench","Scientific research-agent suite covering literature, code execution, data analysis, and end-to-end workflows.","https://github.com/allenai/asta-bench"],["Research replication","PaperBench","Agent benchmark for replicating AI research papers and artifacts.","https://github.com/openai/frontier-evals/tree/main/project/paperbench"],["ML research","MLE-bench","Kaggle-style ML engineering tasks and research execution.","https://github.com/openai/mle-bench"],["Physics niche","CritPt","Research-level physics tasks for niche frontier reasoning.","https://github.com/CritPt-Benchmark/CritPt"],["Anti-gaming","WeirdML / Pencil Puzzle Bench","Unusual ML and deterministic puzzle tasks for reasoning stress tests.","https://www.haavardihlen.no/weirdml/"]]},{"domain":"Healthcare","title":"Healthcare, medicine, and clinical agents","copy":"Healthcare needs separate signals for medical conversation quality, clinical QA, EHR workflow agents, biomedical literature, medical calculations, and PHI-safe extraction.","picks":[["Health conversations","HealthBench","Physician-rubric health conversations for accuracy, relevance, safety, and uncertainty.","https://openai.com/index/healthbench/"],["Clinical workflows","HealthBench Professional","Clinician-workflow-oriented health benchmark variant.","https://arxiv.org/abs/2604.27470"],["Medical harness","MedHELM","HELM-style medical evaluation across clinical task categories and benchmarks.","https://crfm.stanford.edu/helm/medhelm/v2.0.0/"],["Clinical agents","MedAgentBench","Virtual EHR/FHIR environment with clinically relevant agent tasks.","https://github.com/stanfordmlgroup/MedAgentBench"],["Medical QA","MedQA / MedMCQA / PubMedQA","Exam-style and biomedical literature QA baselines.","https://github.com/jind11/MedQA"],["Medical suite","MultiMedQA","Composite medical QA benchmark used in Med-PaLM research.","https://pubmed.ncbi.nlm.nih.gov/37438534/"],["Clinical decisions","ClinicBench / CliBench","Open-ended clinical decision, diagnosis, procedures, orders, and prescriptions.","https://github.com/AI-in-Health/ClinicBench"],["Medical calculations","MedCalc-Bench","Clinical calculator and medical arithmetic tasks.","https://github.com/ncbi-nlp/MedCalc-Bench"],["Medical safety","MedSafetyBench","Safety benchmark for medical LLM responses and clinical-risk behaviors.","https://github.com/AI4LIFE-GROUP/med-safety-bench"],["OpenMed resources","OpenMed","Open-source healthcare NLP toolkit, model hub, and curated datasets; useful source layer, not a leaderboard replacement.","https://github.com/maziyarpanahi/openmed"]]},{"domain":"RAG","title":"RAG, retrieval, and document QA","copy":"RAG quality is a system property. Evaluate retriever choice, grounding, citation behavior, long context, and final answer quality separately.","picks":[["Embedding benchmark","MTEB / BEIR","Pick retrieval and embedding models before blaming the generator.","https://huggingface.co/mteb"],["RAG benchmark","CRAG / RAGBench","Test factual QA, retrieval use, attribution, and grounded generation.","https://github.com/facebookresearch/CRAG"],["Document benchmark","DocVQA / LongBench","Stress PDFs, document images, and long-context reasoning.","https://www.docvqa.org/"],["Long context","LongBench / InfiniteBench","Long-context understanding and very-long-window stress tests.","https://github.com/THUDM/LongBench"],["Legal RAG","LegalBench-RAG","Domain-specific legal retrieval and grounded generation.","https://github.com/zeroentropy-ai/legalbenchrag"],["Finance RAG","FinanceBench","Open-book filings QA with evidence strings.","https://github.com/patronus-ai/financebench"],["Document parsing","OmniDocBench","Document parsing, OCR, layout, formula, and table extraction for RAG pipelines.","https://github.com/opendatalab/OmniDocBench"],["Healthcare RAG","PubMedQA / BioASQ","Biomedical literature QA and semantic search challenge signals.","https://pubmedqa.github.io/"]]},{"domain":"Agents","title":"Tool, browser, desktop, and enterprise agents","copy":"Agent benchmarks test the scaffold as much as the model. Use these to compare browsing, computer use, API calls, workflow execution, and task persistence.","picks":[["General agents","GAIA","Real-world assistant tasks requiring reasoning, browsing, multimodality, and tools.","https://huggingface.co/datasets/gaia-benchmark/GAIA"],["MCP tools","MCP-Atlas","Tool-use benchmark over real MCP servers and multi-call tasks.","https://github.com/scaleapi/mcp-atlas"],["Browsing","BrowseComp","Browsing-agent benchmark for hard-to-find web answers.","https://openai.com/index/browsecomp/"],["Browser agents","WebArena / VisualWebArena","Self-hosted websites and visual web tasks for browser automation.","https://webarena.dev/"],["Computer use","OSWorld","Desktop, web, and app tasks in realistic computer environments.","https://os-world.github.io/"],["Enterprise work","WorkArena / TheAgentCompany","Business-process, digital-worker, and simulated coworker tasks.","https://github.com/ServiceNow/WorkArena"],["Office work","OfficeBench","Office documents, email, calendar, and productivity-task automation.","https://github.com/zlwang-cs/OfficeBench"],["Tool use","BFCL / ToolSandbox / ToolBench","Executable function calling, stateful tool use, and API task suites.","https://gorilla.cs.berkeley.edu/leaderboard"],["Legacy agents","AgentBench","Classic multi-environment agent benchmark.","https://github.com/THUDM/AgentBench"]]},{"domain":"Multimodal","title":"Vision, documents, video, and multimodal reasoning","copy":"Use multimodal benchmarks when the image, document layout, chart, or video actually changes the answer.","picks":[["Expert VLM","MMMU / MMMU-Pro","Expert multimodal reasoning where images materially affect answers.","https://github.com/MMMU-Benchmark/MMMU"],["General VLM","MMBench","Broad multimodal model evaluation suite.","https://github.com/open-compass/MMBench"],["Visual math","MathVista","Mathematical reasoning over diagrams and visual inputs.","https://mathvista.github.io/"],["Document vision","DocVQA","Question answering over document images, forms, and scanned pages.","https://www.docvqa.org/"],["Document parsing","OmniDocBench","PDF/document parsing and layout extraction for OCR-heavy workflows.","https://github.com/opendatalab/OmniDocBench"],["Multimodal browsing","MM-BrowseComp","Emerging benchmark for browsing tasks that require multimodal reasoning.","https://arxiv.org/abs/2508.13186"],["Video","Video-MME","Video understanding across temporal and multimodal questions.","https://video-mme.github.io/home_page.html"],["Medical VLM","OmniMedVQA / Med-VLM Bench","Medical image and vision-language evaluation resources.","https://github.com/yezanting/Med-VLM-Bench-Summary"]]},{"domain":"Safety","title":"Security, safety, and hazardous tool use","copy":"Safety benchmarks should be handled carefully: useful for risk discovery, but often hazardous or easy to misinterpret without controls.","picks":[["Cyber risk","CyberSecEval / PurpleLlama","Cybersecurity capability and risk evals for LLMs.","https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks"],["Agent safety","AgentHarm","Harmful multi-step agent tasks under tool-use settings.","https://huggingface.co/datasets/ai-safety-institute/AgentHarm"],["Refusal/red team","HarmBench","Robust refusal and harmful-request evaluation.","https://www.harmbench.org/"],["Security agents","SEC-bench","Software-security tasks including vulnerability discovery and patching.","https://sec-bench.github.io/"],["Smart contracts","EVMbench","Sandboxed smart-contract detect, patch, and exploit tasks.","https://github.com/paradigmxyz/evmbench"],["Medical safety","MedSafetyBench","Medical safety benchmark for risky clinical responses.","https://github.com/AI4LIFE-GROUP/med-safety-bench"],["Computer-use safety","OS-Harm","Safety benchmark for computer-use agents and harmful action sequences.","https://github.com/tml-epfl/os-harm"],["Static safety","SafetyBench","Broad static safety MCQ baseline.","https://github.com/thu-coai/SafetyBench"]]}],"categories":[{"name":"General Models","slug":"general-models","short":"Broad model quality, reasoning, factuality, instruction following, and meta-evaluation.","title":"General AI Model Benchmarks","intro":"Use these benchmarks to build an initial model shortlist, inspect broad reasoning and factuality, and understand how aggregate leaderboards are assembled. They are scouting tools—not substitutes for task-specific evaluation.","question":"Which benchmarks are useful for comparing general-purpose AI models?","guidance":"Start with one transparent aggregate or harness, then inspect the component benchmarks behind any composite score. Prefer fresh, contamination-aware sets and confirm that the tested model snapshot and inference settings match what you can actually deploy.","url":"/ai-evals/general-models/","count":13},{"name":"Agents & Tool Use","slug":"agents-tool-use","short":"Tool calling, browsers, computer use, office work, and long-horizon agent behavior.","title":"AI Agent and Tool-Use Benchmarks","intro":"Agent benchmarks test more than a base model: the scaffold, tools, environment, retry policy, and evaluator all affect the result. Use this collection to find realistic task environments and harnesses for end-to-end agent behavior.","question":"How should teams choose an AI agent benchmark?","guidance":"Match the environment to the unit of work. Prefer benchmarks with inspectable task states, deterministic end conditions, and runnable harnesses. Treat results from different scaffolds as separate system measurements, not direct model comparisons.","url":"/ai-evals/agents-tool-use/","count":30},{"name":"Coding & SWE","slug":"coding-swe","short":"Repository repair, shell tasks, code generation, debugging, and long-horizon software work.","title":"Coding and Software Engineering Benchmarks","intro":"These benchmarks cover code generation, repository-level issue resolution, terminal work, debugging, and multi-hour software tasks. The strongest choices run generated work against tests in a reproducible environment.","question":"What are the best benchmarks for coding agents?","guidance":"Use repository repair for end-to-end coding agents, terminal benchmarks for shell-native execution, and fresh problem sets for code generation. Record the agent scaffold, tool permissions, time budget, and pass criteria with every result.","url":"/ai-evals/coding-swe/","count":17},{"name":"Retrieval & RAG","slug":"retrieval-rag","short":"Retrieval, embeddings, long context, document AI, grounded generation, and citations.","title":"RAG, Retrieval, and Embedding Benchmarks","intro":"RAG evaluation should separate retrieval quality from answer quality. This collection includes embedding and retrieval suites, grounded generation tests, long-context probes, and document understanding benchmarks.","question":"How should teams evaluate a RAG system?","guidance":"Measure retrieval with labeled query-document pairs and metrics such as Recall@k or MRR. Evaluate generation separately for faithfulness, answer relevance, citation support, refusals, and domain-specific failure modes.","url":"/ai-evals/retrieval-rag/","count":13},{"name":"Customer Support","slug":"customer-support","short":"Support resolution, CRM workflows, handoffs, voice CX, policy, and grounded answers.","title":"Customer Support AI Benchmarks","intro":"Customer-support evals should measure resolved user goals, correct tool and CRM actions, grounded policy answers, and high-quality handoffs—not just conversational fluency.","question":"What should a customer-support agent benchmark measure?","guidance":"Start with end-to-end resolution and final system state. Add checks for policy compliance, tool side effects, knowledge grounding, escalation timing, and whether a human receives enough context to finish the job.","url":"/ai-evals/customer-support/","count":5},{"name":"Legal","slug":"legal","short":"Legal reasoning, research, document review, retrieval, and jurisdiction-sensitive tasks.","title":"Legal AI Benchmarks","intro":"Legal benchmarks can reveal reasoning, retrieval, drafting, and document-review limits, but results are jurisdiction- and task-dependent. Use public sets to shortlist systems, then validate against your own authorities and workflows.","question":"How should legal teams use public AI benchmarks?","guidance":"Check the jurisdiction, task definitions, source dates, and grading method before interpreting a score. Build private tests around citations, deadlines, forms, privilege, abstention, and the specific work product your professionals review.","url":"/ai-evals/legal/","count":11},{"name":"Finance","slug":"finance","short":"Filings, financial reasoning, professional work, extraction, and evidence-backed analysis.","title":"Finance AI Benchmarks","intro":"Finance benchmarks span question answering, calculations, filing analysis, extraction, and professional workflows. Evidence quality and freshness matter as much as final-answer accuracy.","question":"What makes a finance AI benchmark useful?","guidance":"Prefer tasks with traceable source documents, dated data, deterministic calculations, and explicit abstention rules. Re-test with the filings, chart of accounts, policies, and approval boundaries used in your own workflow.","url":"/ai-evals/finance/","count":9},{"name":"Healthcare","slug":"healthcare","short":"Clinical knowledge, biomedical reasoning, safety, medical agents, and decision support.","title":"Healthcare and Biomedical AI Benchmarks","intro":"Healthcare benchmarks cover medical knowledge, clinical reasoning, biomedical research, safety, and agentic workflows. High aggregate scores do not establish clinical readiness.","question":"How should healthcare AI benchmarks be interpreted?","guidance":"Separate factual knowledge from clinical workflow reliability. Validate with qualified domain experts, patient-specific safety constraints, current guidance, abstention behavior, and the exact decisions the system is allowed to influence.","url":"/ai-evals/healthcare/","count":15},{"name":"Science & Reasoning","slug":"science-reasoning","short":"Math, physics, biology, research, scientific coding, and complex verifiable reasoning.","title":"Science, Math, and Reasoning Benchmarks","intro":"These suites probe mathematical reasoning, research-level science, biology, physics, and scientific coding. Many are intentionally frontier or contamination-resistant and may have very low scores.","question":"What do hard science and reasoning benchmarks tell us?","guidance":"Use them to locate capability boundaries and compare reasoning setups, not to claim research autonomy. Inspect whether tools were enabled, answers are machine-verifiable, tasks were private, and repeated runs were used.","url":"/ai-evals/science-reasoning/","count":19},{"name":"Multimodal & Voice","slug":"multimodal-voice","short":"Images, video, audio, speech, multimodal browsing, and voice-agent behavior.","title":"Multimodal and Voice AI Benchmarks","intro":"Multimodal and voice evals add perception, timing, interaction, and media quality to ordinary language-model evaluation. End-to-end latency and recoverability often matter as much as semantic accuracy.","question":"How should multimodal and voice agents be evaluated?","guidance":"Measure perception and task completion separately, then add end-to-end checks for latency, interruptions, turn taking, tool use, channel noise, accessibility, and graceful recovery from misunderstood input.","url":"/ai-evals/multimodal-voice/","count":16},{"name":"Safety & Security","slug":"safety-security","short":"Harm, misuse, cyber capability, policy compliance, and dangerous capability evaluation.","title":"AI Safety and Security Benchmarks","intro":"Safety and security benchmarks probe policy behavior, misuse resistance, harmful capabilities, and cyber tasks. Some sources are intentionally restricted or hazardous and require careful handling.","question":"How should teams use AI safety and security benchmarks?","guidance":"Use least-privilege environments, approved datasets, controlled disclosure, and qualified reviewers. Keep capability evaluation separate from release policy, and never treat a public safety score as evidence that a deployed system is safe in your context.","url":"/ai-evals/safety-security/","count":11}],"benchmarks":[{"slug":"artificial-analysis","name":"Artificial Analysis","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","scoreboard"],"type":"scoreboard","whatItTests":"Model intelligence, coding, math, science, speed, latency, and pricing across commercial and open models.","bestFor":"Model shortlist and cost-performance tradeoffs","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://artificialanalysis.ai/api-reference/","leaderboard":"https://artificialanalysis.ai/leaderboards/models"},"sourceCount":2},{"slug":"benchmark-health-index","name":"Benchmark Health Index","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Benchmark quality auditing","tags":["meta-evaluation","saturation","discrimination","benchmark health"],"type":"toolkit","whatItTests":"Benchmarks themselves across discrimination, anti-saturation, and ecosystem impact.","bestFor":"Choosing which public benchmarks still carry useful signal.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/SKYLENAGE-AI/benchmark-health-index","paper":"https://arxiv.org/abs/2602.11674"},"sourceCount":2},{"slug":"benchmark-squared","name":"Benchmark²","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Benchmark validity meta-evaluation","tags":["meta-evaluation","validity","ranking","discrimination"],"type":"toolkit","whatItTests":"Benchmark quality through cross-ranking consistency, discriminability, and capability alignment.","bestFor":"Auditing whether an evaluation measures the capability its label implies.","runnability":"partial","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2601.03986"},"sourceCount":1},{"slug":"eleutherai-lm-evaluation-harness","name":"EleutherAI LM Evaluation Harness","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","harness"],"type":"harness","whatItTests":"Standardized runner for many classic language-model tasks and benchmark suites.","bestFor":"CI-style baseline model evaluation","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/EleutherAI/lm-evaluation-harness"},"sourceCount":1},{"slug":"helm","name":"HELM","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","harness"],"type":"harness","whatItTests":"Holistic evaluation framework with transparent scenarios, metrics, prompts, and model predictions.","bestFor":"Academic reproducibility and broad model audits","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://crfm.stanford.edu/helm/latest/","repo":"https://github.com/stanford-crfm/helm"},"sourceCount":2},{"slug":"hugging-face-open-llm-leaderboard","name":"Hugging Face Open LLM Leaderboard","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","scoreboard"],"type":"scoreboard","whatItTests":"Open-weight model leaderboard and result archive; useful historically, but check freshness before treating it as current.","bestFor":"Open model comparison and historical baselines","runnability":"partial","maturity":"legacy","access":"mixed","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"leaderboard":"https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard","source":"https://huggingface.co/docs/hub/leaderboard-data-guide"},"sourceCount":2},{"slug":"ifeval","name":"IFEval","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Instruction following","tags":["Instruction following"],"type":"benchmark","whatItTests":"Instruction-following benchmark with verifiable formatting and constraint adherence.","bestFor":"Production assistants with strict output rules","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/google/IFEval"},"sourceCount":1},{"slug":"ifstruct-v1","name":"IFStruct v1.0","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Structured instruction following","tags":["instruction following","JSON","YAML","constraints"],"type":"benchmark","whatItTests":"Whether models satisfy compositional structural constraints in JSON and YAML outputs.","bestFor":"Comparing schema compliance before testing private structured-output contracts.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/LiquidAI/ifstruct-v1.0","results":"https://huggingface.co/api/datasets/LiquidAI/ifstruct-v1.0/leaderboard"},"sourceCount":2},{"slug":"lmarena-chatbot-arena","name":"LMArena / Chatbot Arena","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","scoreboard"],"type":"scoreboard","whatItTests":"Blind pairwise human preference for chat, coding, and style-controlled model comparisons.","bestFor":"Human preference and conversational quality","runnability":"hosted","maturity":"current","access":"hosted","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://lmarena.ai/leaderboard/","dataset":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset"},"sourceCount":2},{"slug":"opencompass-compassrank","name":"OpenCompass / CompassRank","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","harness"],"type":"harness","whatItTests":"Open evaluation platform across many datasets with public and private benchmark dimensions.","bestFor":"Broad open-source evaluation runs","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/open-compass/opencompass","leaderboard":"https://rank.opencompass.org.cn/"},"sourceCount":2},{"slug":"simpleqa","name":"SimpleQA","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Factuality","tags":["Factuality"],"type":"benchmark","whatItTests":"Short factual questions with clear answers for hallucination and factual recall checks.","bestFor":"Factuality and hallucination screening","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/openai/simple-evals","source":"https://openai.com/index/introducing-simpleqa/"},"sourceCount":2},{"slug":"toloka-arena","name":"Toloka Arena","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Meta","tags":["Meta","scoreboard"],"type":"scoreboard","whatItTests":"Hosted agentic-intelligence leaderboard with composite pass rates and enterprise-domain datasets.","bestFor":"Commercial agent benchmark comparison","runnability":"hosted","maturity":"current","access":"commercial","risk":"standard","editorialStatus":"commercial","lastVerified":"2026-07-15","links":{"leaderboard":"https://toloka.ai/arena/"},"sourceCount":1},{"slug":"truthfulqa","name":"TruthfulQA","primaryDomain":"General Models","primaryDomainSlug":"general-models","originalDomain":"Factuality","tags":["Factuality"],"type":"benchmark","whatItTests":"Questions designed to test whether models repeat common falsehoods.","bestFor":"Truthfulness baseline and regression testing","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/sylinrl/TruthfulQA"},"sourceCount":1},{"slug":"ace-benchmark","name":"ACE","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Everyday agent capability","tags":["agents","consumer","tasks","capability"],"type":"benchmark","whatItTests":"Agent capability across consumer task areas such as shopping, food, gaming, and DIY.","bestFor":"Tracking an emerging cross-domain agent benchmark; inspect task coverage before use.","runnability":"partial","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/mercor/ACE","results":"https://huggingface.co/api/datasets/mercor/ACE/leaderboard"},"sourceCount":2},{"slug":"agentbench","name":"AgentBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Agents","tags":["Agents","agents"],"type":"agent benchmark","whatItTests":"LLM agent evaluation across multiple interactive environments.","bestFor":"General agent benchmarking","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/THUDM/AgentBench"},"sourceCount":1},{"slug":"apex-v1-extended","name":"APEX v1 Extended","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Economically valuable professional tasks","tags":["agents","professional work","economy","long horizon"],"type":"benchmark","whatItTests":"Agent performance on extended economically valuable tasks spanning multiple jobs.","bestFor":"Testing whether professional-work rankings survive broader task coverage.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/mercor/APEX-v1-extended","results":"https://huggingface.co/api/datasets/mercor/APEX-v1-extended/leaderboard"},"sourceCount":2},{"slug":"apex-agents","name":"APEX-Agents","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Long-horizon professional work","tags":["agents","professional services","cross-application","long horizon"],"type":"benchmark","whatItTests":"Cross-application completion of long-horizon professional-services tasks.","bestFor":"Shortlisting agent systems for realistic knowledge-work workflows.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/mercor/apex-agents","results":"https://huggingface.co/api/datasets/mercor/apex-agents/leaderboard"},"sourceCount":2},{"slug":"astabench","name":"AstaBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Science / Agents","tags":["Science","Agents","agents"],"type":"agent benchmark","whatItTests":"Scientific research-agent suite covering literature understanding, code execution, data analysis, and end-to-end discovery workflows.","bestFor":"Scientific research agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://allenai.org/asta/bench","repo":"https://github.com/allenai/asta-bench"},"sourceCount":2},{"slug":"berkeley-function-calling-leaderboard-bfcl","name":"Berkeley Function Calling Leaderboard / BFCL","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Tool use","tags":["Tool use"],"type":"benchmark","whatItTests":"Executable function-calling tests including multi-turn and tool-use scenarios.","bestFor":"Tool calling and API accuracy","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"leaderboard":"https://gorilla.cs.berkeley.edu/leaderboard","repo":"https://github.com/ShishirPatil/gorilla/tree/main/berkeley-function-call-leaderboard"},"sourceCount":2},{"slug":"browsecomp","name":"BrowseComp","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Browser agents","tags":["Browser agents"],"type":"benchmark","whatItTests":"Browsing-agent benchmark for hard-to-find web answers that require search and synthesis.","bestFor":"Web browsing agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://openai.com/index/browsecomp/","repo":"https://github.com/openai/simple-evals"},"sourceCount":2},{"slug":"cctu","name":"CCTU","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Tool use under complex constraints","tags":["agents","tools","constraints","validation"],"type":"benchmark","whatItTests":"Tool use across 200 cases containing resource, behavior, toolset, and response constraints.","bestFor":"Finding constraint violations that task-success-only agent evals miss.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/Junjie-Ye/CCTU","paper":"https://arxiv.org/abs/2603.15309"},"sourceCount":2},{"slug":"claw-bench","name":"Claw Bench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Cross-domain agent products","tags":["agents","pytest","tasks","leaderboard"],"type":"benchmark","whatItTests":"Agent products on 314 reproducible tasks across 33 domains and four difficulty levels.","bestFor":"Regression testing general-purpose agents with inspectable end-state verifiers.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/claw-bench/claw-bench","leaderboard":"https://clawbench.net/"},"sourceCount":2},{"slug":"claw-eval","name":"Claw-Eval","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Real-world agent evaluation","tags":["agents","tools","workflows","leaderboard"],"type":"benchmark","whatItTests":"Practical task completion by tool-using agent systems.","bestFor":"A current, source-backed signal for open agent frameworks.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/claw-eval/Claw-Eval","results":"https://huggingface.co/api/datasets/claw-eval/Claw-Eval/leaderboard"},"sourceCount":2},{"slug":"gaia","name":"GAIA","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Agents","tags":["Agents","agents"],"type":"agent benchmark","whatItTests":"Real-world assistant tasks requiring reasoning, browsing, multimodality, and tool use.","bestFor":"General agent task solving","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/gaia-benchmark/GAIA","leaderboard":"https://huggingface.co/spaces/gaia-benchmark/leaderboard"},"sourceCount":2},{"slug":"mcp-atlas","name":"MCP-Atlas","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Agents / Tool use","tags":["Agents","Tool use","agents"],"type":"agent benchmark","whatItTests":"Large-scale tool-use benchmark over real MCP servers and multi-call tasks.","bestFor":"Production-style MCP tool agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/scaleapi/mcp-atlas","leaderboard":"https://labs.scale.com/leaderboard/mcp_atlas","paper":"https://labs.scale.com/papers/mcpatlas"},"sourceCount":3},{"slug":"mind2web","name":"Mind2Web","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Browser agents","tags":["Browser agents"],"type":"benchmark","whatItTests":"Web navigation tasks across many real websites and domains.","bestFor":"Web-agent planning and UI grounding","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/OSU-NLP-Group/Mind2Web"},"sourceCount":1},{"slug":"officebench","name":"OfficeBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Office agents","tags":["Office agents","agents"],"type":"agent benchmark","whatItTests":"Office-document, email, calendar, and productivity-task automation.","bestFor":"Knowledge-worker office workflows","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"repo":"https://github.com/zlwang-cs/OfficeBench"},"sourceCount":1},{"slug":"osworld","name":"OSWorld","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Computer use","tags":["Computer use","agents"],"type":"agent benchmark","whatItTests":"Desktop/web/app tasks with execution scripts in realistic computer environments.","bestFor":"Computer-use agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://os-world.github.io/","repo":"https://github.com/xlang-ai/OSWorld"},"sourceCount":2},{"slug":"paperbench","name":"PaperBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Science / Agents","tags":["Science","Agents","agents"],"type":"agent benchmark","whatItTests":"Research-replication benchmark for agents attempting to reproduce AI papers and artifacts.","bestFor":"Research replication agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://openai.com/index/paperbench/","repo":"https://github.com/openai/frontier-evals/tree/main/project/paperbench"},"sourceCount":2},{"slug":"pm-bench","name":"PM-Bench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Prospective memory in agents","tags":["agents","memory","long horizon","2026"],"type":"benchmark","whatItTests":"Whether agents remember and execute delayed intentions during an ongoing simulated week.","bestFor":"Testing assistants that must reliably follow up when future cues occur.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2607.12385"},"sourceCount":1},{"slug":"researchclawbench","name":"ResearchClawBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Autonomous research agents","tags":["agents","research","discovery","long horizon"],"type":"benchmark","whatItTests":"Automated research agents on rediscovery and new-discovery workflows.","bestFor":"Comparing research-agent scaffolds rather than chat-model knowledge alone.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/InternScience/ResearchClawBench","results":"https://huggingface.co/api/datasets/InternScience/ResearchClawBench/leaderboard"},"sourceCount":2},{"slug":"runescape-bench-runebench","name":"runescape-bench / runebench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Emerging / Agents","tags":["Emerging","Agents","agents"],"type":"agent benchmark","whatItTests":"Game-world agent benchmark using RuneScape-like task environments.","bestFor":"Long-horizon game agents","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/MaxBittker/rs-bench","results":"https://maxbittker.github.io/runebench/"},"sourceCount":2},{"slug":"skillsbench","name":"SkillsBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Reusable agent skills","tags":["agents","skills","tools","workflows"],"type":"benchmark","whatItTests":"Whether packaged skills improve agent performance across diverse task environments.","bestFor":"Evaluating skill libraries, harness design, and portable agent procedures.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/benchflow/skillsbench","results":"https://huggingface.co/api/datasets/benchflow/skillsbench/leaderboard"},"sourceCount":2},{"slug":"sop-bench","name":"SOP-Bench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Industrial standard operating procedures","tags":["agents","SOP","industry","tools"],"type":"benchmark","whatItTests":"Agents on thousands of multi-step procedures across industrial domains.","bestFor":"Evaluating process adherence, completion, and tool accuracy in operations workflows.","runnability":"runnable","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/amazon-science/SOP-Bench","paper":"https://arxiv.org/abs/2506.08119"},"sourceCount":2},{"slug":"theagentcompany","name":"TheAgentCompany","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Enterprise agents","tags":["Enterprise agents","agents"],"type":"agent benchmark","whatItTests":"Digital-worker benchmark involving browsing, coding, programs, and simulated coworkers.","bestFor":"Professional task automation","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://the-agent-company.com/"},"sourceCount":1},{"slug":"toolbench-toolllm","name":"ToolBench / ToolLLM","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Tool use","tags":["Tool use"],"type":"benchmark","whatItTests":"API-use tasks for tool-augmented LLMs across large tool collections.","bestFor":"Tool-use research baselines","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/OpenBMB/ToolBench"},"sourceCount":1},{"slug":"toolsandbox","name":"ToolSandbox","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Tool use","tags":["Tool use","agents"],"type":"agent benchmark","whatItTests":"Stateful conversational tool-use tasks with tool execution and environment state.","bestFor":"Robust multi-turn tool agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/apple/ToolSandbox"},"sourceCount":1},{"slug":"traject-bench","name":"TRAJECT-Bench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Trajectory-aware tool use","tags":["agents","trajectories","tools","process"],"type":"benchmark","whatItTests":"Agent tool use with evaluation of the intermediate trajectory as well as the final answer.","bestFor":"Diagnosing how an agent arrived at an outcome, not only whether it succeeded.","runnability":"partial","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"source":"https://www.amazon.science/publications/traject-bench-a-trajectory-aware-benchmark-for-evaluating-agentic-tool-use"},"sourceCount":1},{"slug":"visualwebarena","name":"VisualWebArena","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Browser agents","tags":["Browser agents","agents"],"type":"agent benchmark","whatItTests":"Browser tasks requiring visual understanding of websites and UI state.","bestFor":"Multimodal web agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/web-arena-x/visualwebarena"},"sourceCount":1},{"slug":"webarena","name":"WebArena","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Browser agents","tags":["Browser agents","agents"],"type":"agent benchmark","whatItTests":"Self-hosted realistic websites and natural-language browser tasks.","bestFor":"Browser automation agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://webarena.dev/","repo":"https://github.com/web-arena-x/webarena"},"sourceCount":2},{"slug":"wildclawbench","name":"WildClawBench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Real-world autonomous agent work","tags":["agents","real world","tools","long horizon"],"type":"benchmark","whatItTests":"Agent performance on diverse real-world tasks and environments.","bestFor":"Comparing general-purpose agent systems on operational work.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/internlm/WildClawBench","results":"https://huggingface.co/api/datasets/internlm/WildClawBench/leaderboard"},"sourceCount":2},{"slug":"workarena-workarena","name":"WorkArena / WorkArena++","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Enterprise agents","tags":["Enterprise agents","agents"],"type":"agent benchmark","whatItTests":"Enterprise workflow tasks modeled in ServiceNow-style environments.","bestFor":"Business process agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/ServiceNow/WorkArena"},"sourceCount":1},{"slug":"yc-bench","name":"YC-Bench","primaryDomain":"Agents & Tool Use","primaryDomainSlug":"agents-tool-use","originalDomain":"Startup CEO simulation","tags":["agents","business","simulation","long horizon"],"type":"benchmark","whatItTests":"Agent decision-making across a simulated year of startup operations.","bestFor":"Exploring long-horizon business-agent behavior and trade-offs.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/collinear-ai/yc-bench","results":"https://huggingface.co/api/datasets/collinear-ai/yc-bench/leaderboard"},"sourceCount":2},{"slug":"aider-polyglot","name":"Aider Polyglot","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding","tags":["Coding"],"type":"benchmark","whatItTests":"Multi-language code editing and test-passing benchmark for practical coding assistants.","bestFor":"Code edit models across languages","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"leaderboard":"https://aider.chat/docs/leaderboards/"},"sourceCount":1},{"slug":"ale-bench","name":"ALE-Bench","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Emerging / Coding","tags":["Emerging","Coding"],"type":"benchmark","whatItTests":"Long-horizon algorithm-engineering contest tasks.","bestFor":"Optimization and algorithm engineering agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/SakanaAI/ALE-Bench"},"sourceCount":1},{"slug":"bigcodebench","name":"BigCodeBench","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding","tags":["Coding"],"type":"benchmark","whatItTests":"Software-engineering-oriented code-generation tasks designed to go beyond HumanEval and MBPP.","bestFor":"Modern code-generation baselines","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/bigcode-project/bigcodebench","source":"https://bigcode-bench.github.io/"},"sourceCount":2},{"slug":"cursorbench","name":"CursorBench","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Emerging / Coding","tags":["Emerging","Coding"],"type":"benchmark","whatItTests":"Cursor's proprietary/internal offline eval suite from real Cursor sessions, focused on correctness, code quality, efficiency, and interaction behavior.","bestFor":"Editor-agent evaluation context","runnability":"hosted","maturity":"current","access":"commercial","risk":"standard","editorialStatus":"commercial","lastVerified":"2026-07-15","links":{"source":"https://cursor.com/blog/cursorbench"},"sourceCount":1},{"slug":"deepswe","name":"DeepSWE","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Emerging / SWE","tags":["Emerging","SWE","agents"],"type":"agent benchmark","whatItTests":"Original long-horizon software engineering tasks across several languages with isolated environments and verifiers.","bestFor":"Emerging SWE agent evaluation","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"repo":"https://github.com/datacurve-ai/deep-swe","dataset":"https://huggingface.co/datasets/datacurve/deep-swe","results":"https://huggingface.co/api/datasets/datacurve/deep-swe/leaderboard"},"sourceCount":3},{"slug":"gba-eval","name":"GBA-Eval","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Emerging / Coding","tags":["Emerging","Coding"],"type":"benchmark","whatItTests":"Single high-quality long-horizon Game Boy Advance SWE eval case; useful signal, not a complete coding benchmark.","bestFor":"Experimental coding-agent signal","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"source":"https://gbaeval.com/","post":"https://www.mechanize.work/technical-blog/introducing-gba-eval/"},"sourceCount":2},{"slug":"humaneval","name":"HumanEval","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding","tags":["Coding"],"type":"benchmark","whatItTests":"Classic Python function synthesis benchmark with unit tests.","bestFor":"Legacy coding baseline","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/openai/human-eval"},"sourceCount":1},{"slug":"livecodebench","name":"LiveCodeBench","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding","tags":["Coding"],"type":"benchmark","whatItTests":"Recent contest-style coding problems with contamination-aware releases.","bestFor":"Code-generation and algorithmic coding signal","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/LiveCodeBench/LiveCodeBench"},"sourceCount":1},{"slug":"mbpp","name":"MBPP","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding","tags":["Coding"],"type":"benchmark","whatItTests":"Mostly Basic Programming Problems for simple Python coding tasks.","bestFor":"Small-model and baseline coding checks","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/google-research-datasets/mbpp"},"sourceCount":1},{"slug":"nvidia-computeeval","name":"NVIDIA ComputeEval","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"CUDA correctness and performance","tags":["CUDA","GPU","code","performance"],"type":"benchmark","whatItTests":"Correctness and runtime performance of generated GPU compute code.","bestFor":"Evaluating coding systems that optimize kernels or write CUDA.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/nvidia/compute-eval","results":"https://huggingface.co/api/datasets/nvidia/compute-eval/leaderboard"},"sourceCount":2},{"slug":"programbench","name":"ProgramBench","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Emerging / Coding","tags":["Emerging","Coding"],"type":"benchmark","whatItTests":"Rebuilding programs from compiled binaries and documentation.","bestFor":"Reverse engineering and deep coding","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/ProgramBench/ProgramBench"},"sourceCount":1},{"slug":"scicode","name":"SciCode","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Science / Coding","tags":["Science","Coding"],"type":"benchmark","whatItTests":"Scientist-curated coding problems from real natural-science contexts.","bestFor":"Scientific coding agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/scicode-bench/SciCode"},"sourceCount":1},{"slug":"swe-bench","name":"SWE-bench","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding / SWE","tags":["Coding","SWE","agents"],"type":"agent benchmark","whatItTests":"Real GitHub issues that require modifying repositories and passing tests.","bestFor":"Software engineering agent capability","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://www.swebench.com/","repo":"https://github.com/swe-bench/SWE-bench"},"sourceCount":2},{"slug":"swe-bench-pro","name":"SWE-bench Pro","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding / SWE","tags":["Coding","SWE","agents"],"type":"agent benchmark","whatItTests":"Harder held-out/private-repo software engineering tasks intended to reduce contamination.","bestFor":"Frontier SWE agents and benchmark saturation checks","runnability":"hosted","maturity":"current","access":"commercial","risk":"standard","editorialStatus":"commercial","lastVerified":"2026-07-15","links":{"leaderboard":"https://labs.scale.com/leaderboard/swe_bench_pro_public","paper":"https://labs.scale.com/papers/swe_bench_pro","dataset":"https://huggingface.co/datasets/ScaleAI/SWE-bench_Pro","results":"https://huggingface.co/api/datasets/ScaleAI/SWE-bench_Pro/leaderboard"},"sourceCount":4},{"slug":"swe-bench-verified","name":"SWE-bench Verified","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding / SWE","tags":["Coding","SWE","agents"],"type":"agent benchmark","whatItTests":"Human-filtered subset of SWE-bench with higher-quality real issue tasks.","bestFor":"Default SWE benchmark slice","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/SWE-bench/SWE-bench_Verified","leaderboard":"https://www.swebench.com/","results":"https://huggingface.co/api/datasets/SWE-bench/SWE-bench_Verified/leaderboard"},"sourceCount":3},{"slug":"swe-marathon","name":"SWE-Marathon","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Emerging / SWE","tags":["Emerging","SWE","agents"],"type":"agent benchmark","whatItTests":"Ultra-long-horizon software engineering tasks.","bestFor":"Long-horizon coding agents","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://www.swe-marathon.org/"},"sourceCount":1},{"slug":"terminal-bench-2-0","name":"Terminal-Bench 2.0","primaryDomain":"Coding & SWE","primaryDomainSlug":"coding-swe","originalDomain":"Coding / Agents","tags":["Coding","Agents","agents"],"type":"agent benchmark","whatItTests":"Terminal-based task execution benchmark for agents working in shell environments.","bestFor":"CLI agents and software task execution","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"leaderboard":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","source":"https://www.tbench.ai/","dataset":"https://huggingface.co/datasets/harborframework/terminal-bench-2.0","results":"https://huggingface.co/api/datasets/harborframework/terminal-bench-2.0/leaderboard"},"sourceCount":4},{"slug":"arguana","name":"ArguAna","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Counterargument retrieval","tags":["retrieval","arguments","MTEB","ranking"],"type":"benchmark","whatItTests":"Retrieval of counterarguments for a given argumentative claim.","bestFor":"Diagnosing semantic retrieval beyond topical similarity.","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/mteb/arguana","results":"https://huggingface.co/api/datasets/mteb/arguana/leaderboard"},"sourceCount":2},{"slug":"beir","name":"BEIR","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Retrieval / RAG","tags":["Retrieval","RAG"],"type":"benchmark","whatItTests":"Heterogeneous information-retrieval benchmark for zero-shot retrieval across many datasets and domains.","bestFor":"Retriever selection for RAG systems","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/beir-cellar/beir","paper":"https://arxiv.org/abs/2104.08663"},"sourceCount":2},{"slug":"bright","name":"BRIGHT","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Reasoning-intensive retrieval","tags":["retrieval","reasoning","search","documents"],"type":"benchmark","whatItTests":"Retrieval where finding relevant evidence requires multi-step reasoning.","bestFor":"Selecting retrievers for difficult research and professional search tasks.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/mteb/BRIGHT","results":"https://huggingface.co/api/datasets/mteb/BRIGHT/leaderboard"},"sourceCount":2},{"slug":"crag","name":"CRAG","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"RAG","tags":["RAG"],"type":"benchmark","whatItTests":"Comprehensive RAG benchmark with factual QA and mock APIs for retrieval.","bestFor":"RAG factuality and retrieval stress tests","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/facebookresearch/CRAG","paper":"https://arxiv.org/abs/2406.04744"},"sourceCount":2},{"slug":"docvqa","name":"DocVQA","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Document AI","tags":["Document AI"],"type":"benchmark","whatItTests":"Visual question answering over document images.","bestFor":"PDF, OCR, and document-agent evaluation","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://www.docvqa.org/"},"sourceCount":1},{"slug":"infinitebench","name":"InfiniteBench","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Long context","tags":["Long context"],"type":"benchmark","whatItTests":"Super-long-context benchmark beyond 100k tokens.","bestFor":"Context-window stress testing","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/OpenBMB/InfiniteBench"},"sourceCount":1},{"slug":"longbench-longbench-v2","name":"LongBench / LongBench v2","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Long context","tags":["Long context"],"type":"benchmark","whatItTests":"Long-context understanding and reasoning across documents and realistic multitask scenarios.","bestFor":"Long-context model screening","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/THUDM/LongBench","source":"https://longbench2.github.io/"},"sourceCount":2},{"slug":"mdpbench","name":"MDPBench","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Multilingual document parsing","tags":["documents","multilingual","parsing","OCR"],"type":"benchmark","whatItTests":"Real-world document parsing across languages, layouts, and content structures.","bestFor":"Comparing document intelligence pipelines serving multilingual corpora.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/Delores-Lin/MDPBench","results":"https://huggingface.co/api/datasets/Delores-Lin/MDPBench/leaderboard"},"sourceCount":2},{"slug":"mteb","name":"MTEB","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Retrieval / Embeddings","tags":["Retrieval","Embeddings"],"type":"benchmark","whatItTests":"Massive Text Embedding Benchmark for comparing embedding models across retrieval, clustering, classification, reranking, and semantic similarity tasks.","bestFor":"Embedding and retrieval model selection","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://huggingface.co/mteb","repo":"https://github.com/embeddings-benchmark/mteb"},"sourceCount":2},{"slug":"olmocr-bench","name":"olmOCR-bench","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"PDF OCR and extraction","tags":["OCR","PDF","documents","tables"],"type":"benchmark","whatItTests":"OCR fidelity across diverse PDF pages using thousands of document-level unit tests.","bestFor":"Choosing extraction components before evaluating downstream document RAG.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/allenai/olmOCR-bench","results":"https://huggingface.co/api/datasets/allenai/olmOCR-bench/leaderboard"},"sourceCount":2},{"slug":"omnidocbench","name":"OmniDocBench","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Document AI","tags":["Document AI"],"type":"benchmark","whatItTests":"Document parsing benchmark for OCR, layout, table, formula, and reading-order extraction.","bestFor":"Document parsing for RAG pipelines","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/opendatalab/OmniDocBench"},"sourceCount":1},{"slug":"parsebench","name":"ParseBench","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"Enterprise document parsing","tags":["documents","parsing","OCR","RAG"],"type":"benchmark","whatItTests":"Document parser accuracy on enterprise layouts and structured content.","bestFor":"Selecting a parser before measuring retrieval and grounded-answer quality.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/llamaindex/ParseBench","results":"https://huggingface.co/api/datasets/llamaindex/ParseBench/leaderboard"},"sourceCount":2},{"slug":"ragbench","name":"RAGBench","primaryDomain":"Retrieval & RAG","primaryDomainSlug":"retrieval-rag","originalDomain":"RAG","tags":["RAG"],"type":"benchmark","whatItTests":"Explainable RAG benchmark across documents, retrieval, generation, and attribution.","bestFor":"RAG system evaluation","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2407.11005","dataset":"https://huggingface.co/datasets/rungalileo/ragbench"},"sourceCount":2},{"slug":"crmarena","name":"CRMArena","primaryDomain":"Customer Support","primaryDomainSlug":"customer-support","originalDomain":"CX / CRM","tags":["CX","CRM","agents"],"type":"agent benchmark","whatItTests":"CRM workflows for service agents, analysts, and business operations.","bestFor":"CRM and customer-ops agents","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://arxiv.org/abs/2411.02305"},"sourceCount":1},{"slug":"tau-bench","name":"tau-bench","primaryDomain":"Customer Support","primaryDomainSlug":"customer-support","originalDomain":"CX / Support","tags":["CX","Support","agents"],"type":"agent benchmark","whatItTests":"Customer-service agents in retail and airline domains using APIs and policy guidelines.","bestFor":"Support-agent reliability","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://taubench.com/","repo":"https://github.com/sierra-research/tau-bench"},"sourceCount":2},{"slug":"tau-knowledge","name":"tau-knowledge","primaryDomain":"Customer Support","primaryDomainSlug":"customer-support","originalDomain":"CX / RAG","tags":["CX","RAG","agents"],"type":"agent benchmark","whatItTests":"Knowledge-intensive support extension to the tau-bench family.","bestFor":"Support agents that retrieve policy knowledge","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://taubench.com/blog/tau-knowledge.html"},"sourceCount":1},{"slug":"tau-voice","name":"tau-voice","primaryDomain":"Customer Support","primaryDomainSlug":"customer-support","originalDomain":"Voice / CX","tags":["Voice","CX","agents"],"type":"agent benchmark","whatItTests":"Full-duplex voice customer-service tasks scored against final database state.","bestFor":"Voice support agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://sierra.ai/blog/tau-voice-benchmarking-real-time-voice-agents-on-real-world-tasks","paper":"https://arxiv.org/abs/2603.13686"},"sourceCount":2},{"slug":"tau2-bench-tau3-bench","name":"tau2-bench / tau3-bench","primaryDomain":"Customer Support","primaryDomainSlug":"customer-support","originalDomain":"CX / Support","tags":["CX","Support","agents"],"type":"agent benchmark","whatItTests":"Customer-service simulation framework with text, voice, policies, tools, multiple domains, and tau3 task-fix updates.","bestFor":"CX eval harness design","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/sierra-research/tau2-bench","source":"https://taubench.com/","tau3":"https://taubench.com/blog/tau3-task-fixes.html"},"sourceCount":3},{"slug":"harvey-biglaw-bench","name":"Harvey BigLaw Bench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal","tags":["Legal","scoreboard"],"type":"scoreboard","whatItTests":"Harvey benchmark context for BigLaw-style legal tasks; useful market signal but not an open runnable benchmark.","bestFor":"Legal AI market context","runnability":"hosted","maturity":"current","access":"commercial","risk":"standard","editorialStatus":"commercial","lastVerified":"2026-07-15","links":{"source":"https://www.harvey.ai/blog/introducing-biglaw-bench"},"sourceCount":1},{"slug":"harvey-legal-agent-benchmark-lab","name":"Harvey Legal Agent Benchmark / LAB","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal","tags":["Legal","agents"],"type":"agent benchmark","whatItTests":"Open legal-agent benchmark with long-horizon tasks across practice areas and expert rubric criteria.","bestFor":"Agentic legal work product","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/harveyai/harvey-labs","source":"https://www.harvey.ai/blog/introducing-harveys-legal-agent-benchmark","results":"https://www.harvey.ai/blog/legal-agent-benchmark-initial-results"},"sourceCount":3},{"slug":"lawbench","name":"LawBench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal","tags":["Legal"],"type":"benchmark","whatItTests":"Legal tasks across entity recognition, reading comprehension, legal consultation, and more.","bestFor":"Chinese/legal reasoning research","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/open-compass/LawBench"},"sourceCount":1},{"slug":"legal-rag-bench-2026","name":"Legal RAG Bench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"End-to-end legal research RAG","tags":["legal","RAG","retrieval","citations"],"type":"benchmark","whatItTests":"End-to-end legal retrieval and reasoning on realistic research tasks.","bestFor":"Comparing legal RAG systems where sources and reasoning both matter.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"post":"https://huggingface.co/blog/isaacus/legal-rag-bench"},"sourceCount":1},{"slug":"legalbench","name":"LegalBench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal","tags":["Legal"],"type":"benchmark","whatItTests":"162 legal reasoning tasks contributed by lawyers, law professors, researchers, and legal practitioners.","bestFor":"Legal reasoning baseline","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://www.legalbench.ai/","repo":"https://github.com/HazyResearch/legalbench"},"sourceCount":2},{"slug":"legalbench-rag","name":"LegalBench-RAG","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal / RAG","tags":["Legal","RAG"],"type":"benchmark","whatItTests":"Legal retrieval and generation benchmark for end-to-end legal RAG systems.","bestFor":"Legal document retrieval and grounded answers","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/zeroentropy-ai/legalbenchrag","paper":"https://arxiv.org/abs/2408.10343"},"sourceCount":2},{"slug":"lexam","name":"LEXam","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Swiss and international legal exams","tags":["legal","Europe","exam","multilingual"],"type":"benchmark","whatItTests":"Legal reasoning on hundreds of Swiss, EU, and international law examination questions.","bestFor":"Comparing legal knowledge across European and international jurisdictions.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/LEXam-Benchmark/LEXam","results":"https://huggingface.co/api/datasets/LEXam-Benchmark/LEXam/leaderboard"},"sourceCount":2},{"slug":"lexglue","name":"LexGLUE","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal","tags":["Legal"],"type":"benchmark","whatItTests":"Legal NLP benchmark suite in a SuperGLUE-like format.","bestFor":"Classic legal NLP tasks","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/coastalcph/lex-glue"},"sourceCount":1},{"slug":"plawbench","name":"PLawBench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Real-world legal practice","tags":["legal","workflows","rubric","documents"],"type":"benchmark","whatItTests":"Legal consultation, case analysis, and document generation with workflow-grounded rubrics.","bestFor":"Evaluating legal-practice outputs rather than multiple-choice legal knowledge.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"paper":"https://aclanthology.org/2026.acl-long.458/","repo":"https://github.com/SKYLENAGE-AI/PLawBench"},"sourceCount":2},{"slug":"redlinebench","name":"RedlineBench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Contract negotiation","tags":["legal","contracts","negotiation","multi-turn"],"type":"benchmark","whatItTests":"Multi-turn contract redlining and negotiation behavior.","bestFor":"Evaluating legal agents that must preserve objectives across revisions.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/crosbylegal/RedlineBench","results":"https://huggingface.co/api/datasets/crosbylegal/RedlineBench/leaderboard"},"sourceCount":2},{"slug":"vals-ai-legalbench","name":"Vals AI LegalBench","primaryDomain":"Legal","primaryDomainSlug":"legal","originalDomain":"Legal","tags":["Legal","scoreboard"],"type":"scoreboard","whatItTests":"Hosted leaderboard for legal-model evaluation.","bestFor":"Current legal model comparison","runnability":"hosted","maturity":"current","access":"commercial","risk":"standard","editorialStatus":"commercial","lastVerified":"2026-07-15","links":{"leaderboard":"https://www.vals.ai/benchmarks/legal_bench"},"sourceCount":1},{"slug":"evasionbench","name":"EvasionBench","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Evasive financial communication","tags":["finance","earnings calls","evasion","language"],"type":"benchmark","whatItTests":"Detection and handling of evasive answers in corporate earnings calls.","bestFor":"Evaluating financial-analysis systems that must distinguish disclosure from deflection.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/FutureMa/EvasionBench","results":"https://huggingface.co/api/datasets/FutureMa/EvasionBench/leaderboard"},"sourceCount":2},{"slug":"financeagent-fab-v2","name":"FinanceAgent / FAB v2","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance","tags":["Finance","agents"],"type":"agent benchmark","whatItTests":"Financial-agent benchmark for realistic analyst and finance workflow tasks.","bestFor":"Finance agents and analyst workflows","runnability":"hosted","maturity":"current","access":"commercial","risk":"standard","editorialStatus":"commercial","lastVerified":"2026-07-15","links":{"leaderboard":"https://www.vals.ai/benchmarks/fabv2"},"sourceCount":1},{"slug":"financebench","name":"FinanceBench","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance","tags":["Finance"],"type":"benchmark","whatItTests":"Open-book financial QA over public company filings with evidence strings.","bestFor":"Financial analyst and SEC filing workflows","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/patronus-ai/financebench"},"sourceCount":1},{"slug":"finben-pixiu","name":"FinBen / PIXIU","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance","tags":["Finance"],"type":"benchmark","whatItTests":"Broad financial benchmark suite across extraction, QA, generation, forecasting, and decision-making.","bestFor":"Finance model evaluation suites","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/The-FinAI/PIXIU","paper":"https://arxiv.org/abs/2402.12659"},"sourceCount":2},{"slug":"finqa-convfinqa","name":"FinQA / ConvFinQA","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance","tags":["Finance"],"type":"benchmark","whatItTests":"Financial numerical reasoning over reports using structured and unstructured evidence.","bestFor":"Financial math and reasoning","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://finqasite.github.io/index.html"},"sourceCount":1},{"slug":"gdpval","name":"GDPval","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance / Professional work","tags":["Finance","Professional work"],"type":"benchmark","whatItTests":"Economically valuable professional tasks across domains including finance, insurance, and operations.","bestFor":"Professional work automation signal","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://openai.com/index/gdpval/","paper":"https://arxiv.org/abs/2510.04374"},"sourceCount":2},{"slug":"financial-services-meta-benchmarks","name":"Meta-Benchmarks for Financial Services","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Task-weighted benchmark aggregation","tags":["meta-evaluation","finance","Elo","model selection"],"type":"toolkit","whatItTests":"How 452 reported benchmarks map into 41 work activities and 38 banking business domains.","bestFor":"Designing evidence-weighted model selection for financial-services workflows.","runnability":"partial","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2607.01740"},"sourceCount":1},{"slug":"open-finllm-leaderboard","name":"Open FinLLM Leaderboard","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance","tags":["Finance","scoreboard"],"type":"scoreboard","whatItTests":"Open leaderboard for financial LLM evaluation.","bestFor":"Financial model comparison","runnability":"hosted","maturity":"current","access":"hosted","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"leaderboard":"https://huggingface.co/spaces/TheFinAI/Open-Financial-LLM-Leaderboard"},"sourceCount":1},{"slug":"qfbench","name":"QFBench","primaryDomain":"Finance","primaryDomainSlug":"finance","originalDomain":"Finance","tags":["Finance","agents"],"type":"agent benchmark","whatItTests":"State-aware quantitative-finance agent tasks requiring code, market logic, and financial reasoning.","bestFor":"Quant finance agents","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://qfbench.com/","repo":"https://github.com/QF-Bench/QuantitativeFinance-Bench"},"sourceCount":2},{"slug":"bioasq","name":"BioASQ","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Biomedical","tags":["Biomedical"],"type":"benchmark","whatItTests":"Biomedical semantic indexing and QA challenges.","bestFor":"Biomedical search and QA","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://www.bioasq.org/"},"sourceCount":1},{"slug":"clibench","name":"CliBench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Clinical decisions on diagnoses, procedures, lab-test orders, and prescriptions with structured output ontologies.","bestFor":"Clinical decision granularity","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"source":"https://clibench.github.io/"},"sourceCount":1},{"slug":"clinicbench","name":"ClinicBench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Clinical language generation, understanding, and reasoning tasks including open-ended clinical decision-making.","bestFor":"Clinical decision support research","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/AI-in-Health/ClinicBench"},"sourceCount":1},{"slug":"healthbench","name":"HealthBench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Realistic health conversations with physician-created rubrics.","bestFor":"Health assistant response quality","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://openai.com/index/healthbench/","repo":"https://github.com/openai/simple-evals","dataset":"https://huggingface.co/datasets/openai/healthbench"},"sourceCount":3},{"slug":"healthbench-professional","name":"HealthBench Professional","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Professional healthcare benchmark variant focused on clinician workflows and higher-stakes medical tasks.","bestFor":"Clinical workflow response quality","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2604.27470","source":"https://openai.com/index/healthbench/"},"sourceCount":2},{"slug":"medagentbench","name":"MedAgentBench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare / Agents","tags":["Healthcare","Agents","agents"],"type":"agent benchmark","whatItTests":"Virtual EHR/FHIR environment with clinically relevant tasks requiring agents to retrieve, record, order, and act in medical-record settings.","bestFor":"Clinical workflow agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://stanfordmlgroup.github.io/projects/medagentbench/","repo":"https://github.com/stanfordmlgroup/MedAgentBench","paper":"https://arxiv.org/abs/2501.14654"},"sourceCount":3},{"slug":"medcalc-bench","name":"MedCalc-Bench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Medical calculation tasks for evaluating whether LLMs can serve as clinical calculators.","bestFor":"Medical arithmetic and calculators","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/ncbi-nlp/MedCalc-Bench"},"sourceCount":1},{"slug":"medhelm","name":"MedHELM","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare","harness"],"type":"harness","whatItTests":"HELM-style medical evaluation framework with a clinical taxonomy, healthcare task categories, and public/gated/private medical benchmarks.","bestFor":"Holistic medical evals","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://crfm.stanford.edu/helm/medhelm/v2.0.0/","site":"https://medhelm.org/"},"sourceCount":2},{"slug":"medmcqa","name":"MedMCQA","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Large-scale medical entrance-exam question set across healthcare subjects.","bestFor":"Medical multiple-choice baseline","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/openlifescienceai/medmcqa"},"sourceCount":1},{"slug":"medqa","name":"MedQA","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Medical board-style QA for clinical knowledge and exam reasoning.","bestFor":"Medical QA baseline","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/jind11/MedQA"},"sourceCount":1},{"slug":"multimedqa","name":"MultiMedQA","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare"],"type":"benchmark","whatItTests":"Composite medical QA benchmark spanning MedQA, MedMCQA, PubMedQA, LiveQA, MedicationQA, MMLU clinical topics, and HealthSearchQA.","bestFor":"Medical QA suite comparison","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"paper":"https://pubmed.ncbi.nlm.nih.gov/37438534/"},"sourceCount":1},{"slug":"openmed","name":"OpenMed","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Healthcare","tags":["Healthcare","toolkit"],"type":"toolkit","whatItTests":"Open-source healthcare NLP toolkit, model hub, and curated clinical/biomedical resources; useful source layer rather than a benchmark leaderboard.","bestFor":"Medical NLP resources and PHI-safe extraction","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/maziyarpanahi/openmed","source":"https://openmed.life/","hub":"https://huggingface.co/OpenMed"},"sourceCount":3},{"slug":"pubmedqa","name":"PubMedQA","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Biomedical","tags":["Biomedical"],"type":"benchmark","whatItTests":"Biomedical research QA requiring reasoning over PubMed abstracts.","bestFor":"Biomedical literature QA","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://pubmedqa.github.io/","paper":"https://arxiv.org/abs/1909.06146"},"sourceCount":2},{"slug":"voxclinbench","name":"VoxClinBench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Voice / Healthcare","tags":["Voice","Healthcare"],"type":"benchmark","whatItTests":"Clinical voice benchmark with cross-lingual expansion.","bestFor":"Medical speech and clinical voice agents","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/voice-bench-submission/voxclinbench","repo":"https://github.com/voice-bench-submission/voxclinbench"},"sourceCount":2},{"slug":"chi-bench","name":"χ-Bench","primaryDomain":"Healthcare","primaryDomainSlug":"healthcare","originalDomain":"Policy-rich healthcare workflows","tags":["healthcare","agents","policy","long horizon"],"type":"benchmark","whatItTests":"Long-horizon healthcare workflows with policy, process, and tool constraints.","bestFor":"Evaluating clinical operations agents beyond medical question answering.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/actava/chi-bench","results":"https://huggingface.co/api/datasets/actava/chi-bench/leaderboard"},"sourceCount":2},{"slug":"aime-2026","name":"AIME 2026","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Current competition mathematics","tags":["math","AIME","reasoning","2026"],"type":"benchmark","whatItTests":"Advanced mathematical problem solving on the 2026 AIME contest set.","bestFor":"A dated, difficult math signal with explicit versioning.","runnability":"runnable","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/MathArena/aime_2026","results":"https://huggingface.co/api/datasets/MathArena/aime_2026/leaderboard"},"sourceCount":2},{"slug":"airs-bench","name":"AIRS-Bench","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Autonomous AI research","tags":["agents","machine learning","research","experiments"],"type":"benchmark","whatItTests":"End-to-end ML research ability on tasks derived from state-of-the-art papers.","bestFor":"Comparing AI research agents on executable experiment outcomes.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/facebookresearch/airs-bench","paper":"https://arxiv.org/abs/2602.06855"},"sourceCount":2},{"slug":"arc-agi-2","name":"ARC-AGI-2","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"General reasoning","tags":["General reasoning"],"type":"benchmark","whatItTests":"Abstract visual reasoning tasks focused on generalization from few examples.","bestFor":"Abstract reasoning and generalization stress testing","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://arcprize.org/","paper":"https://arxiv.org/abs/2505.11831"},"sourceCount":2},{"slug":"bbh-big-bench-hard","name":"BBH / BIG-Bench Hard","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"General reasoning","tags":["General reasoning"],"type":"benchmark","whatItTests":"Hard tasks from BIG-bench covering symbolic, logical, and multi-step reasoning.","bestFor":"Legacy reasoning regression checks","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"repo":"https://github.com/suzgunmirac/BIG-Bench-Hard"},"sourceCount":1},{"slug":"critpt","name":"CritPt","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Physics","tags":["Physics"],"type":"benchmark","whatItTests":"Research-level physics tasks from modern physics areas.","bestFor":"Niche frontier physics reasoning","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/CritPt-Benchmark/CritPt"},"sourceCount":1},{"slug":"gpqa-gpqa-diamond","name":"GPQA / GPQA Diamond","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Science","tags":["Science"],"type":"benchmark","whatItTests":"Graduate-level science questions designed by domain experts and difficult for non-experts.","bestFor":"Expert science reasoning","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/idavidrein/gpqa","paper":"https://arxiv.org/abs/2311.12022","dataset":"https://huggingface.co/datasets/Idavidrein/gpqa","results":"https://huggingface.co/api/datasets/Idavidrein/gpqa/leaderboard"},"sourceCount":4},{"slug":"gsm8k","name":"GSM8K","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Grade-school math reasoning","tags":["math","reasoning","chain of thought"],"type":"benchmark","whatItTests":"Multi-step arithmetic reasoning on grade-school word problems.","bestFor":"Historical baselines and regression checks; it is increasingly saturated for frontier-model selection.","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/openai/gsm8k","results":"https://huggingface.co/api/datasets/openai/gsm8k/leaderboard"},"sourceCount":2},{"slug":"hmmt-feb-2026","name":"HMMT February 2026","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Current competition mathematics","tags":["math","HMMT","reasoning","2026"],"type":"benchmark","whatItTests":"Advanced competition mathematics on the February 2026 HMMT set.","bestFor":"A fresh math signal that complements AIME-style evaluation.","runnability":"runnable","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/MathArena/hmmt_feb_2026","results":"https://huggingface.co/api/datasets/MathArena/hmmt_feb_2026/leaderboard"},"sourceCount":2},{"slug":"humanity-s-last-exam","name":"Humanity's Last Exam","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"General reasoning","tags":["General reasoning"],"type":"benchmark","whatItTests":"Expert-level closed-ended multimodal academic questions intended for frontier model differentiation.","bestFor":"Hard frontier academic reasoning","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/centerforaisafety/hle","source":"https://lastexam.ai/","dataset":"https://huggingface.co/datasets/cais/hle","results":"https://huggingface.co/api/datasets/cais/hle/leaderboard"},"sourceCount":4},{"slug":"lab-bench-labbench2","name":"LAB-Bench / LABBench2","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Biology","tags":["Biology"],"type":"benchmark","whatItTests":"Biology research tasks covering literature, protocols, databases, DNA/protein sequences, and lab reasoning; LABBench2 adds a newer dataset and harness.","bestFor":"Biology research assistants","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2604.09554","dataset":"https://huggingface.co/datasets/futurehouse/labbench2","harness":"https://github.com/EdisonScientific/labbench2"},"sourceCount":3},{"slug":"lifescibench","name":"LifeSciBench","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Real-world life science research","tags":["life science","research","expert","agents"],"type":"benchmark","whatItTests":"Expert-written, expert-reviewed tasks grounded in practical life-science research.","bestFor":"Evaluating whether systems can support realistic research work beyond biology QA.","runnability":"hosted","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"source":"https://openai.com/index/introducing-life-sci-bench/"},"sourceCount":1},{"slug":"livebench","name":"LiveBench","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"General reasoning","tags":["General reasoning"],"type":"benchmark","whatItTests":"Fresh, contamination-conscious questions with objective scoring across reasoning, math, coding, language, and data analysis.","bestFor":"General model signal that resists stale benchmark gaming","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://livebench.ai/","repo":"https://github.com/livebench/livebench"},"sourceCount":2},{"slug":"math-aime-style-evals","name":"MATH / AIME-style evals","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Math","tags":["Math"],"type":"benchmark","whatItTests":"Competitive math problem solving and formal reasoning tasks.","bestFor":"Mathematical reasoning and launch-card comparison","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/hendrycks/math"},"sourceCount":1},{"slug":"mle-bench","name":"MLE-bench","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"ML research","tags":["ML research","agents"],"type":"agent benchmark","whatItTests":"Machine-learning engineering tasks drawn from Kaggle-style competitions.","bestFor":"ML engineering agents","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/openai/mle-bench","paper":"https://arxiv.org/abs/2410.07095"},"sourceCount":2},{"slug":"mmlu-pro","name":"MMLU-Pro","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"General reasoning","tags":["General reasoning"],"type":"benchmark","whatItTests":"Harder multiple-choice academic and professional knowledge benchmark derived from MMLU with more options.","bestFor":"Broad expert knowledge screening","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/TIGER-AI-Lab/MMLU-Pro","paper":"https://arxiv.org/abs/2406.01574","dataset":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro","results":"https://huggingface.co/api/datasets/TIGER-Lab/MMLU-Pro/leaderboard"},"sourceCount":4},{"slug":"nanofold-public","name":"NanoFold Public","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Protein folding","tags":["protein","biology","science","folding"],"type":"benchmark","whatItTests":"Scientific-model performance on public protein-folding tasks.","bestFor":"Specialist life-science model comparisons with a public result feed.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/ChrisHayduk/nanofold-public","results":"https://huggingface.co/api/datasets/ChrisHayduk/nanofold-public/leaderboard"},"sourceCount":2},{"slug":"pencil-puzzle-bench","name":"Pencil Puzzle Bench","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Reasoning","tags":["Reasoning"],"type":"benchmark","whatItTests":"Deterministically verifiable constraint-satisfaction puzzle tasks.","bestFor":"Reasoning without LLM judges","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"repo":"https://github.com/approximatelabs/pencil-puzzle-bench"},"sourceCount":1},{"slug":"sciagentarena","name":"SciAgentArena","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"Scientific research agents","tags":["agents","science","research","biomedicine"],"type":"benchmark","whatItTests":"Scientific agents on about 200 stepwise-verified tasks across five biomedical domains.","bestFor":"Comparing research-agent reliability, cost, and task-level scientific contribution.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"site":"https://sciagentarena.github.io/","dataset":"https://huggingface.co/datasets/iLOVE2D/SciAgentArena","paper":"https://arxiv.org/abs/2606.12736"},"sourceCount":3},{"slug":"weirdml","name":"WeirdML","primaryDomain":"Science & Reasoning","primaryDomainSlug":"science-reasoning","originalDomain":"ML research","tags":["ML research"],"type":"benchmark","whatItTests":"Unusual ML tasks designed to reward actual understanding over rote benchmark skill.","bestFor":"Anti-gaming ML reasoning","runnability":"partial","maturity":"current","access":"mixed","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"source":"https://www.haavardihlen.no/weirdml/"},"sourceCount":1},{"slug":"eva-bench","name":"EVA-Bench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Voice","tags":["Voice"],"type":"benchmark","whatItTests":"End-to-end voice-agent evaluation framework for realistic simulated conversations and voice-specific failure modes.","bestFor":"Voice-agent quality beyond transcripts","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2605.13841"},"sourceCount":1},{"slug":"mathvista","name":"MathVista","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Multimodal / Math","tags":["Multimodal","Math"],"type":"benchmark","whatItTests":"Mathematical reasoning over visual inputs.","bestFor":"Visual math reasoning","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://mathvista.github.io/"},"sourceCount":1},{"slug":"mm-browsecomp","name":"MM-BrowseComp","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Multimodal / Browser agents","tags":["Multimodal","Browser agents"],"type":"benchmark","whatItTests":"Emerging multimodal browsing benchmark for web tasks where visual context matters.","bestFor":"Multimodal browsing agents","runnability":"partial","maturity":"emerging","access":"mixed","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2508.13186"},"sourceCount":1},{"slug":"mmbench","name":"MMBench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Multimodal","tags":["Multimodal"],"type":"benchmark","whatItTests":"Broad multimodal model evaluation suite.","bestFor":"General VLM comparison","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/open-compass/MMBench"},"sourceCount":1},{"slug":"mmmu-mmmu-pro","name":"MMMU / MMMU-Pro","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Multimodal","tags":["Multimodal"],"type":"benchmark","whatItTests":"Expert multimodal reasoning where images materially affect the answer.","bestFor":"Vision-language reasoning","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/MMMU-Benchmark/MMMU","source":"https://mmmu-benchmark.github.io/","dataset":"https://huggingface.co/datasets/MMMU/MMMU_Pro","results":"https://huggingface.co/api/datasets/MMMU/MMMU_Pro/leaderboard"},"sourceCount":4},{"slug":"multivox","name":"MultiVox","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Multimodal voice assistants","tags":["voice","vision","paralinguistics","multimodal"],"type":"benchmark","whatItTests":"Voice assistants on spoken and visual cues including emotion, pitch, timbre, and ambient audio.","bestFor":"Evaluating omni assistants that must combine paralinguistic speech with images or video.","runnability":"partial","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2507.10859"},"sourceCount":1},{"slug":"open-asr-leaderboard","name":"Open ASR Leaderboard","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Automatic speech recognition","tags":["ASR","audio","speech","leaderboard"],"type":"scoreboard","whatItTests":"Speech-to-text systems across public ASR datasets and efficiency measures.","bestFor":"Shortlisting open speech-recognition models with source-linked results.","runnability":"hosted","maturity":"emerging","access":"hosted","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/hf-audio/open-asr-leaderboard","results":"https://huggingface.co/api/datasets/hf-audio/open-asr-leaderboard/leaderboard"},"sourceCount":2},{"slug":"pbench","name":"PBench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Referring-expression segmentation","tags":["vision","segmentation","grounding","multimodal"],"type":"benchmark","whatItTests":"Pixel-level visual grounding from referring expressions.","bestFor":"Specialist comparison of multimodal grounding and segmentation systems.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/tiiuae/PBench","results":"https://huggingface.co/api/datasets/tiiuae/PBench/leaderboard"},"sourceCount":2},{"slug":"screenspot-pro","name":"ScreenSpot-Pro","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Professional GUI grounding","tags":["GUI","vision","grounding","agents"],"type":"benchmark","whatItTests":"Visual grounding of interface elements in high-resolution professional software.","bestFor":"Choosing vision-language models for computer-use agents.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/likaixin/ScreenSpot-Pro","results":"https://huggingface.co/api/datasets/likaixin/ScreenSpot-Pro/leaderboard"},"sourceCount":2},{"slug":"vaani-benchmark","name":"Vaani Benchmark","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Hindi automatic speech recognition","tags":["ASR","Hindi","speech","multilingual"],"type":"benchmark","whatItTests":"Hindi speech recognition across real acoustic and language conditions.","bestFor":"Selecting ASR systems for Hindi-language products.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/ARTPARK-IISc/Vaani-Benchmark-V1.0","results":"https://huggingface.co/api/datasets/ARTPARK-IISc/Vaani-Benchmark-V1.0/leaderboard"},"sourceCount":2},{"slug":"video-mme","name":"Video-MME","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Video","tags":["Video"],"type":"benchmark","whatItTests":"Video understanding benchmark across temporal and multimodal questions.","bestFor":"Video model comparison","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"source":"https://video-mme.github.io/home_page.html","repo":"https://github.com/MME-Benchmarks/Video-MME","dataset":"https://huggingface.co/datasets/MME-Benchmarks/Video-MME-v2","results":"https://huggingface.co/api/datasets/MME-Benchmarks/Video-MME-v2/leaderboard"},"sourceCount":4},{"slug":"vlabench","name":"VLABench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Vision-language-action robotics","tags":["robotics","vision","action","embodied AI"],"type":"benchmark","whatItTests":"Vision-language-action systems on primitive robotic manipulation tasks.","bestFor":"Comparing embodied models on reproducible task primitives.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/VLABench/vlabench_primitive_ft_lerobot_video","results":"https://huggingface.co/api/datasets/VLABench/vlabench_primitive_ft_lerobot_video/leaderboard"},"sourceCount":2},{"slug":"vocalbench","name":"VocalBench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Voice","tags":["Voice"],"type":"benchmark","whatItTests":"Speech-interaction benchmark for vocal communication and multi-round voice tasks.","bestFor":"Speech interaction models","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"repo":"https://github.com/SJTU-OmniAgent/VocalBench"},"sourceCount":1},{"slug":"voiceagentbench","name":"VoiceAgentBench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Voice / Agents","tags":["Voice","Agents","agents"],"type":"agent benchmark","whatItTests":"Speech-based agentic tasks with spoken queries, tool/function specifications, multi-turn dialogue, and safety cases.","bestFor":"Voice agents with tools","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"watch","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/krutrim-ai-labs/VoiceAgentBench","paper":"https://arxiv.org/abs/2510.07978"},"sourceCount":2},{"slug":"voicebench","name":"VoiceBench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Voice","tags":["Voice"],"type":"benchmark","whatItTests":"LLM-based voice assistant benchmark across speech QA, reasoning, instruction following, safety, and robustness.","bestFor":"Voice assistant model comparison","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/MatthewCYM/VoiceBench","paper":"https://arxiv.org/abs/2410.17196"},"sourceCount":2},{"slug":"wbench","name":"WBench","primaryDomain":"Multimodal & Voice","primaryDomainSlug":"multimodal-voice","originalDomain":"Interactive video world models","tags":["video","world model","multimodal","interaction"],"type":"benchmark","whatItTests":"Interactive video world models across multiple behavioral dimensions and metrics.","bestFor":"Comparing action-conditioned world models rather than passive video QA.","runnability":"runnable","maturity":"emerging","access":"open","risk":"standard","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/meituan-longcat/WBench","results":"https://huggingface.co/api/datasets/meituan-longcat/WBench/leaderboard"},"sourceCount":2},{"slug":"agentharm","name":"AgentHarm","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Safety / Agents","tags":["Safety","Agents","agents"],"type":"agent benchmark","whatItTests":"Harmful multi-step agent tasks for evaluating agent safety under tool-use settings.","bestFor":"Agent safety research","runnability":"runnable","maturity":"current","access":"open","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"dataset":"https://huggingface.co/datasets/ai-safety-institute/AgentHarm","paper":"https://arxiv.org/abs/2410.09024"},"sourceCount":2},{"slug":"cyberseceval-purplellama","name":"CyberSecEval / PurpleLlama","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Security","tags":["Security"],"type":"benchmark","whatItTests":"Cybersecurity risk and capability evaluations for LLMs.","bestFor":"Defensive cyber-risk evaluation","runnability":"runnable","maturity":"current","access":"open","risk":"standard","editorialStatus":"recommended","lastVerified":"2026-07-15","links":{"repo":"https://github.com/meta-llama/PurpleLlama/tree/main/CybersecurityBenchmarks"},"sourceCount":1},{"slug":"evmbench","name":"EVMbench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Security","tags":["Security","agents"],"type":"agent benchmark","whatItTests":"Sandboxed smart-contract benchmark for detecting, patching, and exploiting EVM vulnerabilities.","bestFor":"Smart-contract security agents","runnability":"runnable","maturity":"current","access":"open","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"source":"https://openai.com/index/introducing-evmbench/","repo":"https://github.com/paradigmxyz/evmbench"},"sourceCount":2},{"slug":"exploitbench","name":"ExploitBench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Security","tags":["Security"],"type":"benchmark","whatItTests":"Capability ladder from vulnerability identification toward exploitation outcomes.","bestFor":"Cyber capability research with defensive caution","runnability":"partial","maturity":"current","access":"mixed","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"source":"https://exploitbench.com/"},"sourceCount":1},{"slug":"harmbench","name":"HarmBench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Safety","tags":["Safety"],"type":"benchmark","whatItTests":"Robust refusal and red-teaming benchmark for evaluating whether models comply with harmful requests.","bestFor":"Safety refusal and red-team evaluation","runnability":"runnable","maturity":"current","access":"open","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"source":"https://www.harmbench.org/","paper":"https://arxiv.org/abs/2402.04249"},"sourceCount":2},{"slug":"medsafetybench","name":"MedSafetyBench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Healthcare / Safety","tags":["Healthcare","Safety"],"type":"benchmark","whatItTests":"Medical safety benchmark for risky clinical responses and unsafe medical advice.","bestFor":"Medical safety evaluation","runnability":"runnable","maturity":"current","access":"open","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"repo":"https://github.com/AI4LIFE-GROUP/med-safety-bench","paper":"https://arxiv.org/abs/2403.03744"},"sourceCount":2},{"slug":"os-harm","name":"OS-Harm","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Safety / Computer use","tags":["Safety","Computer use","agents"],"type":"agent benchmark","whatItTests":"Safety benchmark for computer-use agents and harmful action sequences.","bestFor":"Computer-use safety","runnability":"runnable","maturity":"current","access":"open","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"repo":"https://github.com/tml-epfl/os-harm","paper":"https://arxiv.org/abs/2506.14866"},"sourceCount":2},{"slug":"safetybench","name":"SafetyBench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Safety","tags":["Safety"],"type":"benchmark","whatItTests":"Broad static safety benchmark for LLMs across safety categories.","bestFor":"Static safety baseline","runnability":"runnable","maturity":"legacy","access":"open","risk":"standard","editorialStatus":"reference","lastVerified":"2026-07-15","links":{"source":"https://github.com/thu-coai/SafetyBench"},"sourceCount":1},{"slug":"scirisk-bench","name":"SciRisk-Bench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"AI-for-science risk","tags":["safety","science","risk","2026"],"type":"benchmark","whatItTests":"AI-for-science safety across explicit risk dimensions and scientific disciplines.","bestFor":"Mapping scientific-agent failure modes to concrete risk categories.","runnability":"partial","maturity":"emerging","access":"open","risk":"hazardous","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"paper":"https://arxiv.org/abs/2606.18936"},"sourceCount":1},{"slug":"sec-bench","name":"SEC-bench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Security","tags":["Security","agents"],"type":"agent benchmark","whatItTests":"Software-security tasks including vulnerability discovery, patching, and proof-of-concept style scenarios.","bestFor":"Security engineering agents","runnability":"runnable","maturity":"current","access":"open","risk":"hazardous","editorialStatus":"restricted","lastVerified":"2026-07-15","links":{"source":"https://sec-bench.github.io/","repo":"https://github.com/SEC-bench/SEC-bench"},"sourceCount":2},{"slug":"sosbench","name":"SOSBench","primaryDomain":"Safety & Security","primaryDomainSlug":"safety-security","originalDomain":"Scientific misuse safety","tags":["safety","science","misuse","regulation"],"type":"benchmark","whatItTests":"Safety alignment on 3,000 regulation-grounded prompts across six high-risk scientific domains.","bestFor":"Testing refusal and policy behavior on knowledge-intensive scientific misuse.","runnability":"runnable","maturity":"emerging","access":"open","risk":"hazardous","editorialStatus":"specialized","lastVerified":"2026-07-15","links":{"site":"https://sosbench.github.io/","paper":"https://arxiv.org/abs/2505.21605"},"sourceCount":2}],"filters":{"domains":["General Models","Agents & Tool Use","Coding & SWE","Retrieval & RAG","Customer Support","Legal","Finance","Healthcare","Science & Reasoning","Multimodal & Voice","Safety & Security"],"types":["agent benchmark","benchmark","harness","scoreboard","toolkit"],"runnability":["runnable","partial","hosted"],"maturity":["current","emerging","legacy"],"access":["open","mixed","hosted","commercial"],"risk":["standard","hazardous"],"sourceTypes":["dataset","harness","hub","leaderboard","paper","post","repo","results","site","source","tau3"]},"changelog":[{"date":"2026-07-16","note":"Expanded the July 2026 census with the Hugging Face official benchmark registry and current agent, scientific, legal, voice, safety, and meta-evaluation releases; added a provenance-preserving score snapshot and evidence-weighted model chooser."},{"date":"2026-07-15","note":"Normalized domain, maturity, access, risk, and runnability fields; repaired confirmed dead links; added static rendering, category hubs, and stable exports."},{"date":"2026-06-07","note":"Published the initial benchmark atlas."}]}