{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_58c7c16842d718aa21a5f1ef0e9bf4e4117c6fbc427e2c30fa17adafbc52f5d7","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_58c7c16842d718aa21a5f1ef0e9bf4e4117c6fbc427e2c30fa17adafbc52f5d7","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"3a17aa0b2e15f30df23fcf7b966eb19a61226ca7f072fd863a99678e4575aa23","published":"Wed, 15 Jul 2026 00:00:00 -0400","receipt_hash":"3a17aa0b2e15f30df23fcf7b966eb19a61226ca7f072fd863a99678e4575aa23","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"3a17aa0b2e15f30df23fcf7b966eb19a61226ca7f072fd863a99678e4575aa23","observed_at":"2026-07-15T04:44:03.592429Z","parent_run_hash":"d49a6cf532e74153266f377b7760fc948d950d80ed41fc3e3eb82b58f5597ead","published":"Wed, 15 Jul 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2607.12338v1 Announce Type: new \nAbstract: Agent benchmarks often compare two agents after all tasks have run, but costly evaluations make partial runs tempting. A task fraction alone does not show whether a partial run supports the same pairwise conclusion as the completed benchmark. We study this question by replaying completed public task-level records from SWE-bench, AppWorld, and tau-bench. A partial budget counts as enough only when it supports the completed benchmark's decision, covers required task groups, and leaves no more than a target fraction of comparisons unresolved. The required task fraction varies sharply. At the strict 0 percentage point threshold on a 5 percentage point budget grid, AppWorld first meets all targets at 15 percent, tau-bench at 25 percent, and SWE-bench Verified at 90 percent; SWE-bench Lite does not meet all targets by 95 percent under the primary coverage rule. Partial-evaluation reports should state how much one agent must outperform another,","title":"How Many Tasks Are Enough for Agent Benchmark Decisions? A Replay Analysis of Public LLM Agent Benchmarks","url":"https://arxiv.org/abs/2607.12338","vendor":"arxiv_cs_ai"},"summary":"arXiv:2607.12338v1 Announce Type: new \nAbstract: Agent benchmarks often compare two agents after all tasks have run, but costly evaluations make partial runs tempting. A task fraction alone does not show whether a partial run supports the same pairwise conclusion as the completed benchmark. We study this question by replaying completed public task-level records from SWE-bench, AppWorld, and tau-bench. A partial budget counts as enough only when it supports the completed benchmark's decision, covers required task groups, and leaves no more than a target fraction of comparisons unresolved. The required task fraction varies sharply. At the strict 0 percentage point threshold on a 5 percentage point budget grid, AppWorld first meets all targets at 15 percent, tau-bench at 25 percent, and SWE-bench Verified at 90 percent; SWE-bench Lite does not meet all targets by 95 percent under the primary coverage rule. Partial-evaluation reports should state how much one agent must outperform another,","title":"How Many Tasks Are Enough for Agent Benchmark Decisions? A Replay Analysis of Public LLM Agent Benchmarks","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-07-15T04:44:03Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2607.12338"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:9ff5d3f8c75aaf96bdfd7a72ce3f6929fe464f0e7c770aed11afc78437c2e004640759a668b62fe3b2a17bb0b27a4974802edc8f30d3f976e53ca071cfaf750a","signer":"crovia.substrate","subject":{"observed_at":"2026-07-15T04:44:03Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2607.12338"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"87b44d28f971f0a148b217508b55574090098966ca7170e5950b05ab9531bf8c","leaf_index":316374,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"a26da677dd74330bdb6572e17f0b81b6e6ff7681312aa65a789b12e2b2eb8d28","side":"right"},{"sibling":"a65f06e0ab06f2b5c0af8a34f14ada4370437805bbef1f0227caf0512218a58b","side":"left"},{"sibling":"0a386129ce191922ed1077db052bcf03e8a9b41c325f2ff6e93f1464c2989796","side":"left"},{"sibling":"f766ebcb9905b696f8cd0aeb3ad250c115069fd01c653027892931b29dbc203b","side":"right"},{"sibling":"afc9249eaf545ce549c76a86654c30dc381dc3a70f8b30b7d724127926786985","side":"left"},{"sibling":"2e6e43c7a0f7f6499621f0f438d8189318d67ceb374377041278440cc38473f6","side":"right"},{"sibling":"8dc6b151fc86f1b43f490c2263dd78189a9778cebe1755f2bcd01eb2c6239831","side":"left"},{"sibling":"9a178a31a5d97bdb1bf054b31ec2fcc6cf9b633c6d3f460e934451a35bac9c25","side":"left"},{"sibling":"31b0ff6eae164fcff8882af2b581471a08fffa2373505e1f33f01e361c94a42f","side":"left"},{"sibling":"e1ebad13fafe4a9a53c35e1f19fa67a19c940f70a4211d1fab42cf49be486a3c","side":"left"},{"sibling":"c265283bb70fb86740bdfa059cfe36cfea4a5f903b841822b2954814237de972","side":"right"},{"sibling":"0cb62c0ada57a2406a6bcb100889d3e8b29a15efeed07adaff5bb90a5e80612a","side":"right"},{"sibling":"0b69289b25462ddd6166f6f49004cfc8ada0ab4f10adafe188347817bdd46e37","side":"left"},{"sibling":"1418b281cd985b5ed411ef25f2017a1826cc14919b6fad3934e6ceeec693699b","side":"right"},{"sibling":"f302542c38ba7c3aab7c9280dd60259ecec777dca6e6f71b6f0729b0b8791b72","side":"left"},{"sibling":"d8b9143917b539c543cf4448cec00131f8b807bd8004979c54ebe09798748c66","side":"left"},{"sibling":"abe4a8c706e530484d1e96a8988cb09eab85928b2050985500ab289753fe3eec","side":"right"},{"sibling":"f436dccf82aa2c1eb7bfa3eb84316e116aaf64dc55cd9592597118f6cb0648f6","side":"right"},{"sibling":"1cecb7f447febd025aac272837c80de218aecc6485d2395a509b2a1f1b9c746e","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":316730,"merkle_root":"a8e6e5be81ea6f5b5f2227422459bf39455fe9f0b6602b4d1ce6977dbfd78bc7","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260715T053701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-07-15T05:38:24Z","sig_algorithm":"ed25519","signature":"df1678d268b5a07413e2ca6e748c3f40b6cfedea930a18d843489a4ab513da791bf0a886caab1b918d0989f8ebaaf3d0035ca2aa777913b1ad29979f1deb9a0c","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_58c7c16842d718aa21a5f1ef0e9bf4e4117c6fbc427e2c30fa17adafbc52f5d7"}}