{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_034f498c5b882d5e2ce4e9c4baeed9884326a9ea1e53a5ae6eb66ca07c136548","bitcoin_anchor":{"bitcoin_attestations":["bitcoin_block_949451"],"calendar_attestations":["https://finney.calendar.eternitywall.com","https://btc.calendar.catallaxy.com","https://alice.btc.calendar.opentimestamps.org","https://bob.btc.calendar.opentimestamps.org"],"ots_url":"/registry/data/substrate/anchors/77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3.ots","stamped_at":"2026-05-15T03:00:03Z","status":"bitcoin"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_034f498c5b882d5e2ce4e9c4baeed9884326a9ea1e53a5ae6eb66ca07c136548","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"a270b4230e6f8c0faec65f7645efb659355e48b7e2c74e9ec07de214f36df051","published":"Tue, 12 May 2026 00:00:00 -0400","receipt_hash":"a270b4230e6f8c0faec65f7645efb659355e48b7e2c74e9ec07de214f36df051","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"a270b4230e6f8c0faec65f7645efb659355e48b7e2c74e9ec07de214f36df051","observed_at":"2026-05-12T04:43:42.564879Z","parent_run_hash":"4cc5aca0c1b8116c9ab92405e0260204f01cf7ce2e49dea9d7e123236f5dc13b","published":"Tue, 12 May 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2605.08545v1 Announce Type: new \nAbstract: Agent benchmarks typically report only final outcomes: pass or fail. This threatens evaluation credibility in three ways. First, scores may be inflated or deflated by shortcuts and benchmark artifacts, misrepresenting capability. Second, benchmark performance may fail to predict real-world utility due to scaffold limitations and recurring failure modes. Finally, capability scores may conceal dangerous or catastrophic actions taken by the agent. We argue that log analysis -- the systematic tracking and analysis of the inputs, execution, and outputs of an AI agent -- is necessary to overcome these validity threats and promote credible agent evaluation. In this paper, we (1) present a taxonomy of threats to credible evaluation documented through log analysis, and (2) develop a set of guiding principles for log analysis. We illustrate these principles on tau-Bench Airline, revealing that pass^5 performance was under-elicited by nearly 50% an","title":"Log analysis is necessary for credible evaluation of AI agents","url":"https://arxiv.org/abs/2605.08545","vendor":"arxiv_cs_ai"},"summary":"arXiv:2605.08545v1 Announce Type: new \nAbstract: Agent benchmarks typically report only final outcomes: pass or fail. This threatens evaluation credibility in three ways. First, scores may be inflated or deflated by shortcuts and benchmark artifacts, misrepresenting capability. Second, benchmark performance may fail to predict real-world utility due to scaffold limitations and recurring failure modes. Finally, capability scores may conceal dangerous or catastrophic actions taken by the agent. We argue that log analysis -- the systematic tracking and analysis of the inputs, execution, and outputs of an AI agent -- is necessary to overcome these validity threats and promote credible agent evaluation. In this paper, we (1) present a taxonomy of threats to credible evaluation documented through log analysis, and (2) develop a set of guiding principles for log analysis. We illustrate these principles on tau-Bench Airline, revealing that pass^5 performance was under-elicited by nearly 50% an","title":"Log analysis is necessary for credible evaluation of AI agents","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-12T04:43:42Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2605.08545"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:874ec11e6d1e9771c7d3a0a21046ab8f375bf33845a66fef88437f584f10081c3079a1ac3b44e95b4d532a020770b3779472777d0f07b8bd4f9e21028adcb901","signer":"crovia.substrate","subject":{"observed_at":"2026-05-12T04:43:42Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2605.08545"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"2a626da198fe43adbc218ea4f2cf214b0bee0ffd374f538ec99d1624d375cf8d","leaf_index":128249,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"4ab4ba4f06a170d1c99d47436edbf61e8029477abccf2edba6b40d3ee7dfd026","side":"left"},{"sibling":"2fb006e1e674d52cb091e828c34124af4b319c74fdabbee9cfd7d9954f315adf","side":"right"},{"sibling":"ef2f42f8440adb1ae7778ff0841501aa11315ef7db2d7b30867be963024c13e5","side":"right"},{"sibling":"550c6998630b91323c089a86c74f1d48649fcca8677ae275d0e6ff048de3e4cd","side":"left"},{"sibling":"6b381cb2a3c8834a952eff43cbf3df42e2282329f60924ade717cafb32d13544","side":"left"},{"sibling":"61ab00acd0475fb6cc41ab2867bb1ccfce54d02d9797a12220f28abefe579e53","side":"left"},{"sibling":"bec646946f214a143c9e93486ba236d585a2f0b43a53baac977c9f2944559842","side":"left"},{"sibling":"0dd5042df5f6760dec082d5c0a4f27a8abafc39ffbdb96ab323bd9dc341376bb","side":"left"},{"sibling":"42655d56270deb2082e77e49993b93afa49d6105df9975a9c20cc7a048dd54e7","side":"right"},{"sibling":"077005bbaed7b42537914de0b512ed52d33243215eab288181aba807d37bf04e","side":"right"},{"sibling":"c6eaf7a4fcab2db96e9e9423acb6922c80f64882d0f3f50d09e53a4807d23084","side":"left"},{"sibling":"df12eaabc0a370aff5d0488478f48d2b3f90d025c903643f98ea804b017688ec","side":"right"},{"sibling":"ffc4d51379293bc3e1910c7d612f409dc610fd9acf8241793fb89f82e1bad4ef","side":"left"},{"sibling":"62ac6554017807bd83187f5a3e5f4f72d6c482616429c2780e9fff1f4845fa04","side":"left"},{"sibling":"3a5e69cf0803f4c91f3895ed7c9a95748fef240bec4422e167c05300f79f06c0","side":"left"},{"sibling":"f2817ab288b5324fe49770372c7a10f33f7cd11005f8d4c0a730316f5229dc98","side":"left"},{"sibling":"725fac972e772ca0dc598810ea1abc70df472f72d2d6ab8a0baee2b80e5d2f4c","side":"left"},{"sibling":"98fc57dfef8873b512edc8340f7181df57302bb96777625e072235c62d7c5895","side":"right"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":134292,"merkle_root":"77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260515T023701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-15T02:37:25Z","sig_algorithm":"ed25519","signature":"68107a834b00b24f5d4501e5ec727445311f132a486567ecc4c72a4e6dff24c8c21f2de3105293353ba5fdbe370d032819af6aa70f694e2e39b6af6737507009","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_034f498c5b882d5e2ce4e9c4baeed9884326a9ea1e53a5ae6eb66ca07c136548"}}