{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_e52c388b0ab24861fb3df249bef4af0de2164397c7e4dc5fc95cfc70fffa6133","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_e52c388b0ab24861fb3df249bef4af0de2164397c7e4dc5fc95cfc70fffa6133","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"c97dfa4211ef41a74114b257e0da38fb647079e0259da3c59eded2891b43ed94","published":"Tue, 26 May 2026 00:00:00 -0400","receipt_hash":"c97dfa4211ef41a74114b257e0da38fb647079e0259da3c59eded2891b43ed94","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"c97dfa4211ef41a74114b257e0da38fb647079e0259da3c59eded2891b43ed94","observed_at":"2026-05-26T04:43:39.018238Z","parent_run_hash":"dca8dedd754ad6a1772113d6b97ee4f4ab9a0afbdeb44aaace5ff2d2446b164b","published":"Tue, 26 May 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2605.24213v1 Announce Type: cross \nAbstract: Evaluation harnesses are software systems that orchestrate model evaluation by managing model invocation, data loading, metric computation, and result reporting. Despite their critical role in machine learning infrastructure, their operational challenges and engineering concerns have received limited attention so far. We present an empirical study of 57 evaluation harnesses, deriving a five-stage harness model and classifying 16,560 issues by workflow stage and root cause. Most harness operational challenges concentrate in the Specification stage (41.4% of issues), where harnesses integrate external models, datasets, and scoring judges. The three most frequent root causes of operational challenges are unimplemented features (24.3%), documentation gaps (20.3%), and missing input validation (17.2%), which together account for 61.7% of classified issues, spanning both defects in existing functionality and capability gaps that block intend","title":"Towards Evaluation Engineering: An Empirical Study of ML Evaluation Harnesses in the Wild","url":"https://arxiv.org/abs/2605.24213","vendor":"arxiv_cs_ai"},"summary":"arXiv:2605.24213v1 Announce Type: cross \nAbstract: Evaluation harnesses are software systems that orchestrate model evaluation by managing model invocation, data loading, metric computation, and result reporting. Despite their critical role in machine learning infrastructure, their operational challenges and engineering concerns have received limited attention so far. We present an empirical study of 57 evaluation harnesses, deriving a five-stage harness model and classifying 16,560 issues by workflow stage and root cause. Most harness operational challenges concentrate in the Specification stage (41.4% of issues), where harnesses integrate external models, datasets, and scoring judges. The three most frequent root causes of operational challenges are unimplemented features (24.3%), documentation gaps (20.3%), and missing input validation (17.2%), which together account for 61.7% of classified issues, spanning both defects in existing functionality and capability gaps that block intend","title":"Towards Evaluation Engineering: An Empirical Study of ML Evaluation Harnesses in the Wild","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-26T04:43:39Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2605.24213"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:4ae04809b9f95562922aa78409ede97d90bda57564974f0985d7a20cf98d4c9bb4a39b3f1690299c8c79ad118684e714a9a4665b2c7624c7bf833d7c1a992709","signer":"crovia.substrate","subject":{"observed_at":"2026-05-26T04:43:39Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2605.24213"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"b4256f5753eed0b9fda837841dc8cfa180f38c67b1ce3ad92d632b2d5e504ab8","leaf_index":151523,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"ef3f22ab92d97f2fee32eb2b388076384e9eee0faa3e2e0d84221a03528df103","side":"left"},{"sibling":"42a77674e95a3cde7601ee1ced122d0bd657e9c2363f7032a2d27ffcead3ef4a","side":"left"},{"sibling":"aaef504788e1cca13db861e1c4126897121b80fd56b46498f919edcb67eb4f44","side":"right"},{"sibling":"ab317fe326a15c76b72ac84dcf77f9698780272af9190b98c2a0fc35b5780292","side":"right"},{"sibling":"e0e00004e928d0132d7a18b1bb364d5aa529ef5a9816ed7ecd0e6c7f11f7b238","side":"right"},{"sibling":"e4c51922cc3a9dfa8ffcb4eac4ab29c5372c2bd53686548afdf9c6203620dbfe","side":"left"},{"sibling":"aca2ea5485625e14e9749df76cdae0268d37544ceeea42aec677e7b6cb11725a","side":"left"},{"sibling":"12607c0dd56c2db51bf46dd95ab0fe795f3f832d068cd7268fed3aeeb8a7cc29","side":"left"},{"sibling":"a808644c9a09dd75253c6c0ba9275dc94a6b6451ad5184ebdafda4a2722b36a5","side":"left"},{"sibling":"fdd22ebd87e5ba37d1153e47753a63818abb25df8fe45dbf735ba5c512c3355b","side":"left"},{"sibling":"cf593c482d17d202b94914915d0c65fb8713053548db64771d0ec5dbb404a07f","side":"left"},{"sibling":"134949308b15cffd6792ee2cf678119af34d69a64764d7c89cd47573c94e1cda","side":"left"},{"sibling":"e20a7391fed5b5f3b675af68344a3f5b050d6701e127b7db010af3941e59dfdb","side":"right"},{"sibling":"e5893793e3591ed7f5e58ca94ffcfba46bb30f69fb1c25d5ba8ef49eb99f9126","side":"right"},{"sibling":"35ca36cee447f0ef7064a25d55f59357c66901e31427729c4c1d8b14aa8adb6c","side":"left"},{"sibling":"e3eecaf996dbe7229a7bb1d234c97aea97a252c5f7c89f8547b6d091db0f0e40","side":"right"},{"sibling":"55bcbd4da3e20d93931f7e58673f10232e81a5b1514d7396cb4b71e8f95788d0","side":"right"},{"sibling":"d841ad93efda0869e5eb97678f348f03f5caab4353e05ff4bf18f47fb945b822","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":152106,"merkle_root":"ac5182c6f3dd09931f2a689df5f4be36df7b55e55bcf106e195671f5ed55fd8f","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260526T053701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-26T05:37:34Z","sig_algorithm":"ed25519","signature":"a401243fbd2c077d29a623c6ef616c78fbd5c8ce1930afa7af7165b2386e5a8cf15d5094983a1c962e71b27b911a3f3ce09c8cfab9393be2ce5ce8ee6513da06","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_e52c388b0ab24861fb3df249bef4af0de2164397c7e4dc5fc95cfc70fffa6133"}}