{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_9e70e7389d392b9c7dd2cb5f52c439057c96d89cf8ed2f7f022c1c814d6c433d","bitcoin_anchor":{"bitcoin_attestations":["bitcoin_block_949451"],"calendar_attestations":["https://finney.calendar.eternitywall.com","https://btc.calendar.catallaxy.com","https://alice.btc.calendar.opentimestamps.org","https://bob.btc.calendar.opentimestamps.org"],"ots_url":"/registry/data/substrate/anchors/77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3.ots","stamped_at":"2026-05-15T03:00:03Z","status":"bitcoin"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_9e70e7389d392b9c7dd2cb5f52c439057c96d89cf8ed2f7f022c1c814d6c433d","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"a68b276480da2ad161cd8039b378b60bb3bb135175d3a3ae2027708e7ca45e5d","published":"Wed, 06 May 2026 00:00:00 -0400","receipt_hash":"a68b276480da2ad161cd8039b378b60bb3bb135175d3a3ae2027708e7ca45e5d","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"a68b276480da2ad161cd8039b378b60bb3bb135175d3a3ae2027708e7ca45e5d","observed_at":"2026-05-06T04:43:18.518819Z","parent_run_hash":"184209fc0f2ebdc4b721e1a74f74ab17637dab3344682b6e6bd1f3e5e8f2cd00","published":"Wed, 06 May 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2605.02122v1 Announce Type: cross \nAbstract: Human evaluation remains the primary standard for assessing modern AI systems, yet annotator disagreement, bias, and variability make system rankings fragile under standard majority vote aggregation. Majority vote discards annotator reliability and item-level ambiguity, often yielding unstable comparisons across annotator subsets. We introduce STABLEVAL, a disagreement-aware evaluation framework that models latent item correctness and annotator-specific confusion patterns to produce posterior expected item credit and calibrated agent-level scores. Unlike label-denoising approaches such as Dawid-Skene, STABLEVAL is explicitly designed for stable and uncertainty-aware system evaluation rather than hard label recovery. We formalize ranking stability as a first-class evaluation objective and analyze how aggregation methods preserve or distort underlying annotator behavior. Across controlled synthetic experiments and multiple real-world hum","title":"STABLEVAL: Disagreement-Aware and Stable Evaluation of AI Systems","url":"https://arxiv.org/abs/2605.02122","vendor":"arxiv_cs_ai"},"summary":"arXiv:2605.02122v1 Announce Type: cross \nAbstract: Human evaluation remains the primary standard for assessing modern AI systems, yet annotator disagreement, bias, and variability make system rankings fragile under standard majority vote aggregation. Majority vote discards annotator reliability and item-level ambiguity, often yielding unstable comparisons across annotator subsets. We introduce STABLEVAL, a disagreement-aware evaluation framework that models latent item correctness and annotator-specific confusion patterns to produce posterior expected item credit and calibrated agent-level scores. Unlike label-denoising approaches such as Dawid-Skene, STABLEVAL is explicitly designed for stable and uncertainty-aware system evaluation rather than hard label recovery. We formalize ranking stability as a first-class evaluation objective and analyze how aggregation methods preserve or distort underlying annotator behavior. Across controlled synthetic experiments and multiple real-world hum","title":"STABLEVAL: Disagreement-Aware and Stable Evaluation of AI Systems","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-06T04:43:18Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2605.02122"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:595051f507382c99214d73222352101bdd60b0462077efa03882567fd67f09bc4a13a16944e691b08d877d37c368a803ff5f97918c1d145f7d20b012d9d2e809","signer":"crovia.substrate","subject":{"observed_at":"2026-05-06T04:43:18Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2605.02122"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"c7e4938c58530584abd0c72ea770a9244fc2fcdca622889dd86c4dfa734729ee","leaf_index":116325,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"f0353615b9ff4b4e0f4bd935424b3d270aaa12567a1b664a7bc02deb9175e220","side":"left"},{"sibling":"5accf17568daa52953554d3f95f927717bb5aef2fcb1e291053e0ecd0d9ba660","side":"right"},{"sibling":"f1d9d7fc9d28cf782ac6366681f5ab86de2ae3eb5b9de3c622b2ee01f7a73644","side":"left"},{"sibling":"1cf38882146a20ade47cdcb680a978bf3b1e09e3825eb82683f9685f9e0975c2","side":"right"},{"sibling":"f4afa9c4fdf9512b81011abff155f8d745d738055216c6270357a177927159b6","side":"right"},{"sibling":"1520586bfd4793dc79a128f0850786d392c8af1a7e3e5f9fdeeabdde90ee49b9","side":"left"},{"sibling":"ad3a284c05693695f2eb1381482f2a0de6f97623fdcf5a64fcab04a5e47aac4a","side":"left"},{"sibling":"29a1d68ae3b47209a045f8f53476ebb8528406538f56ab779e7eb4bfc2794b92","side":"right"},{"sibling":"7a43f13f6340a269b939be4b6e73a199a4e8061f0cf9dbb9ba629243174142d3","side":"right"},{"sibling":"c750384c973eaf46aa237cdabe7a75729862dd4101dca10cb8e0626e0ec2d34a","side":"left"},{"sibling":"ddc060ac400459417799f688c75d5b271636afa40d89ec7fc98652473a8061f6","side":"left"},{"sibling":"282afa51266e47629e34d808a360bbb276348bcc5a24bdcc93e83a76e580293e","side":"right"},{"sibling":"05f89b32c00462e60adf95c1fe4579cdc2791b36e8b17573d8f3b5fd5da95a0b","side":"right"},{"sibling":"8ccd9937a2c0d5c04044d07d1557791b7d07bb31eac41a39a675608d44b38f23","side":"right"},{"sibling":"3a5e69cf0803f4c91f3895ed7c9a95748fef240bec4422e167c05300f79f06c0","side":"left"},{"sibling":"f2817ab288b5324fe49770372c7a10f33f7cd11005f8d4c0a730316f5229dc98","side":"left"},{"sibling":"725fac972e772ca0dc598810ea1abc70df472f72d2d6ab8a0baee2b80e5d2f4c","side":"left"},{"sibling":"98fc57dfef8873b512edc8340f7181df57302bb96777625e072235c62d7c5895","side":"right"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":134292,"merkle_root":"77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260515T023701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-15T02:37:25Z","sig_algorithm":"ed25519","signature":"68107a834b00b24f5d4501e5ec727445311f132a486567ecc4c72a4e6dff24c8c21f2de3105293353ba5fdbe370d032819af6aa70f694e2e39b6af6737507009","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_9e70e7389d392b9c7dd2cb5f52c439057c96d89cf8ed2f7f022c1c814d6c433d"}}