{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_d74bbe5c2e2da0c77ea63c114d6e9b16f3409a69b8c014e1f922ca81f275e5df","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_d74bbe5c2e2da0c77ea63c114d6e9b16f3409a69b8c014e1f922ca81f275e5df","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"2f4d4827f9dc7de1702af2092ab2e69233fb64d1c0d6836cf4aca8a80f44fb5e","published":"Mon, 18 May 2026 00:00:00 -0400","receipt_hash":"2f4d4827f9dc7de1702af2092ab2e69233fb64d1c0d6836cf4aca8a80f44fb5e","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"2f4d4827f9dc7de1702af2092ab2e69233fb64d1c0d6836cf4aca8a80f44fb5e","observed_at":"2026-05-18T04:43:11.219741Z","parent_run_hash":"a8aad7414ebb6b75c726f09cd673410576a7f87e191fbdb9ddac99e9b2b95a05","published":"Mon, 18 May 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2603.04459v3 Announce Type: replace-cross \nAbstract: The rapid expansion of research in LLM safety presents challenges in tracking advancements, making benchmarks important evaluation infrastructures for identifying key trends and facilitating systematic comparisons. Yet no systematic assessment exists of their code quality and runnability, nor of what factors are associated with the community's adoption of certain benchmarks over others. To address this gap, we conduct a systematic measurement study of 31 LLM safety benchmarks (covering prompt injection, jailbreak, and hallucination) with 382 non-benchmark papers as a control group, combining automated static analysis, human runnability testing (220+ person-hours), and bibliometric analysis. We find that only 39\\% of benchmark repositories can run without modification, only 16\\% provide flawless installation guides, and a mere 6\\% include ethical considerations despite containing potentially harmful content. These deficiencies p","title":"Benchmark of Benchmarks: Unpacking Influence and Code Repository Quality in LLM Safety Benchmarks","url":"https://arxiv.org/abs/2603.04459","vendor":"arxiv_cs_ai"},"summary":"arXiv:2603.04459v3 Announce Type: replace-cross \nAbstract: The rapid expansion of research in LLM safety presents challenges in tracking advancements, making benchmarks important evaluation infrastructures for identifying key trends and facilitating systematic comparisons. Yet no systematic assessment exists of their code quality and runnability, nor of what factors are associated with the community's adoption of certain benchmarks over others. To address this gap, we conduct a systematic measurement study of 31 LLM safety benchmarks (covering prompt injection, jailbreak, and hallucination) with 382 non-benchmark papers as a control group, combining automated static analysis, human runnability testing (220+ person-hours), and bibliometric analysis. We find that only 39\\% of benchmark repositories can run without modification, only 16\\% provide flawless installation guides, and a mere 6\\% include ethical considerations despite containing potentially harmful content. These deficiencies p","title":"Benchmark of Benchmarks: Unpacking Influence and Code Repository Quality in LLM Safety Benchmarks","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-18T04:43:11Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2603.04459"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:a9ec76f652d0c93b22be79161afdf4b92cd41c49249896dccd88686251aeacb591f1535f8d20316c731c31866ed5794b11a8b332d9ea38adba728e1ab2f03009","signer":"crovia.substrate","subject":{"observed_at":"2026-05-18T04:43:11Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2603.04459"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"dab866eba4d4e7b5a5bf7cdbc876a1d628aeabcc8141198937f9270edb33d325","leaf_index":140794,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"5b8a58e6d4a07d4867ff8d3996302aef30d5c76f93367317f87fc842c08dbd89","side":"right"},{"sibling":"41450ff2dea2f2156e8f831e127a8863e7b11f8e6627659f502d712c663eb350","side":"left"},{"sibling":"f2a5de79109956f5e4a72aa001b28e02e6e9b675ae437cb539f42e22c098f5ed","side":"right"},{"sibling":"b7433287ed8ad52435e88ecdb27dd0e195b793d536a19824622780e066834e2f","side":"left"},{"sibling":"1030005f0dffd7184c294c1675687c2f7200ac641111290389abd489bb5ac4af","side":"left"},{"sibling":"b59f5b6589804050df943e9de027b74c1fffb33440b7e6dfcf198d1d29863180","side":"left"},{"sibling":"518d2c79e4bbff1e18ea05bbb3663a3fb58b3b1682a658f34e706ea23e091089","side":"left"},{"sibling":"30c60828b6b0ade79197e585b88c06ccf4f348c1e62fbf6710d89ae2c2e9dcfb","side":"left"},{"sibling":"6bedf73520cf3dd8758d8bdedf3be245de9aea97abd42934aae25539176ae1b2","side":"left"},{"sibling":"07abc3bad689e74e6304772503dc9372a118e6f66883b8e88c43414efddac063","side":"right"},{"sibling":"28b78fb112bcf26b6801664db97eb8f52a9bccbf0a7ae6766e11845d443692df","side":"left"},{"sibling":"68d0a4634c1460a19c92edd9480df3aa733b814463e7420d1e14471bf61b2f83","side":"right"},{"sibling":"8af64f275b862349aa3bbb9d5cd7fa9a7fdd5620af3bf1b36b2a4519b0b53bdf","side":"right"},{"sibling":"8f4c0fbe56b6c010fbb8c782ebcd478079bb3f991d8704e2534209a075d9163c","side":"left"},{"sibling":"b98c2afadb358e5387e88f19588f8343a81b488d9b44a6f7e57a032db3a1b030","side":"right"},{"sibling":"11b0c1591747f09f7c8971a6caa19befcd81317ca9dfd417b143234df4e10c79","side":"right"},{"sibling":"87206f3bcc342797c990d87f7235c01f78d32ca59cfaf8ad18d71afc879ba477","side":"right"},{"sibling":"d841ad93efda0869e5eb97678f348f03f5caab4353e05ff4bf18f47fb945b822","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":140892,"merkle_root":"6cca56ead155990456b8a014cc50bddbe710f409b26e3d1bfa6fb12b0bfcf6bf","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260518T053701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-18T05:37:30Z","sig_algorithm":"ed25519","signature":"1e1135f7595f79b14fb11f5fa81a2e17ad31b11b44b427a5e40a7d511cd86447daf492babd368ab571cf26404c8c74c450d460130fca4b064eb2760367489a0f","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_d74bbe5c2e2da0c77ea63c114d6e9b16f3409a69b8c014e1f922ca81f275e5df"}}