{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_f10ee38074689a706520c7aaf20da057b59b63254b27973b4d193d402b82c302","bitcoin_anchor":{"bitcoin_attestations":["bitcoin_block_949451"],"calendar_attestations":["https://finney.calendar.eternitywall.com","https://btc.calendar.catallaxy.com","https://alice.btc.calendar.opentimestamps.org","https://bob.btc.calendar.opentimestamps.org"],"ots_url":"/registry/data/substrate/anchors/77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3.ots","stamped_at":"2026-05-15T03:00:03Z","status":"bitcoin"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_arxiv_retraction_v1","root_at_anchor":"spider_arxiv_retraction_v1"}],"axiom_id":"axm_f10ee38074689a706520c7aaf20da057b59b63254b27973b4d193d402b82c302","axiom_type":"AX.OBS","body":{"axiom_subtype":"research.arxiv_retraction.v1","category":"research","fingerprint":"2e9d557c9dc613a63065947d187eecb3cbe9a0ce95ce2426eb9b985c8256f7bf","published":"2025-04-29T15:48:49Z","receipt_hash":"2e9d557c9dc613a63065947d187eecb3cbe9a0ce95ce2426eb9b985c8256f7bf","schema":"spider.research.arxiv_retraction.v1","spider":"arxiv_retraction","spider_record":{"axiom_subtype":"research.arxiv_retraction.v1","category":"research","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"2e9d557c9dc613a63065947d187eecb3cbe9a0ce95ce2426eb9b985c8256f7bf","observed_at":"2026-05-03T16:20:09.776706Z","parent_run_hash":null,"published":"2025-04-29T15:48:49Z","runtime_version":"0.1.0","schema":"spider.research.arxiv_retraction.v1","source_status":200,"source_url":"http://export.arxiv.org/api/query?search_query=cat:cs.AI+AND+%28abs:withdrawn+OR+abs:retracted%29&max_results=20&sortBy=submittedDate&sortOrder=descending","spider":"arxiv_retraction","summary_excerpt":"Measuring progress is fundamental to the advancement of any scientific field. As benchmarks play an increasingly central role, they also grow more susceptible to distortion. Chatbot Arena has emerged as the go-to leaderboard for ranking the most capable AI systems. Yet, in this work we identify systematic issues that have resulted in a distorted playing field. We find that undisclosed private testing practices benefit a handful of providers who are able to test multiple variants before public release and retract scores if desired. We establish that the ability of these providers to choose the best score leads to biased Arena scores due to selective disclosure of performance results. At an extreme, we identify 27 private LLM variants tested by Meta in the lead-up to the Llama-4 release. We also establish that proprietary closed models are sampled at higher rates (number of battles) and have fewer models removed from the arena than open-weight and open-source alternatives. Both these pol","title":"The Leaderboard Illusion","url":"https://arxiv.org/pdf/2504.20879v2","vendor":"arxiv"},"summary":"Measuring progress is fundamental to the advancement of any scientific field. As benchmarks play an increasingly central role, they also grow more susceptible to distortion. Chatbot Arena has emerged as the go-to leaderboard for ranking the most capable AI systems. Yet, in this work we identify systematic issues that have resulted in a distorted playing field. We find that undisclosed private testing practices benefit a handful of providers who are able to test multiple variants before public release and retract scores if desired. We establish that the ability of these providers to choose the best score leads to biased Arena scores due to selective disclosure of performance results. At an extreme, we identify 27 private LLM variants tested by Meta in the lead-up to the Llama-4 release. We also establish that proprietary closed models are sampled at higher rates (number of battles) and have fewer models removed from the arena than open-weight and open-source alternatives. Both these pol","title":"The Leaderboard Illusion","vendor":"arxiv"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-03T16:20:09Z","notes":"Spider arxiv_retraction (research) research.arxiv_retraction.v1","object":{"captured_by":"crovia.spider.arxiv_retraction","primary_source_url":"https://arxiv.org/pdf/2504.20879v2"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:debe5424faac104f90c8f3deb17c01bbb3accc8f03f166cdf3d2c0f960dd3fe7973d3aa34ccaa501c5b35a7159b0aa9cf1170c18bd63eb90485bb8aaec598e04","signer":"crovia.substrate","subject":{"observed_at":"2026-05-03T16:20:09Z","source_collector":"spider:arxiv_retraction","target_id":"https://arxiv.org/pdf/2504.20879v2"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/research/arxiv_retraction_v1.jsonl\",\"source_seal_merkle_root\":\"spider_arxiv_retraction_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"1d6279babfc2b25467a8e8dec66c4eaed9ca41fa0b65111fb3720984d648ebc5","leaf_index":111928,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"81d844f08629f09e6e0ae68cc90e5ea05df9a5256cc456aaca4fc5d5b9976683","side":"right"},{"sibling":"9f2af3441395ca979cd6a2a66036a10935fbac2176c6a0f58b78e2abaf11809b","side":"right"},{"sibling":"40c8247f052e40391e6d0cda65135d0da9c1a87f217aa8933d275768aaacb7e0","side":"right"},{"sibling":"1fe18d6e1f125ddc853ec1ff04103ac3853d57508235b90088c539ca2ae65408","side":"left"},{"sibling":"b8d0b4c7a94732eaec522fc5042dc6d11f071f5280501aa7f3c5c58084a0d4b9","side":"left"},{"sibling":"787e416b9f66536a62992df40e090f8c704839faf8301403b2d1bc76052ed5c5","side":"left"},{"sibling":"19b85bdc8998a840b5999169743b627ed26d5e26bd05a788142a0e4fbfe2a2dc","side":"right"},{"sibling":"0994d51d2178913b0e3ac7346ceed16c5ea436d841a8e7f28a99362fa269bad0","side":"right"},{"sibling":"015d2ce3c58c7a790ad078159267c63cd54f9ae7e20530115a6a423419cefe22","side":"left"},{"sibling":"5bcdb02d4310fc8bfd791789d990714b0a10f44358ffa98c446f1755352c49b8","side":"right"},{"sibling":"cf8897e1feec248f5d05e2cafec4f8c2739e2005227d28b5ccc8cfa11ed66ee4","side":"left"},{"sibling":"a6d6e5bc888edc394757fbba02b675bac6ad0831e36a6f5ebbaa4e5548f25603","side":"right"},{"sibling":"76d7c4385d71ae8104107105be66eddb792e01ed6e493db5a9b4eec5441abb76","side":"left"},{"sibling":"b03f005860bf95148ea89c703a32ca19ab7a2ca71b58bb628ff94a9edb8703b0","side":"left"},{"sibling":"21d1e30b556f553dc939fddc23e6367f0a7755ebe3dd489dc0001cef017beb7f","side":"right"},{"sibling":"f2817ab288b5324fe49770372c7a10f33f7cd11005f8d4c0a730316f5229dc98","side":"left"},{"sibling":"725fac972e772ca0dc598810ea1abc70df472f72d2d6ab8a0baee2b80e5d2f4c","side":"left"},{"sibling":"98fc57dfef8873b512edc8340f7181df57302bb96777625e072235c62d7c5895","side":"right"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":134292,"merkle_root":"77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260515T023701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-15T02:37:25Z","sig_algorithm":"ed25519","signature":"68107a834b00b24f5d4501e5ec727445311f132a486567ecc4c72a4e6dff24c8c21f2de3105293353ba5fdbe370d032819af6aa70f694e2e39b6af6737507009","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_f10ee38074689a706520c7aaf20da057b59b63254b27973b4d193d402b82c302"}}