{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_2c945da2f7efdb6eeacec6bc53622b9efe40235cdcc2ea0100057dee7a47a802","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_2c945da2f7efdb6eeacec6bc53622b9efe40235cdcc2ea0100057dee7a47a802","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"e3da0f08539eb0096dbd3a63ac3bfdad8dea8aca8df9c1e321be5773e2dc1661","published":"Tue, 09 Jun 2026 00:00:00 -0400","receipt_hash":"e3da0f08539eb0096dbd3a63ac3bfdad8dea8aca8df9c1e321be5773e2dc1661","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"e3da0f08539eb0096dbd3a63ac3bfdad8dea8aca8df9c1e321be5773e2dc1661","observed_at":"2026-06-09T04:43:45.619596Z","parent_run_hash":"f2344865fd128464efd1bacba326b5a7ccea707694b8c5650dd51ae8c46ac8a1","published":"Tue, 09 Jun 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2606.08840v1 Announce Type: new \nAbstract: Code generation models are typically compared using compact execution benchmarks and aggregate pass rates, but such summaries obscure how performance varies across programming languages, problem families, and failure modes. We present a large-scale, execution-grounded evaluation of 9 openly accessible LLMs specialized for coding on 2,707 free LeetCode problems across 12 programming languages. Our corpus contains 325,343 problem-model-language jobs, each linked to prompt metadata, extracted code, LeetCode execution outcomes, and static-analysis signals. The results show that current open models remain far from the human acceptance reference: the best model, Yi-Coder-9B-Chat, reaches 23.64% mean correctness, compared with a 57.2% human acceptance baseline. Rankings are also slice-dependent: Qwen2.5-Coder-14B-Instruct is strongest on hard problems and distinct-problem coverage, while Gemma-2-27B-IT achieves the highest all-language lint pas","title":"Beyond Pass Rate: A Multilingual, Execution-Grounded Evaluation of Open Code LLMs","url":"https://arxiv.org/abs/2606.08840","vendor":"arxiv_cs_ai"},"summary":"arXiv:2606.08840v1 Announce Type: new \nAbstract: Code generation models are typically compared using compact execution benchmarks and aggregate pass rates, but such summaries obscure how performance varies across programming languages, problem families, and failure modes. We present a large-scale, execution-grounded evaluation of 9 openly accessible LLMs specialized for coding on 2,707 free LeetCode problems across 12 programming languages. Our corpus contains 325,343 problem-model-language jobs, each linked to prompt metadata, extracted code, LeetCode execution outcomes, and static-analysis signals. The results show that current open models remain far from the human acceptance reference: the best model, Yi-Coder-9B-Chat, reaches 23.64% mean correctness, compared with a 57.2% human acceptance baseline. Rankings are also slice-dependent: Qwen2.5-Coder-14B-Instruct is strongest on hard problems and distinct-problem coverage, while Gemma-2-27B-IT achieves the highest all-language lint pas","title":"Beyond Pass Rate: A Multilingual, Execution-Grounded Evaluation of Open Code LLMs","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-06-09T04:43:45Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2606.08840"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:5d2fdc0437b9c2d685679b6b104c464190ab01463463b854d9dfac4b9289b1adf57762c20155da500a3455350f92263c88db8accbc7e705f67097c337f33840d","signer":"crovia.substrate","subject":{"observed_at":"2026-06-09T04:43:45Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2606.08840"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"8dfbb40172058771b1d5daf0d93c66a00f6249f59d072d4f60a4ce98084aff4b","leaf_index":224014,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"42d7563fd3ad4ce1b38692e564e1e14eb84df09df7e3d620ea10a0bbe8428151","side":"right"},{"sibling":"1c39d967978949c88af8c98e22e2ebf1c60f6435c6b5f475391b749a728f6bf7","side":"left"},{"sibling":"56bbac7a79ffc2b70251df7f99980461887f3d3ede91a9dcc5f881d5af86fe8e","side":"left"},{"sibling":"6ebcc6c130355ce33880787482e079e0c75370e2d9783be131f7a9d8d05d90bf","side":"left"},{"sibling":"9ed56cc86b90a849b68f64b862b9c79218ce6e4189ae2d71f48eba981d511a7c","side":"right"},{"sibling":"7639b7b8450a9c35a0488bbe3a9d302d9d58b4404f478bd3f1362ff40c9706c1","side":"right"},{"sibling":"7d319425db25fb3c3f1998c36dfe19db51428e3c4629360b23d76d78a49611fd","side":"right"},{"sibling":"e9dc257685cfad8bb0dae07b2db5d353319e50598398bbbe195a915649748f95","side":"right"},{"sibling":"dd5d61819b080d26c4380e26887a8eb889d51e2c022f796a39855defe6f68d95","side":"left"},{"sibling":"e8dcea313a54920d83e4f72d5a4223f986c719f264241d171c4712efaaf1fc54","side":"left"},{"sibling":"5480e1ea31f4744f9bd7c4261771fe51f2cdb01e705cc17320bfc202d935ca12","side":"right"},{"sibling":"24fdc29d461691aedb6fa920758206b5bb43851f477ef7a04c34aaed84b8971b","side":"left"},{"sibling":"036922da4e1e2c46d948f070454bfad299b7406fb00735ea9d8bd1e687f5f445","side":"right"},{"sibling":"533d82482604463aa4a281b9d8b85917b383b7c5f924b2b494039524c55e8797","side":"left"},{"sibling":"b2590791b920ca2a4ed39de126d2c0b1a10d9e7e62f572f12425f214e767b6e1","side":"left"},{"sibling":"87c6b850dfec08ac35a693d9db3a3315250a68adb1cfab9b1015f212b63b15bd","side":"right"},{"sibling":"c300cf0154c136afc09b1702a0be98f4ba5b6dc5cf57e8cc714ec1eaf4196eff","side":"left"},{"sibling":"d841ad93efda0869e5eb97678f348f03f5caab4353e05ff4bf18f47fb945b822","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":224761,"merkle_root":"e9f7b49b652e869ab97ffba9c5a31356b2d0e3dc5d00bb28944adf737c46b1e7","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260609T103805Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-06-09T14:15:34Z","sig_algorithm":"ed25519","signature":"8ad8076fb12c8e486ae1d1559a9a7ba8e2ee996a9ad3d8ba7bcdbdbd88ab3a15bcb429707aca6d3e9d8b97e2ba755b3dcc77b1abb6601ccb829842719a6fb30d","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_2c945da2f7efdb6eeacec6bc53622b9efe40235cdcc2ea0100057dee7a47a802"}}