{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_3126c259cc9993acbf42de72fc0a7dee7099ef7ce3c6fef0952367116f454d4f","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_3126c259cc9993acbf42de72fc0a7dee7099ef7ce3c6fef0952367116f454d4f","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"cfc3c238c9c2eb3dbe3958eaf25de6c6d4604e4dff8df6766ede3daf33f42022","published":"Tue, 26 May 2026 00:00:00 -0400","receipt_hash":"cfc3c238c9c2eb3dbe3958eaf25de6c6d4604e4dff8df6766ede3daf33f42022","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"cfc3c238c9c2eb3dbe3958eaf25de6c6d4604e4dff8df6766ede3daf33f42022","observed_at":"2026-05-26T04:43:39.018238Z","parent_run_hash":"dca8dedd754ad6a1772113d6b97ee4f4ab9a0afbdeb44aaace5ff2d2446b164b","published":"Tue, 26 May 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2605.24217v1 Announce Type: new \nAbstract: As Large Language Models (LLMs) transition from research environments to production deployments, evaluating their performance against strict Service Level Objectives (SLOs) has become critical. However, current evaluation methodologies suffer from severe measurement bias at scale. We demonstrate that widely used benchmarking utilities rely on single-process, asyncio-driven architectures that introduce fundamental client-side queuing bottlenecks under high concurrency. By modeling the benchmarking client as an $M/G/1$ queue, we mathematically demonstrate how the Python Global Interpreter Lock (GIL) artificially inflates Time to First Token (TTFT) and Time Per Output Token (TPOT) metrics as request rates scale. To resolve this systematic inaccuracy, we propose an unbiased, multi-process evaluation framework that effectively distributes client-side load, ensuring negligible queuing overhead. Furthermore, we formalize a composite metric, Nor","title":"Identifying and Mitigating Systemic Measurement Bias in Production LLM Inference Benchmarks","url":"https://arxiv.org/abs/2605.24217","vendor":"arxiv_cs_ai"},"summary":"arXiv:2605.24217v1 Announce Type: new \nAbstract: As Large Language Models (LLMs) transition from research environments to production deployments, evaluating their performance against strict Service Level Objectives (SLOs) has become critical. However, current evaluation methodologies suffer from severe measurement bias at scale. We demonstrate that widely used benchmarking utilities rely on single-process, asyncio-driven architectures that introduce fundamental client-side queuing bottlenecks under high concurrency. By modeling the benchmarking client as an $M/G/1$ queue, we mathematically demonstrate how the Python Global Interpreter Lock (GIL) artificially inflates Time to First Token (TTFT) and Time Per Output Token (TPOT) metrics as request rates scale. To resolve this systematic inaccuracy, we propose an unbiased, multi-process evaluation framework that effectively distributes client-side load, ensuring negligible queuing overhead. Furthermore, we formalize a composite metric, Nor","title":"Identifying and Mitigating Systemic Measurement Bias in Production LLM Inference Benchmarks","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-26T04:43:39Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2605.24217"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:fab0cfc093a96ab5306adf07d16bb02cbc37c546f87d260059f8178d23784f603ad251e10a030e7af58828ec2ea6e5dbd00d6fd19ebab74fe68b2cb74e8ecf01","signer":"crovia.substrate","subject":{"observed_at":"2026-05-26T04:43:39Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2605.24217"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"aeff38d7586f33a0923f393dad3b7c314ba3072f78ed50dca34e558ac410ee8b","leaf_index":151334,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"1b03abe2a2a35de13d349db571dd179158e2d676b744f9550c9f8e6fe9846b7b","side":"right"},{"sibling":"5889191f4dd393883561336d3dadd9bd2ec1983c25be58be249195dd7104d98c","side":"left"},{"sibling":"f83d7c14b48f11a8f3e891c5c6fb98926ae35004348e91fd81b64ccb3266f461","side":"left"},{"sibling":"c3194e910f463fd1601da6837a7da431a0665c83b3a0809ec4fdfe59e80f950d","side":"right"},{"sibling":"5f6da7be2bab48da24b5792e93704fb947f6dbf1e1d989460d7f45f5cc3f8c35","side":"right"},{"sibling":"0d98b6c220b97d75c61bc879f0037aadcbac8b635bc16e14852dbd89a27a253f","side":"left"},{"sibling":"e5bcb64fb5869d64722cefc3be31e82a01b365efdec3937cfb54d8b941368e56","side":"right"},{"sibling":"8d693959d4763b28b351e1bc64d35b03213077740e93193dee0f2ab1f6459eb7","side":"right"},{"sibling":"a808644c9a09dd75253c6c0ba9275dc94a6b6451ad5184ebdafda4a2722b36a5","side":"left"},{"sibling":"fdd22ebd87e5ba37d1153e47753a63818abb25df8fe45dbf735ba5c512c3355b","side":"left"},{"sibling":"cf593c482d17d202b94914915d0c65fb8713053548db64771d0ec5dbb404a07f","side":"left"},{"sibling":"134949308b15cffd6792ee2cf678119af34d69a64764d7c89cd47573c94e1cda","side":"left"},{"sibling":"e20a7391fed5b5f3b675af68344a3f5b050d6701e127b7db010af3941e59dfdb","side":"right"},{"sibling":"e5893793e3591ed7f5e58ca94ffcfba46bb30f69fb1c25d5ba8ef49eb99f9126","side":"right"},{"sibling":"35ca36cee447f0ef7064a25d55f59357c66901e31427729c4c1d8b14aa8adb6c","side":"left"},{"sibling":"e3eecaf996dbe7229a7bb1d234c97aea97a252c5f7c89f8547b6d091db0f0e40","side":"right"},{"sibling":"55bcbd4da3e20d93931f7e58673f10232e81a5b1514d7396cb4b71e8f95788d0","side":"right"},{"sibling":"d841ad93efda0869e5eb97678f348f03f5caab4353e05ff4bf18f47fb945b822","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":152106,"merkle_root":"ac5182c6f3dd09931f2a689df5f4be36df7b55e55bcf106e195671f5ed55fd8f","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260526T053701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-26T05:37:34Z","sig_algorithm":"ed25519","signature":"a401243fbd2c077d29a623c6ef616c78fbd5c8ce1930afa7af7165b2386e5a8cf15d5094983a1c962e71b27b911a3f3ce09c8cfab9393be2ce5ce8ee6513da06","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_3126c259cc9993acbf42de72fc0a7dee7099ef7ce3c6fef0952367116f454d4f"}}