{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_16714f49bfeb2651bdb5e497553e3a03ebe02dd96364297201976a731e04b39b","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_16714f49bfeb2651bdb5e497553e3a03ebe02dd96364297201976a731e04b39b","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"dfa97eed53aa9eeece7ccde83d10ad3fdb99e7f01cac64a74496d4e276d9af43","published":"Fri, 10 Jul 2026 00:00:00 -0400","receipt_hash":"dfa97eed53aa9eeece7ccde83d10ad3fdb99e7f01cac64a74496d4e276d9af43","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"dfa97eed53aa9eeece7ccde83d10ad3fdb99e7f01cac64a74496d4e276d9af43","observed_at":"2026-07-10T04:43:53.465232Z","parent_run_hash":"06997be187ba20932a2030c56de194579eacf484085252a25bee544eab183e91","published":"Fri, 10 Jul 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2606.03238v2 Announce Type: replace-cross \nAbstract: RLHF evaluation should track how failures emerge, where they localize, and which warning signals appear before external quality degrades. We study this problem with a compact RLHF pipeline built for this paper, including PPO, DPO, uncertainty-penalized PPO (UP-PPO), reward-model uncertainty, approximate policy drift, diversity and repetition diagnostics, and two external LLM judges. Rather than treating reward hacking as a single terminal event, we classify matched checkpoint and prompt-level transitions by the directions of learned reward R_phi, judge scores R_dag and R2_dag, and their average R_dag. The main empirical findings are that aggressive PPO produces the clearest localized reward-hacking signal, UP-PPO reduces but does not eliminate that signal, row-level diagnostics reveal failures hidden by checkpoint averages, and pre-transition features partially anticipate future localized reward hacking. The central conclusion ","title":"When RLHF Fails: A Mechanistic Taxonomy of Reward Hacking, Collapse, and Evaluator Gaming","url":"https://arxiv.org/abs/2606.03238","vendor":"arxiv_cs_ai"},"summary":"arXiv:2606.03238v2 Announce Type: replace-cross \nAbstract: RLHF evaluation should track how failures emerge, where they localize, and which warning signals appear before external quality degrades. We study this problem with a compact RLHF pipeline built for this paper, including PPO, DPO, uncertainty-penalized PPO (UP-PPO), reward-model uncertainty, approximate policy drift, diversity and repetition diagnostics, and two external LLM judges. Rather than treating reward hacking as a single terminal event, we classify matched checkpoint and prompt-level transitions by the directions of learned reward R_phi, judge scores R_dag and R2_dag, and their average R_dag. The main empirical findings are that aggressive PPO produces the clearest localized reward-hacking signal, UP-PPO reduces but does not eliminate that signal, row-level diagnostics reveal failures hidden by checkpoint averages, and pre-transition features partially anticipate future localized reward hacking. The central conclusion ","title":"When RLHF Fails: A Mechanistic Taxonomy of Reward Hacking, Collapse, and Evaluator Gaming","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-07-10T04:43:53Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2606.03238"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:deb72bfe7459025ae06552d4d68f708f2acc849f821c455f37431b85baceec008227b8f8d0f1f4963d3288f17889f27caba71f0ef50019d7b206aebd47ddd202","signer":"crovia.substrate","subject":{"observed_at":"2026-07-10T04:43:53Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2606.03238"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"168ba4c1a71d82633aebb98eb20dfa29da4577a677193f1ffa7a35a46c10ff15","leaf_index":299578,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"d29ac61a403ca28858308433a53388d54e82dbd60ab33a09b90c802fe7c8d1f8","side":"right"},{"sibling":"b13f6231c87b78c0fc6acef80ae3870e46ec3684ad9c4633c469ad96b3dbae5d","side":"left"},{"sibling":"bd00665c4791280bf1a0b50099b77b8e97141688cbc47e1ac39fc8d83d258b5d","side":"right"},{"sibling":"f18f986d3f1a2a64c94fc8bea1a519c246387e8a6032e903712ca6d9fc4fa5df","side":"left"},{"sibling":"32ecd7ff82343e157dcd26129da18afddd735706312ce5635169e126f8b5a694","side":"left"},{"sibling":"69bc55ca52b46f95381f84888ef673a19fb31347f7ce79a9f0cd60764972014e","side":"left"},{"sibling":"7e9fd2eac39b5e168659c352c1d3113d1e9ed668787b28903e9c9bddf865d98d","side":"right"},{"sibling":"81caca35f4fef3b5f8e889f7c59fae5c0d4d1710c9d35739fc37260996545e89","side":"right"},{"sibling":"2ca25a5e8823f400322cb0217ea4ad63fb3f05e17cb64a507a02c6bca2008beb","side":"right"},{"sibling":"8200c85c25a948eae071916caaa22b35b677accd5950f58babc0d7656c5f33dc","side":"left"},{"sibling":"f80e8d47e0860527b906fc2dba9a52609f7f623f2772479ae922cc019bab36d9","side":"right"},{"sibling":"cae83500ab2c25555aa6b5eaf9232696d15a868d91b34f7531dd955daadf70f7","side":"right"},{"sibling":"64dab64d51bdcb909e2a5e37efb8909d6704ecf824be484b5d2b60ee6e518890","side":"left"},{"sibling":"576f134a23c19a758ae5efd53016092a74b9900e878cf6eb4f3dab6be682b395","side":"right"},{"sibling":"3c65f53d7c3e4feba7c745e8df1327760ffa768eec84336db14d515a31731532","side":"right"},{"sibling":"d8b9143917b539c543cf4448cec00131f8b807bd8004979c54ebe09798748c66","side":"left"},{"sibling":"8728642cdb98496d916cc653f0d919d7eb2e89d0c529927c9e89091074ad584c","side":"right"},{"sibling":"8025674cb002a22ae243ca0c295c18c1d0ee119189ea88e08ac14a3a1468b8e3","side":"right"},{"sibling":"1cecb7f447febd025aac272837c80de218aecc6485d2395a509b2a1f1b9c746e","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":299721,"merkle_root":"f7115d63193d3285ca28cb9f741ecea2513f9b3e492e785f97076f3cf8f9bb98","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260710T053701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-07-10T05:38:19Z","sig_algorithm":"ed25519","signature":"4eeedeb744885bff6523d66b1cb86bde62b36917696367addfd980d90b31011daece2e01b20608e4c0870ec31dd0bcb57d297447f01f153bf0716ad527142b00","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_16714f49bfeb2651bdb5e497553e3a03ebe02dd96364297201976a731e04b39b"}}