{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_0bdedce294de5be70776885d63eb359ea2faedf4d6a919c11c3ebd95e52d8fb9","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_0bdedce294de5be70776885d63eb359ea2faedf4d6a919c11c3ebd95e52d8fb9","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"5cd9eec04531ab5f7a3e1aa41cb51a732ead329f9e11e5df186a4ba1f9d694ec","published":"Fri, 22 May 2026 00:00:00 -0400","receipt_hash":"5cd9eec04531ab5f7a3e1aa41cb51a732ead329f9e11e5df186a4ba1f9d694ec","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"5cd9eec04531ab5f7a3e1aa41cb51a732ead329f9e11e5df186a4ba1f9d694ec","observed_at":"2026-05-22T04:43:12.593252Z","parent_run_hash":"dd4d56660d55b2d65dc84dc5e7c8f83487d90da2dd343b5707d6948a3bb0d917","published":"Fri, 22 May 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2605.20834v1 Announce Type: new \nAbstract: Direct Preference Optimization (DPO) has emerged as a popular alternative to Reinforcement Learning from Human Feedback (RLHF), offering theoretical equivalence with simpler implementation. We prove this equivalence is conditional rather than universal, depending on an implicit assumption frequently violated in practice: the RLHF-optimal policy must prefer human-preferred responses. When this assumption fails, DPO optimizes relative advantage over the reference policy rather than absolute alignment with human preferences, leading to pathological convergence where policies decrease DPO loss while preferring dispreferred responses. We characterize when this assumption is violated, show the existence of an undesirable solution space, and prove that DPO and RLHF optimize fundamentally different objectives in such cases. To address this, we introduce Constrained Preference Optimization (CPO), augmenting RLHF with constraints for provable alig","title":"Conditional Equivalence of DPO and RLHF: Implicit Assumption, Failure Modes, and Provable Alignment","url":"https://arxiv.org/abs/2605.20834","vendor":"arxiv_cs_ai"},"summary":"arXiv:2605.20834v1 Announce Type: new \nAbstract: Direct Preference Optimization (DPO) has emerged as a popular alternative to Reinforcement Learning from Human Feedback (RLHF), offering theoretical equivalence with simpler implementation. We prove this equivalence is conditional rather than universal, depending on an implicit assumption frequently violated in practice: the RLHF-optimal policy must prefer human-preferred responses. When this assumption fails, DPO optimizes relative advantage over the reference policy rather than absolute alignment with human preferences, leading to pathological convergence where policies decrease DPO loss while preferring dispreferred responses. We characterize when this assumption is violated, show the existence of an undesirable solution space, and prove that DPO and RLHF optimize fundamentally different objectives in such cases. To address this, we introduce Constrained Preference Optimization (CPO), augmenting RLHF with constraints for provable alig","title":"Conditional Equivalence of DPO and RLHF: Implicit Assumption, Failure Modes, and Provable Alignment","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-22T04:43:12Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2605.20834"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:2d83d78742b20b80500742643698b6f24c8c5de41a3f47e9f4e06fed52d87744fdb139d0786dcfec0669e9f17846e6c7fecfb3fc62e2c1a74e668680eba4fd0d","signer":"crovia.substrate","subject":{"observed_at":"2026-05-22T04:43:12Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2605.20834"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"1d2236f5149b9e2435c2edf6c09cd3463821f49e66337d46a4b3aad223e4e2c2","leaf_index":148137,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"5806bbe342c7ed459ef704c0351ae5d08053b28ae49500c70fe9f1f8bc2c772c","side":"left"},{"sibling":"b18b7860dbdad5df742381e2b6ccf878b128e7650a85111d5f9ff5973dd8db88","side":"right"},{"sibling":"a59d401f2c04458387cd009070ef1ff27bcecac7ba3b428c1e63159b51745590","side":"right"},{"sibling":"ed63fd2ae3e05d943fb5d9eaa66e9b17e31f62ae901ae01764e2d6d8c4cf468a","side":"left"},{"sibling":"caeda7718b109ff4688623e9b54375b552882ee46b11a3bc10a377200d1d3d6d","side":"right"},{"sibling":"319d0bc7a0be1af1a0d62a223097c19ac88c19de2e03726088c23c061bd95804","side":"left"},{"sibling":"7b84b9572ef31f052959edc4c36dca1cb76c6fe010107495a0b57345fea8a82e","side":"right"},{"sibling":"ce8423e7b33fd98ad2188ec515d860afebe3ce0e74717c41376c90ea6acfb384","side":"left"},{"sibling":"c5d582bc1cdd6d6494fd9e29c8b4aadd02d777f7ef92fc6d4afad7ee42785e79","side":"right"},{"sibling":"d337a9fdcfc121e9691d8db9173af6a3fe0c33d4a6d5f0c8a7a01af04f9fb856","side":"left"},{"sibling":"8b39e07457f5cc5d687d2ae42284dbe705bb87084e7db4b626aff81e51dacd19","side":"right"},{"sibling":"79a713e1e345ccb99c5fe994a11708c8e9bcfa2e91f940d70421cb7d8d77ecc6","side":"right"},{"sibling":"249870fb494bef050c409081e5de45f9042938d7b2823524ee496f296aa63667","side":"right"},{"sibling":"96c48ee8328f1b7925a4cc4421df5cb0bd81c92a5d8354c93126fa5f0166d225","side":"right"},{"sibling":"35ca36cee447f0ef7064a25d55f59357c66901e31427729c4c1d8b14aa8adb6c","side":"left"},{"sibling":"4e13b4a3e69bb83d13913477a782913ce03937edd65046853c1964d3cbb6564b","side":"right"},{"sibling":"0f7b2df1c4580bf7bb7c24b9158ba20593a06af18a5f18d0973e5eff20c35cd8","side":"right"},{"sibling":"d841ad93efda0869e5eb97678f348f03f5caab4353e05ff4bf18f47fb945b822","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":148601,"merkle_root":"44900cffd986f40535c83f46a250e86fbf1019d41f27080a00fbf9b8d77ec33a","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260524T133701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-24T13:37:32Z","sig_algorithm":"ed25519","signature":"059e428c3c5241de303721ad6ac7b748758372180f3f0a717810312aecd6fab073abb3ea264157666de581a2b361c4c6e2aa8ca081b08ce9be090721f0e3400e","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_0bdedce294de5be70776885d63eb359ea2faedf4d6a919c11c3ebd95e52d8fb9"}}