{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_b193404fdcf92177ee5321528fbf17fbd8f977f58e2c5e955feea7399d6a387c","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_b193404fdcf92177ee5321528fbf17fbd8f977f58e2c5e955feea7399d6a387c","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"585b0572891cae0063aa34a898428e5493d5ef2cb469c8fa49b23435511ff8e3","published":"Tue, 16 Jun 2026 00:00:00 -0400","receipt_hash":"585b0572891cae0063aa34a898428e5493d5ef2cb469c8fa49b23435511ff8e3","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"585b0572891cae0063aa34a898428e5493d5ef2cb469c8fa49b23435511ff8e3","observed_at":"2026-06-16T04:43:43.281320Z","parent_run_hash":"eb6edcf82c3507c59161a4ab46d2e904e507004f44677402bb24d106997ed7c2","published":"Tue, 16 Jun 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2606.16733v1 Announce Type: new \nAbstract: Policy gradient algorithms for language models optimize the same objective $J(\\theta) = \\mathbb{E}*{\\tau \\sim p*\\theta(\\tau)}[R(\\tau)]$, which has exactly two factors: the trajectory probability $p_\\theta(\\tau)$ and the reward $R(\\tau)$. Every method from REINFORCE to PPO to GRPO and their descendants modifies one or both factors to address a specific failure in the preceding formulation. Existing surveys organize these methods by domain or chronology, which obscures the rationale behind each design choice and the precise location of its intervention within the gradient estimator. This survey revisits the landscape of LLM policy optimization from $J(\\theta)$ on first principles and uses the trajectory side, induced by $p_\\theta(\\tau)$, and the reward side, induced by $R(\\tau)$, as the two axes along which methods are located. It covers the path from REINFORCE and PPO to GRPO, as well as post-GRPO variants, Agentic RL, and GRPO-OPD. The r","title":"A First-Principles Derivation of LLM Policy Optimization: From Expected Reward to GRPO and Its Structural Extensions","url":"https://arxiv.org/abs/2606.16733","vendor":"arxiv_cs_ai"},"summary":"arXiv:2606.16733v1 Announce Type: new \nAbstract: Policy gradient algorithms for language models optimize the same objective $J(\\theta) = \\mathbb{E}*{\\tau \\sim p*\\theta(\\tau)}[R(\\tau)]$, which has exactly two factors: the trajectory probability $p_\\theta(\\tau)$ and the reward $R(\\tau)$. Every method from REINFORCE to PPO to GRPO and their descendants modifies one or both factors to address a specific failure in the preceding formulation. Existing surveys organize these methods by domain or chronology, which obscures the rationale behind each design choice and the precise location of its intervention within the gradient estimator. This survey revisits the landscape of LLM policy optimization from $J(\\theta)$ on first principles and uses the trajectory side, induced by $p_\\theta(\\tau)$, and the reward side, induced by $R(\\tau)$, as the two axes along which methods are located. It covers the path from REINFORCE and PPO to GRPO, as well as post-GRPO variants, Agentic RL, and GRPO-OPD. The r","title":"A First-Principles Derivation of LLM Policy Optimization: From Expected Reward to GRPO and Its Structural Extensions","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-06-16T04:43:43Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2606.16733"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:3e0ccbb7cf614a158abfcf31ef9003aa80cafa2570c67e26f8c59d07b33efa6ae89c591c0f3abf38f2e264af9098708d9a94be8fd40602559be72d55e9bcfd02","signer":"crovia.substrate","subject":{"observed_at":"2026-06-16T04:43:43Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2606.16733"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"f697ded87ce3ff3a650d72781e7a491118c2df8966ec05875f41e95b1c3c15e3","leaf_index":230288,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"75ef70a83ee6df44ebd7d62bf1ead8d5e339f6381652e2cf975e0c7d1c121829","side":"right"},{"sibling":"576a6833518c193152bd38496194c7fb5f297b1c9e1a636e06ece5d8e0886c64","side":"right"},{"sibling":"cdd261b7a3803406f17ea761b05bc042f9a7df2982f25d15200139a87d23d201","side":"right"},{"sibling":"4c1cce8186773f6da3c0096beb4204cfdac4e54f7a81fc54ee26b99d840aeeb4","side":"right"},{"sibling":"0bfe3275614b2742240775f9c4ac2223043d18b815dd481d571712b59059dc74","side":"left"},{"sibling":"f3484bbe9e5a6851aaba461de29ea5ee40da90419525554ab3fc968e0dab7b1e","side":"right"},{"sibling":"1563a36474162fbf76fa1a3ea70f0abdfd1b93b8b7dc614ada2d128d3e0d62e7","side":"right"},{"sibling":"a920a6db38b29333b686b08bcb39326c5a67bb5f4588a0647130fe767a058e04","side":"left"},{"sibling":"03ebe4791d56166247160f6ee7788c15745bdd7e274c62ec21b2438669941587","side":"left"},{"sibling":"74897e850164dddc689c3c65b33f9bae0268ab0bf429867a4e193d9b9b685040","side":"left"},{"sibling":"bde25d7e94e64717e426a97f6fcb4907e92b5c61fc89d92d7e0947a2249c3f6b","side":"right"},{"sibling":"d10d772a4984cae00e65ab24af21d1d260e475bbaae3f17871859eb705bd3999","side":"right"},{"sibling":"e79159853f2f35ddae8e3247d515e433c534277b287d65bbd77ae989aa4992fa","side":"right"},{"sibling":"3054319f1840cce0eaaf0bc4b1ae38e5bf8b6210927d924a750775cc7d77cca6","side":"right"},{"sibling":"0fd8b5059f279c4a4a6688de2472fdbc25543fee183df19b21dacca880354cff","side":"right"},{"sibling":"a04392fb9f2a3a620840e3b3fecd93d12e6d224e481c162389d8ae64e7194599","side":"left"},{"sibling":"c300cf0154c136afc09b1702a0be98f4ba5b6dc5cf57e8cc714ec1eaf4196eff","side":"left"},{"sibling":"d841ad93efda0869e5eb97678f348f03f5caab4353e05ff4bf18f47fb945b822","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":232015,"merkle_root":"62bfb7809bb55667ad7eeebdb48267b9b2c1ee89bb808ea1e6d3a814a15aa402","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260617T133701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-06-17T13:39:59Z","sig_algorithm":"ed25519","signature":"930a3563c643cc7518d048b12a1f5392a96a5edce49533ba0a56f11b6cc69319bb1c800aacc46b52a6941c4d05dae255a45cac757d6093c971b96852c24f140e","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_b193404fdcf92177ee5321528fbf17fbd8f977f58e2c5e955feea7399d6a387c"}}