{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_204e61f01ddf7fa1c899a52660dc9de0e4ea4d52bcc4c5a53c338f8139eb64df","bitcoin_anchor":{"bitcoin_attestations":[],"calendar_attestations":[],"ots_url":"","stamped_at":"","status":"pending_next_stamp"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_vendor_press_v1","root_at_anchor":"spider_vendor_press_v1"}],"axiom_id":"axm_204e61f01ddf7fa1c899a52660dc9de0e4ea4d52bcc4c5a53c338f8139eb64df","axiom_type":"AX.OBS","body":{"axiom_subtype":"news.vendor_press.v1","category":"news","fingerprint":"dfc500aef4cba30abed5312a5df631adab1fcc520eab83f92fe54561ab8a5536","published":"Wed, 22 Jul 2026 00:00:00 -0400","receipt_hash":"dfc500aef4cba30abed5312a5df631adab1fcc520eab83f92fe54561ab8a5536","schema":"spider.news.vendor_press.v1","spider":"vendor_press","spider_record":{"axiom_subtype":"news.vendor_press.v1","category":"news","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"dfc500aef4cba30abed5312a5df631adab1fcc520eab83f92fe54561ab8a5536","observed_at":"2026-07-22T04:43:18.261256Z","parent_run_hash":"4765c85b8b4b27ff9a690c1ae11c3b009baa2c60297295f395ad422f5afed68c","published":"Wed, 22 Jul 2026 00:00:00 -0400","runtime_version":"0.1.0","schema":"spider.news.vendor_press.v1","source_status":200,"source_url":"https://export.arxiv.org/rss/cs.AI","spider":"vendor_press","summary_excerpt":"arXiv:2607.18966v1 Announce Type: new \nAbstract: Language models trained with reinforcement learning may learn to optimize the grader's judgment rather than the intended objective. This \"reward-seeking\" is difficult to measure because a model that pursues the grader's judgment and one that pursues the intended objective behave identically whenever the grader rewards the intended behavior. We measure reward-seeking using Contrastive Synthetic Document Finetuning to change a model's beliefs about what the grader rewards, putting those beliefs in conflict with what users or developers want, and measuring the rate at which the model adopts each party's preferred behavior. Applied to intermediate checkpoints of a capabilities-focused OpenAI o3 RL run, without safety training, we find that these checkpoints often side with grader preferences over those of users or developers on coding and alignment tasks. This tendency to side with the grader trends upward throughout RL training. For example","title":"Measuring Reward-Seeking via Contrastive Belief Updates","url":"https://arxiv.org/abs/2607.18966","vendor":"arxiv_cs_ai"},"summary":"arXiv:2607.18966v1 Announce Type: new \nAbstract: Language models trained with reinforcement learning may learn to optimize the grader's judgment rather than the intended objective. This \"reward-seeking\" is difficult to measure because a model that pursues the grader's judgment and one that pursues the intended objective behave identically whenever the grader rewards the intended behavior. We measure reward-seeking using Contrastive Synthetic Document Finetuning to change a model's beliefs about what the grader rewards, putting those beliefs in conflict with what users or developers want, and measuring the rate at which the model adopts each party's preferred behavior. Applied to intermediate checkpoints of a capabilities-focused OpenAI o3 RL run, without safety training, we find that these checkpoints often side with grader preferences over those of users or developers on coding and alignment tasks. This tendency to side with the grader trends upward throughout RL training. For example","title":"Measuring Reward-Seeking via Contrastive Belief Updates","vendor":"arxiv_cs_ai"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-07-22T04:43:18Z","notes":"Spider vendor_press (news) news.vendor_press.v1","object":{"captured_by":"crovia.spider.vendor_press","primary_source_url":"https://arxiv.org/abs/2607.18966"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:5b60a9d2df4ce2c86fbef527e47f83795c940392e5eb4bf1053b16387a0aaef6e7de7f7b0147cc0a51edf2c0f17da79073bffaa65f7dcf4b72a5befb523cca0b","signer":"crovia.substrate","subject":{"observed_at":"2026-07-22T04:43:18Z","source_collector":"spider:vendor_press","target_id":"https://arxiv.org/abs/2607.18966"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/news/vendor_press_v1.jsonl\",\"source_seal_merkle_root\":\"spider_vendor_press_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"80b72009a86accf9020367be7d097089db6d3ceeb719745991e4feaedc27c019","leaf_index":340170,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"fc6ef2db577c07656f490b563b0b9e89a90a3aeae5aef2869c803b0276d77006","side":"right"},{"sibling":"f0df501a7edd12a7ba927feb91315c4de00a64b10adfa421531ec35a31ab6abd","side":"left"},{"sibling":"64979b85bbc7a21bc054ae4a64f11fbf0788068948ded8062da57e8809cde5ba","side":"right"},{"sibling":"868f4813668d2bc20c1a2df6b91137dcb9a1896548d45bb8f131355d7976bc6b","side":"left"},{"sibling":"6b2f1bdf6e0ad7f7176531c9dac3fa401224ef14ea02a337888d043b4010169d","side":"right"},{"sibling":"08f669cf62f9d6d640804bc9893be0035eaa2efdd5e539b9010e2fc83200caac","side":"right"},{"sibling":"3ef60c9ed8999dfaf391c6bba67940285b69f69ba30251f62e74b5bc37deb1a8","side":"left"},{"sibling":"acd872eabf3ff3deb3760bc63ca3a4ca483306d9b2eb2ddf59ac880a653ae17a","side":"left"},{"sibling":"c0cb3fbdae423fd11084f7ab87b57c412a0ef2e6dcda395ebb9831d480ad609b","side":"right"},{"sibling":"c7fc9d4187cdc36f4c03b4b13daf4b880ea65536b051f71a5cc2543839d02697","side":"right"},{"sibling":"1758ec6ac206ce40e8368cb702195322fe3737d0fb03d8bd9e3b30acc4fa7d81","side":"right"},{"sibling":"0c407f0d553cf3fab8f9bd79205b8180e090cbf29fa0490ebb55155041ad5c86","side":"right"},{"sibling":"2dd9cb2521044ee7c6b74f2315e0a0253b8df0d04a7b810bbbbe7da5a9788769","side":"left"},{"sibling":"21d66dd41003813f710b7617944f1bfba3258658a5d3370c21cad8f9e945bc99","side":"left"},{"sibling":"787ee3744642ff909d610b0514cb100784f0ef1ba0ef4c70a0dc91f0ab2bb192","side":"right"},{"sibling":"9fc8a8ebbc1bff7e62b9f1e1c681c91e7196092ce9551573df6e23096df13e4d","side":"right"},{"sibling":"77025bcb374a7ad74f520643e20a8ae1205a7ee78507b0beb93117760f1c29d3","side":"left"},{"sibling":"ee6f33920899d9bdef2eb706dfff29e26eea61e29ea9824c6c8e6bcd48275d76","side":"right"},{"sibling":"1cecb7f447febd025aac272837c80de218aecc6485d2395a509b2a1f1b9c746e","side":"left"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":340557,"merkle_root":"7d45d94f20b5bf82263df45b87749e19e06161972f25141dd573cc138d566338","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260722T053701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-07-22T05:38:39Z","sig_algorithm":"ed25519","signature":"812cb61e90d3582ba508db8515c4168bbf8ee1c6762885609049d012f680f8068f15c98b9accf2042047dcfc6a6fc3885820ee34b0be350846386f64f2591309","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_204e61f01ddf7fa1c899a52660dc9de0e4ea4d52bcc4c5a53c338f8139eb64df"}}