{"_canonicalization":{"envelope_id":"axm_ + sha256(envelope minus {signature, axiom_id, anchors})","envelope_signature":"ed25519(envelope minus {signature, axiom_id})","json":"sort_keys=True, separators=(',',':'), ensure_ascii=False, allow_nan=False, utf-8","leaf_hash":"sha256(0x00 || canonical_json(envelope_full))","seal_signature":"ed25519(seal minus {signature, sig_algorithm})"},"axiom_id":"axm_cdf1e07367262e71dc83474e1a54a4cfa4419e4c6b627a5cdeea4c7807c3afbe","bitcoin_anchor":{"bitcoin_attestations":["bitcoin_block_949451"],"calendar_attestations":["https://finney.calendar.eternitywall.com","https://btc.calendar.catallaxy.com","https://alice.btc.calendar.opentimestamps.org","https://bob.btc.calendar.opentimestamps.org"],"ots_url":"/registry/data/substrate/anchors/77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3.ots","stamped_at":"2026-05-15T03:00:03Z","status":"bitcoin"},"envelope":{"anchors":[{"chain":"crovia.axiom_graph","height":0,"merkle_proof":"spider_arxiv_retraction_v1","root_at_anchor":"spider_arxiv_retraction_v1"}],"axiom_id":"axm_cdf1e07367262e71dc83474e1a54a4cfa4419e4c6b627a5cdeea4c7807c3afbe","axiom_type":"AX.OBS","body":{"axiom_subtype":"research.arxiv_retraction.v1","category":"research","fingerprint":"ac9c1e5157080f1a7ca0d0c5130a6f9ec2d870dbd278568b3b5f7c791d2b69c9","published":"2025-05-17T05:45:16Z","receipt_hash":"ac9c1e5157080f1a7ca0d0c5130a6f9ec2d870dbd278568b3b5f7c791d2b69c9","schema":"spider.research.arxiv_retraction.v1","spider":"arxiv_retraction","spider_record":{"axiom_subtype":"research.arxiv_retraction.v1","category":"research","decision_hint":"POSITIVE","envelope_target":"AX.OBS","fingerprint":"ac9c1e5157080f1a7ca0d0c5130a6f9ec2d870dbd278568b3b5f7c791d2b69c9","observed_at":"2026-05-03T16:20:11.270308Z","parent_run_hash":null,"published":"2025-05-17T05:45:16Z","runtime_version":"0.1.0","schema":"spider.research.arxiv_retraction.v1","source_status":200,"source_url":"http://export.arxiv.org/api/query?search_query=cat:cs.CL+AND+%28abs:withdrawn+OR+abs:retracted%29&max_results=20&sortBy=submittedDate&sortOrder=descending","spider":"arxiv_retraction","summary_excerpt":"Recent advances in large language models (LLMs) have fueled the vision of automated scientific discovery, often called AI Co-Scientists. To date, prior work casts these systems as generative co-authors responsible for crafting hypotheses, synthesizing code, or drafting manuscripts. In this work, we explore a complementary application: using LLMs as verifiers to automate the \\textbf{academic verification of scientific manuscripts}. To that end, we introduce SPOT, a dataset of 83 published papers paired with 91 errors significant enough to prompt errata or retraction, cross-validated with actual authors and human annotators. Evaluating state-of-the-art LLMs on SPOT, we find that none surpasses 21.1\\% recall or 6.1\\% precision (o3 achieves the best scores, with all others near zero). Furthermore, confidence estimates are uniformly low, and across eight independent runs, models rarely rediscover the same errors, undermining their reliability. Finally, qualitative analysis with domain exper","title":"When AI Co-Scientists Fail: SPOT-a Benchmark for Automated Verification of Scientific Research","url":"https://arxiv.org/pdf/2505.11855v1","vendor":"arxiv"},"summary":"Recent advances in large language models (LLMs) have fueled the vision of automated scientific discovery, often called AI Co-Scientists. To date, prior work casts these systems as generative co-authors responsible for crafting hypotheses, synthesizing code, or drafting manuscripts. In this work, we explore a complementary application: using LLMs as verifiers to automate the \\textbf{academic verification of scientific manuscripts}. To that end, we introduce SPOT, a dataset of 83 published papers paired with 91 errors significant enough to prompt errata or retraction, cross-validated with actual authors and human annotators. Evaluating state-of-the-art LLMs on SPOT, we find that none surpasses 21.1\\% recall or 6.1\\% precision (o3 achieves the best scores, with all others near zero). Furthermore, confidence estimates are uniformly low, and across eight independent runs, models rarely rediscover the same errors, undermining their reliability. Finally, qualitative analysis with domain exper","title":"When AI Co-Scientists Fail: SPOT-a Benchmark for Automated Verification of Scientific Research","vendor":"arxiv"},"confidence":{"method":"deterministic"},"decision":"POSITIVE","issued_at":"2026-05-03T16:20:11Z","notes":"Spider arxiv_retraction (research) research.arxiv_retraction.v1","object":{"captured_by":"crovia.spider.arxiv_retraction","primary_source_url":"https://arxiv.org/pdf/2505.11855v1"},"predecessors":[],"schema":"crovia.axiom.v1","signature":"ed25519:b11621164ac965ab0c49fd8219eb6f559a175538ad0235a790d26fc3d8d1d8abcad1db9428e396acb18a054dedde5bd3297b2dc54666c33f5d8f9dc470838600","signer":"crovia.substrate","subject":{"observed_at":"2026-05-03T16:20:11Z","source_collector":"spider:arxiv_retraction","target_id":"https://arxiv.org/pdf/2505.11855v1"},"tsa":{"authority":"crovia.substrate.bootstrap","rfc3161_token":"{\"kind\":\"crovia.bootstrap.tsa\",\"source_jsonl\":\"/opt/crovia/spider/data/research/arxiv_retraction_v1.jsonl\",\"source_seal_merkle_root\":\"spider_arxiv_retraction_v1\",\"upgrade_path\":\"Sessione H \\u2014 OpenTimestamps weekly anchor\"}"},"zk_mode":"clear","zk_proof":null},"ledger":{"leaf_hash":"3e8128f6ad723dbdc22bc10b689234e351c03e15d7494458c4543d36df5e4041","leaf_index":111956,"ledger_path":"/opt/crovia/substrate/axiom_ledger.jsonl"},"merkle_proof":{"hash_alg":"sha256","leaf_prefix":"0x00","node_prefix":"0x01","odd_leaf_rule":"duplicate_last","path":[{"sibling":"743d0ba78ef434f46d8527b577b431135cd9b8942cb9a16cc25253ef0779bf19","side":"right"},{"sibling":"7132f83858b09c4fe8585057692441ee26373f846c992c9c62d4c6b235771182","side":"right"},{"sibling":"86b59e7b21fa356a87edff73166037df3aaa05ac0990aa988f0d011b9e95b802","side":"left"},{"sibling":"951fa52a59563d328ea8458383c0c32e618e11ed499290b0404cf6e89f3c05f8","side":"right"},{"sibling":"873972dd2f23ac861d3231b64ea0df76f3b27107aae4b882b5aa10e66a923761","side":"left"},{"sibling":"04f08a05f60539e5d320a1d658ae9463ad3d1673f88287d19b483af959df3046","side":"right"},{"sibling":"7cc6ec0331a098790dc04ced8e8abc2b910e0fcce6830c8442fff364ca231498","side":"left"},{"sibling":"0994d51d2178913b0e3ac7346ceed16c5ea436d841a8e7f28a99362fa269bad0","side":"right"},{"sibling":"015d2ce3c58c7a790ad078159267c63cd54f9ae7e20530115a6a423419cefe22","side":"left"},{"sibling":"5bcdb02d4310fc8bfd791789d990714b0a10f44358ffa98c446f1755352c49b8","side":"right"},{"sibling":"cf8897e1feec248f5d05e2cafec4f8c2739e2005227d28b5ccc8cfa11ed66ee4","side":"left"},{"sibling":"a6d6e5bc888edc394757fbba02b675bac6ad0831e36a6f5ebbaa4e5548f25603","side":"right"},{"sibling":"76d7c4385d71ae8104107105be66eddb792e01ed6e493db5a9b4eec5441abb76","side":"left"},{"sibling":"b03f005860bf95148ea89c703a32ca19ab7a2ca71b58bb628ff94a9edb8703b0","side":"left"},{"sibling":"21d1e30b556f553dc939fddc23e6367f0a7755ebe3dd489dc0001cef017beb7f","side":"right"},{"sibling":"f2817ab288b5324fe49770372c7a10f33f7cd11005f8d4c0a730316f5229dc98","side":"left"},{"sibling":"725fac972e772ca0dc598810ea1abc70df472f72d2d6ab8a0baee2b80e5d2f4c","side":"left"},{"sibling":"98fc57dfef8873b512edc8340f7181df57302bb96777625e072235c62d7c5895","side":"right"}]},"schema":"crovia.axiom_proof.v1","seal":{"first_collector_run_id":"","first_receipt_hash":"","jsonl_path":"/opt/crovia/substrate/axiom_ledger.jsonl","key_id":"430895f101d38164","last_collector_run_id":"","last_receipt_hash":"","leaf_count":134292,"merkle_root":"77fc9c28fae777b81da5b495b3115474df6592dfac590333213d3bdf8b94a9b3","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","run_id":"hourly_json_retrofit_20260515T023701Z","schema":"crovia.seal.v1","seal_family_version":"crovia-seal-family/1","seal_kind":"substrate_batch","sealed_at":"2026-05-15T02:37:25Z","sig_algorithm":"ed25519","signature":"68107a834b00b24f5d4501e5ec727445311f132a486567ecc4c72a4e6dff24c8c21f2de3105293353ba5fdbe370d032819af6aa70f694e2e39b6af6737507009","signer_version":"1.1.0"},"trust_root":{"key_id":"430895f101d38164","public_key_hex":"cf742e26f75669dc673cb5c0786a1ae23ae8ca19c347317192ce40c28a7ff25c","signature_algorithm":"ed25519","url":"/registry/canon/TRUST_ROOT.md"},"verifier":{"spec":"/registry/canon/AXIOM_RECEIPT_v1.md","url":"/v/axm_cdf1e07367262e71dc83474e1a54a4cfa4419e4c6b627a5cdeea4c7807c3afbe"}}