{ "schema_version": "1.0", "criteria_version": "1.0", "status": "CONTENT_REVIEWED_PUBLICATION_AUTHORIZED", "publication_authorized_at": "2026-10-20", "claims": [ { "claim_id": "M1", "exact_statement": "METR\u2019s entry-level 1\u20135 rating measures importance or interestingness, not confidence.", "source_ids": [ "S-METR" ], "source_location": "Classifier sweeps, web extraction lines 952\u2013961", "relevant_excerpt": "important or interesting", "evidence_status": "supported_as_written", "evidence_type": "procedure disclosure", "supporting_evidence": "Purpose explicitly stated; follow-up examines high-rated and random examples.", "contrary_evidence": "Selection is a discovery tool, not necessarily an attempt at probability sampling.", "limitations": "This concerns entry scores, not the separate reaction-importance score.", "unresolved_questions": "Exact anchors and validation were not located in bounded search.", "checks_performed": [ "Read method and follow-up sections" ], "checks_not_performed": [ "Historical classifier replay" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "M2", "exact_statement": "METR discloses manual checks and model-analysis limitations, while public rubric reconstruction remains incomplete within the searched materials.", "source_ids": [ "S-METR" ], "source_location": "Limitations lines 303\u2013317; methodology 919; timeline 905\u2013909", "relevant_excerpt": "manually spot-check and validate", "evidence_status": "supported_with_qualification", "evidence_type": "bounded disclosure assessment", "supporting_evidence": "Manual checks, expanded dataset, and incomplete verification are disclosed.", "contrary_evidence": "Substantial methods are public; absence from search is not historical nonexistence.", "limitations": "Report, relevant linked materials, link inventory and one official-site search only.", "unresolved_questions": "Unpublished prompts, anchors, calibration, claim-level validation coverage.", "checks_performed": [ "Read cited sections; inspect link inventory and bounded search log" ], "checks_not_performed": [ "Private METR data or prompts; full historical replication" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "M3", "exact_statement": "Displayed completeness-table counts sum to 101 for a described sample of 100; overlap is plausible but unreconciled.", "source_ids": [ "S-METR", "S-METR-PDF" ], "source_location": "Estimating completeness; HTML lines 923\u2013934; PDF printed pp.73\u201374 and footnote 116", "relevant_excerpt": "100", "evidence_status": "supported_with_qualification", "evidence_type": "deterministic arithmetic plus unresolved interpretation", "supporting_evidence": "93+3+2+1+1+1=101. Prose says 100 and six additional.", "contrary_evidence": "Footnote describes a background-script message among truncated transcripts; rows may overlap.", "limitations": "Does not refute coverage or identify a corrected estimate.", "unresolved_questions": "Are rows disjoint? Which stable message IDs populate each row?", "checks_performed": [ "Python addition; visual PDF check; HTML comparison" ], "checks_not_performed": [ "Unique-ID reconciliation" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "M4", "exact_statement": "METR displays a command/output mismatch as evidence of a spoofed tool call.", "source_ids": [ "S-METR" ], "source_location": "Tool-call spoofing discussion, lines 225\u2013228", "relevant_excerpt": "SPOOFTEST", "evidence_status": "supported_with_qualification", "evidence_type": "reported observation", "supporting_evidence": "The displayed command and returned token differ.", "contrary_evidence": "Report warns subtle spoofing could have been missed.", "limitations": "A published illustration is not our independent access to the historical execution.", "unresolved_questions": "Unavailable raw execution context.", "checks_performed": [ "Read displayed example and adjacent limitations" ], "checks_not_performed": [ "Independent historical execution validation" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "A1", "exact_statement": "Anthropic reports approximately 1% versus 50% action flag rates with versus without reasoning in the Mythos 5 incident.", "source_ids": [ "S-ANTH" ], "source_location": "Offline monitoring, lines 311\u2013318", "relevant_excerpt": "about 50%", "evidence_status": "supported_with_qualification", "evidence_type": "reported input intervention", "supporting_evidence": "Same report describes withholding reasoning and changed flag rate.", "contrary_evidence": "For the internal research model incident, verbalized harm supplied evidence the monitor otherwise missed.", "limitations": "Action flag rate is not incident-detection probability or general accuracy; approximate rates.", "unresolved_questions": "Paired outputs, denominator, configurations and randomness needed for replication.", "checks_performed": [ "Read results and contrary case" ], "checks_not_performed": [ "Monitor rerun; significance or causal generalization" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "A2", "exact_statement": "Anthropic describes a 1\u201310 biased-reasoning grader, human review, and 12 scenarios run five times per checkpoint.", "source_ids": [ "S-ANTH" ], "source_location": "Analysis of biased reasoning; Figure 36, lines 323\u2013327", "relevant_excerpt": "seemed well calibrated", "evidence_status": "supported_as_written", "evidence_type": "measurement disclosure", "supporting_evidence": "12\u00d75=60; human review is explicitly reported.", "contrary_evidence": "Human review is positive evidence; no quantified agreement study located in scoped section.", "limitations": "Does not establish measured grader accuracy; distinct from offline monitor.", "unresolved_questions": "Human reference labels, selection, agreement estimates and anchors.", "checks_performed": [ "Python 12\u00d75; read Figure 36 caption" ], "checks_not_performed": [ "Grader execution or human-label agreement study" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "H1", "exact_statement": "CISA\u2019s maintainer confirmed republication with base score 6.5 for the unchanged vector originally paired with 5.3.", "source_ids": [ "S-CISA", "S-FIRST" ], "source_location": "Issue body; comment 5526230961; FIRST \u00a77.1 and \u00a77.4", "relevant_excerpt": "republished with a correctly calculated CVSS", "evidence_status": "supported_as_written", "evidence_type": "maintainer acknowledgement plus deterministic recomputation", "supporting_evidence": "Decimal implementation yields 6.5; comment gives vector and corrected score.", "contrary_evidence": "Original issue proposed either adjusting score or vector conditionally.", "limitations": "Numerical consistency does not establish impact-metric truth, exploitability in deployment, or downstream harm reduction.", "unresolved_questions": "No unresolved arithmetic question.", "checks_performed": [ "Read issue and response; recompute FIRST equation" ], "checks_not_performed": [ "Historical full corpus scan; real-world impact validation" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "H2", "exact_statement": "Public acceptance and arithmetic do not establish autonomous discovery or the causal value of subagents.", "source_ids": [ "S-CISA" ], "source_location": "Issue and response establish correction; causal inference boundary", "relevant_excerpt": "", "evidence_status": "supported_as_written", "evidence_type": "scope limitation", "supporting_evidence": "Public primary record does not contain the originating instruction and full execution history.", "contrary_evidence": "Private historical records were inspected and retained separately; they are not public replication evidence.", "limitations": "Do not infer that no machine-led work occurred. Private records and their contents are excluded from this public package.", "unresolved_questions": "A controlled alternative workflow would be needed for a causal subagent-value claim.", "checks_performed": [ "Separate public result from private process evidence" ], "checks_not_performed": [ "Causal comparison of workflows" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "H3", "exact_statement": "Hermes Rubric exposes score safeguards and coverage documentation without establishing general assessment accuracy.", "source_ids": [ "S-RUBRIC" ], "source_location": "https://github.com/hermes-labs-ai/hermes-rubric/blob/1001fd49c2e204df33c54f0a3c6ecbe5d97cee65/src/hermes_rubric/score.py (see src/hermes_rubric/score.py lines 101\u2013114); EVAL-COVERAGE.md; calibration/META-RUBRIC.md", "relevant_excerpt": "They do not establish general assessment accuracy.", "evidence_status": "supported_with_qualification", "evidence_type": "committed implementation and declared evidence boundary", "supporting_evidence": "Clamps enforce limited score behavior; EVAL-COVERAGE separates mechanical checks from unproven human-label agreement.", "contrary_evidence": "A structurally sound rubric can still yield incorrect scores; private calibration corpus limits external reconstruction.", "limitations": "Source inspection is not a new target assessment. No Rubric aggregate or evaluator receipt exists for this note because none was run.", "unresolved_questions": "Domain validity, human agreement, and superiority remain unproven.", "checks_performed": [ "Inspect committed code/docs and frozen meta-rubric" ], "checks_not_performed": [ "Rubric assessment of note; general-validity experiment" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "P1", "exact_statement": "Precise Records distinguishes record-level measurements from warranted construct claims and explicitly does not validate recursive interpretive conditioning.", "source_ids": [ "S-PRECISE" ], "source_location": "Original deposited PDF, \u00a71 pp.1\u20132 and limitations", "relevant_excerpt": "does not depend on, test, or validate the framework of recursive interpretive conditioning.", "evidence_status": "supported_as_written", "evidence_type": "conceptual/empirical boundary in original text", "supporting_evidence": "Original introduction explicitly separates its question from the companion framework.", "contrary_evidence": "Single institutional deployment and AI-mediated authorship limit generalization.", "limitations": "Used as conceptual antecedent, not independent confirmation of these incidents.", "unresolved_questions": "No priority claim beyond verified deposit/report chronology.", "checks_performed": [ "Read original version PDF and metadata" ], "checks_not_performed": [ "Reproduce underlying telemetry study" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "P2", "exact_statement": "The Generative Horizon proposes recursive interpretive conditioning; both July 30 deposits precede METR\u2019s August 26 report, not the June\u2013July incident.", "source_ids": [ "S-HORIZON", "S-PRECISE", "S-METR" ], "source_location": "Original deposited PDFs title/abstract; Zenodo version metadata; METR date and dates in scope", "relevant_excerpt": "the output of one horizon becomes part of another.", "evidence_status": "supported_as_written", "evidence_type": "conceptual antecedent and chronology", "supporting_evidence": "July 30 publication dates; METR August 26 report and June 26\u2013July 13 scope.", "contrary_evidence": "Publication timing is not prediction or experimental validation.", "limitations": "No proof of mechanism or formal attractor dynamics from this audit.", "unresolved_questions": "Historical file-identity limitations are recorded in workpapers.", "checks_performed": [ "Inspect version-specific metadata/PDF text and report dates" ], "checks_not_performed": [ "Mechanistic experiment" ], "derivation_links": [], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "R1", "exact_statement": "Publish scoring definitions, retain separately sampled checks, and measure human/model disagreement.", "source_ids": [ "S-EVALS" ], "source_location": "Types of graders; Step 5", "relevant_excerpt": "Requires calibration with human graders for accuracy", "evidence_status": "supported_with_qualification", "evidence_type": "proposed control informed by guidance", "supporting_evidence": "Addresses rubric inspection and accuracy boundaries.", "contrary_evidence": "Human labeling itself needs qualified reviewers and uncertainty reporting.", "limitations": "This control was proposed, not tested; no efficacy claim.", "unresolved_questions": "Would held-out agreement and category error rates improve?", "checks_performed": [ "Inspect guidance and connect to findings" ], "checks_not_performed": [ "Prospective control evaluation" ], "derivation_links": [ "M1", "M2", "A2", "H3" ], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "R2", "exact_statement": "Retain claim provenance and contrary evidence so a new reviewer can reconstruct conclusions.", "source_ids": [ "S-AS1105", "S-AS1215", "S-TOB" ], "source_location": "AS1105 .02/.05/.29; AS1215 .06\u2013.08; Trail of Bits 04\u201307", "relevant_excerpt": "information that contradicts such assertions", "evidence_status": "supported_with_qualification", "evidence_type": "adapted principle and proposed control", "supporting_evidence": "Evidence quality, documentation, distinct findings and corrective actions support inspectability.", "contrary_evidence": "PCAOB financial-audit standards do not govern this note; commercial example is illustrative.", "limitations": "No certification, new standard, or demonstrated control efficacy.", "unresolved_questions": "Would new reviewers recover the same evidence and inferential boundary?", "checks_performed": [ "Read primary methods references" ], "checks_not_performed": [ "Certification or intervention study" ], "derivation_links": [ "M2", "M3", "H2" ], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "R3", "exact_statement": "Test both reasoning-inclusive and action-only monitoring, and compare subagent workflows against a stated alternative.", "source_ids": [ "S-ANTH", "S-CISA" ], "source_location": "Lead recommendations derived from A1 and H2", "relevant_excerpt": "", "evidence_status": "supported_with_qualification", "evidence_type": "proposed controls", "supporting_evidence": "Matched-case comparison addresses context sensitivity; workflow comparison addresses causal attribution.", "contrary_evidence": "Reasoning can help detection; accepted fixes still demonstrate real outcomes.", "limitations": "Neither control has been evaluated in this audit.", "unresolved_questions": "False alarms, incident-level surfacing, and causal contribution remain to be measured.", "checks_performed": [ "Derive testable proposals from bounded findings" ], "checks_not_performed": [ "New experiments" ], "derivation_links": [ "A1", "H2" ], "criteria_version": "1.0", "publication_boundary": "Public-source-based candidate for Roli review; not publication authorization" }, { "claim_id": "H4", "exact_statement": "The historical T1 below-one stability sentence conflicts with its table if read across all runs, while its heading supports a narrower main_b reading.", "source_ids": [ "S-RUBRIC" ], "source_location": [ "https://github.com/hermes-labs-ai/hermes-rubric/blob/1001fd49c2e204df33c54f0a3c6ecbe5d97cee65/evals/repeatability/INTERPRETATION.md", "https://github.com/hermes-labs-ai/hermes-rubric/blob/1001fd49c2e204df33c54f0a3c6ecbe5d97cee65/evals/repeatability/REPEATABILITY-TABLE.md" ], "relevant_excerpt": "T1 is the only target with \u03c3 consistently <1.0 across modes and runs.", "evidence_status": "supported_with_qualification", "evidence_type": "internal document consistency check", "supporting_evidence": "Two main_a T1 values, 1.824 and 1.129, exceed 1 and conflict with the literal all-runs wording.", "contrary_evidence": "The section heading explicitly singles out main_b; both main_b values, 0.548 and 0.916, are below 1.", "limitations": "Compares interpretation with cited table, not fresh raw-run statistical validation. Current EVAL-COVERAGE is more restrained.", "unresolved_questions": "Clarification of intended scope is needed; raw-run derivation was not reaudited.", "checks_performed": [ "Lead read interpretation and all four T1 rows; threshold comparison" ], "checks_not_performed": [ "Recompute sigma from raw model runs" ], "derivation_links": [ "H3" ], "criteria_version": "1.0", "publication_boundary": "Public primary source; local review candidate only" } ], "manuscript_sha256": "f2bfbbed7aba920cc60aa0e64b0ff105943ea6aa562a7bf6ad8be94b4b64eed3" }