{
  "title": "Figures registered on the research on checking AI, bentleymoon.com/research/checking-ai/",
  "publisher": "Bentley Moon",
  "ledgerUpdated": "2026-09-29",
  "howToCite": "Cite the page a figure appears on, with its checked date. For the attack results, cite the released record: Moon, Bentley. \"Detection is not resistance: frontier prompt-injection receipts.\" https://github.com/BentleyMoon/frontier-repro, doi:10.5281/zenodo.22805797.",
  "rules": [
    "A figure is registered with its source and the day it was read; the build fails if a registered figure changes on its page or its source link disappears.",
    "A programme figure names its unit of analysis; an outside figure links its source.",
    "A result that a later run reinterprets carries the reinterpretation in the same section of the page.",
    "Not every figure on the site is registered yet. This file lists the ones that are."
  ],
  "figures": [
    {
      "id": "gating-correlation",
      "kind": "programme",
      "statement": "The correlation between an auditor's own task accuracy and its false-alarm rate is −0.87, replicated across three independent runs at −0.86, −0.84 and −0.90.",
      "numbers": [
        "−0.87",
        "−0.86",
        "−0.84",
        "−0.90"
      ],
      "unit": "auditor model, n = 7. The 63 task-and-auditor cells are the measurements behind each auditor's two rates, so they are seven clusters rather than 63 independent points. 95% CI −0.98 to −0.34 (Fisher z, n = 7).",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier public fact sheet, frozen 2026-07-02",
        "url": null
      },
      "checked": "2026-08-14"
    },
    {
      "id": "false-alarm-split",
      "kind": "programme",
      "statement": "Low-competence auditors disagreed with a stronger model's correct answers 59–67% of the time; high-competence auditors 2–3.5%.",
      "numbers": [
        "59–67%",
        "2–3.5%"
      ],
      "unit": "task-and-auditor cell, 63 cells across seven auditor models.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier public fact sheet, frozen 2026-07-02",
        "url": null
      },
      "checked": "2026-08-13"
    },
    {
      "id": "shared-basin-rho",
      "kind": "programme",
      "statement": "Independently built checkers collide in shared error basins at rho of about 0.55.",
      "numbers": [
        "0.55"
      ],
      "unit": "model pair.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier public fact sheet, frozen 2026-07-02",
        "url": null
      },
      "checked": "2026-08-13"
    },
    {
      "id": "foothold-repair",
      "kind": "programme",
      "statement": "Execution-feedback self-repair crossed a local failure boundary on one task from 0.0 to 0.69 against a sampling baseline that never solved it, and the crossing replicated.",
      "numbers": [
        "0.0 → 0.69"
      ],
      "unit": "task.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier public fact sheet, frozen 2026-07-02",
        "url": null
      },
      "checked": "2026-08-13"
    },
    {
      "id": "repair-regression",
      "kind": "programme",
      "statement": "On one task the repair loop regressed accuracy from 0.53 to 0.19 by entrenching a wrong approach.",
      "numbers": [
        "0.53",
        "0.19"
      ],
      "unit": "task.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier public fact sheet, frozen 2026-07-02",
        "url": null
      },
      "checked": "2026-08-13"
    },
    {
      "id": "frontier-legibility",
      "kind": "programme",
      "statement": "On one identical blatant injection placed as the LAST text in the prompt, Claude Sonnet 5 obeys 0.80 of the time while detecting it 1.00 of the time; Opus 4.8 obeys 0.00 and detects 1.00; Haiku 4.5 obeys 0.00 and detects 0.59. The Sonnet rate is position-dependent: appending neutral trailing text suppresses it monotonically to 0.000 by 200 characters.",
      "numbers": [
        "0.80",
        "1.00",
        "0.00",
        "0.59",
        "0.713",
        "0.650",
        "0.263",
        "0.013",
        "0.738",
        "0.787",
        "0.850",
        "0.825",
        "0.800"
      ],
      "unit": "task; n = 20 tasks per model, four samples each, false alarms 0.00 for all three models. The trailing-text ladder is n = 20 tasks per rung at 25/50/100/200/400 characters.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "frontier-repro, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-08-30"
    },
    {
      "id": "frontier-channel-effect",
      "kind": "programme",
      "statement": "The identical malicious payload moves Sonnet 5 only when it wears a System, tool-output or cited-spec costume, not when presented as untrusted data: a channel effect of +0.307.",
      "numbers": [
        "+0.307"
      ],
      "unit": "task; n = 14, 90% CI 0.226 to 0.390.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "frontier-repro, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-08-14"
    },
    {
      "id": "access-by-reference",
      "kind": "programme",
      "statement": "One local checker, gemma4:12b, separates right answers from wrong ones at 1.000 when handed the correct value, 0.536 when it can apply a one-step test, and 0.034 when it has to work the answer out, on sums it solves itself at 1.000.",
      "numbers": [
        "1.000",
        "0.536",
        "0.034"
      ],
      "unit": "task, one checker model. Rows one and three n = 500, pre-registered (conf_family_gemma12, 2026-07-30); row two n = 250, one exploratory run (acc_testable_low, 2026-07-31).",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier ledger, design ids conf_family_gemma12 and acc_testable_low",
        "url": null
      },
      "checked": "2026-09-19"
    },
    {
      "id": "card-landing-lab",
      "kind": "programme",
      "statement": "The checker ladder for one checker, gemma4:12b: 1.000 when handed the right answer and 0.042 when it had to work the answer out (the leak-free re-run; the July run's 0.034 is the footnote).",
      "numbers": [
        "1.000",
        "0.042",
        "0.034"
      ],
      "unit": "task, one checker model, n = 500 each, pre-registered 2026-07-30; re-run 23 September 2026 after the template-leak fix.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "The figures registered on /findings/ as access-by-reference, reference-laundering and gemma-reserves",
        "url": null
      },
      "checked": "2026-10-12"
    },
    {
      "id": "voting-inverted-u",
      "kind": "programme",
      "statement": "On one procedurally generated task family, an eight-sample vote lifted Claude Opus 4.8 by +0.156 where single-sample competence was partial; the bundle’s own post-hoc split-sample check puts that lift at +0.064.",
      "numbers": [
        "+0.156",
        "0.093",
        "0.216",
        "+0.064",
        "0.094",
        "+0.013"
      ],
      "unit": "task. Primary: n = 89 tasks in the partial band of 768, 90% CI 0.093 to 0.216. Split-sample: 147 tasks, 90% CI 0.034 to 0.094, four evaluation samples per fold.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "opus_transfer_reproduction_bundle_2026-07-21, voting/followup_d5",
        "url": null
      },
      "checked": "2026-09-19"
    },
    {
      "id": "which-model-not-how-big",
      "kind": "programme",
      "statement": "On an identical task list with a shared seed, qwen3.5:9b scored 0.804 at separating right answers from wrong ones and the larger gemma4:12b scored 0.034.",
      "numbers": [
        "0.804",
        "0.034"
      ],
      "unit": "task; n = 500 per arm, paired seed, pre-registered with a matched anti-claim (conf_family_qwen35, conf_family_gemma12).",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier ledger, design ids conf_family_qwen35 and conf_family_gemma12",
        "url": null
      },
      "checked": "2026-09-19"
    },
    {
      "id": "reference-laundering",
      "kind": "programme",
      "statement": "A checker handed a false reference that agrees with a corrupted answer passes the corrupted answer: gemma4:12b on 0.969 of items. One of nineteen defences held, and only on gemma4:31b (0.094) where three other large models passed 0.693, 0.980 and 0.994. A correct peer reference introduced as another model’s answer moved the 12B checker only to 0.072.",
      "numbers": [
        "0.969",
        "0.094",
        "0.693",
        "0.980",
        "0.994",
        "0.072",
        "0.113"
      ],
      "unit": "task; n = 512 per cell for laundering, two seeds on gemma4:31b; n = 250 per cell, one exploratory run, for the composition cells.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier FINDINGS.json, findings 21, 23 and 27; ledger design ids compose_*",
        "url": null
      },
      "checked": "2026-09-19"
    },
    {
      "id": "inversion-crossing",
      "kind": "programme",
      "statement": "The width at which a monitor’s correctness discrimination turns negative is located for four local models: gemma4:12b between three and four digits, gemma4:31b between four and five, qwen2.5-coder:32b between three and four, qwen2.5-coder:7b never positive.",
      "numbers": [
        "+0.257",
        "+0.884",
        "+0.050",
        "+0.564"
      ],
      "unit": "task; n = 424 per rung, one predicate family, local models only (inv_gemma_d3/d4, invc_gemma31_d3/d5, invs_coder32_d3/d4/d5).",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier FINDINGS.json finding 18, and the ledger rows it names",
        "url": null
      },
      "checked": "2026-09-19"
    },
    {
      "id": "plan-a-public-scorecards",
      "kind": "external",
      "statement": "Plan A’s epistemics supplement recommends that members of the public create scorecards of AI models’ epistemic virtue and that consumers and employees take them into account.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "AI for Epistemics",
        "url": "https://ai-2040.com/supplements/ai-for-epistemics"
      },
      "checked": "2026-09-21"
    },
    {
      "id": "lab-prediction-record",
      "kind": "programme",
      "statement": "The lab registers a prediction before each run and scores it afterwards against a no-theory guess; the figures are read from the lab’s own scorer.",
      "numbers": [],
      "unit": "prediction; the n, the share inside the stated range, the share that beat the naive guess and the mean margin are all rendered from scripts/lab-record.snapshot.json and checked by scripts/lab-record-audit.mjs.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/evidence/",
        "https://futurisminstitute.org/neighbours/"
      ],
      "source": {
        "title": "verifier_frontier predictions.py score",
        "url": null
      },
      "checked": "2026-09-21"
    },
    {
      "id": "lineage-gender-shades-benchmarks",
      "kind": "external",
      "statement": "The face-analysis benchmarks IJB-A and Adience were 79.6% and 86.2% lighter-skinned.",
      "numbers": [
        "79.6",
        "86.2"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Gender Shades: Intersectional Accuracy Disparities in Commercial Gender Classification",
        "url": "https://proceedings.mlr.press/v81/buolamwini18a/buolamwini18a.pdf"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-gender-shades-error",
      "kind": "external",
      "statement": "Darker-skinned women were misclassified up to 34.7% of the time; the largest error for lighter-skinned men was 0.8%.",
      "numbers": [
        "34.7",
        "0.8"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Gender Shades",
        "url": "https://proceedings.mlr.press/v81/buolamwini18a/buolamwini18a.pdf"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-gender-shades-ppb",
      "kind": "external",
      "statement": "The Pilot Parliaments Benchmark holds 1,270 parliamentarians from Rwanda, Senegal, South Africa, Iceland, Finland and Sweden, skin type labelled on the Fitzpatrick scale by a dermatologist, gender recorded as perceived.",
      "numbers": [
        "1,270"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Gender Shades",
        "url": "https://proceedings.mlr.press/v81/buolamwini18a/buolamwini18a.pdf"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-actionable-auditing",
      "kind": "external",
      "statement": "Seven months after the audit, error on darker-skinned women had fallen 34.7 to 16.97 (IBM), 20.8 to 1.52 (Microsoft), 34.5 to 4.1 (Face++); Amazon and Kairos stood at 31.37 and 22.50.",
      "numbers": [
        "16.97",
        "20.8",
        "1.52",
        "34.5",
        "4.1",
        "31.37",
        "22.50"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Actionable Auditing: Investigating the Impact of Publicly Naming Biased Performance Results of Commercial AI Products",
        "url": "https://www.aies-conference.com/2019/wp-content/uploads/2019/01/AIES-19_paper_223.pdf"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-ibm-2020",
      "kind": "external",
      "statement": "On 8 June 2020 IBM wrote to Congress that it no longer offers general purpose facial recognition or analysis software.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "IBM CEO letter to Congress, 8 June 2020 (archived)",
        "url": "https://web.archive.org/web/20210227110744/https://www.ibm.com/policy/facial-recognition-sunset-racial-justice-reforms/"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-amazon-2020",
      "kind": "external",
      "statement": "On 10 June 2020 Amazon announced a one-year moratorium on police use of Rekognition.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "We are implementing a one-year moratorium on police use of Rekognition",
        "url": "https://www.aboutamazon.com/news/policy-news-views/we-are-implementing-a-one-year-moratorium-on-police-use-of-rekognition"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-microsoft-2020",
      "kind": "external",
      "statement": "On 11 June 2020 Microsoft said it would not sell facial recognition to US police departments until a national law governs it.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Microsoft's Brad Smith says company will not sell facial recognition tech to police",
        "url": "https://techcrunch.com/2020/06/11/microsofts-brad-smith-says-company-will-not-sell-facial-recognition-tech-to-police/"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-documentation-debt",
      "kind": "external",
      "statement": "Stochastic Parrots names documentation debt: training datasets both undocumented and too large to document post hoc.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "On the Dangers of Stochastic Parrots: Can Language Models Be Too Big?",
        "url": "https://doi.org/10.1145/3442188.3445922"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-wef-ai-talent",
      "kind": "external",
      "statement": "Women were 29.4% of LinkedIn members listing AI engineering skills in 2025, up from 23.5% in 2018; the gap narrowed in 74 of the 75 economies examined.",
      "numbers": [
        "29.4",
        "23.5",
        "74",
        "75"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Gender Parity in the Intelligent Age",
        "url": "https://reports.weforum.org/docs/WEF_Gender_Parity_in_the_Intelligent_Age_2025.pdf"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-element-ai-authors",
      "kind": "external",
      "statement": "Women were 18% of authors published at the 21 leading AI conferences in 2018.",
      "numbers": [
        "18%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Element AI Announces 2019 Global AI Talent Report",
        "url": "https://www.newswire.ca/news-releases/element-ai-announces-2019-global-ai-talent-report-833031229.html"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-nces-cs-degrees",
      "kind": "external",
      "statement": "Women earned 37.1% of US bachelor's degrees in computer and information sciences in 1983-84 and 22.6% in 2021-22.",
      "numbers": [
        "37.1",
        "22.6"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Digest of Education Statistics, table 325.35",
        "url": "https://nces.ed.gov/programs/digest/d23/tables/dt23_325.35.asp"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-uis-ict-graduates",
      "kind": "external",
      "statement": "Women were 46.0% of tertiary ICT graduates in Malaysia and 46.3% in India in 2018, and 23.6% in the United States in 2016.",
      "numbers": [
        "46.0",
        "46.3",
        "23.6"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Female share of graduates in Information and Communication Technologies programmes, tertiary (%)",
        "url": "https://genderdata.worldbank.org/en/indicator/uis-fgp-5t8-f600"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-van-dijk-2012",
      "kind": "external",
      "statement": "A meta-analysis of 146 studies found the harm attributed to demographic diversity may come from rater bias; it did not appear with more objective measures of performance.",
      "numbers": [
        "146"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Defying conventional wisdom: A meta-analytical examination of the differences between demographic and job-related diversity relationships with performance",
        "url": "https://repub.eur.nl/pub/37816"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-vasilescu-2015",
      "kind": "external",
      "statement": "On GitHub, gender and tenure diversity were positive and significant predictors of productivity.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "Gender and Tenure Diversity in GitHub Teams",
        "url": "https://cmustrudel.github.io/papers/chi15.pdf"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "lineage-green-hand-2024",
      "kind": "external",
      "statement": "Using the S&P 500, Green and Hand did not find McKinsey's link between executives' racial and ethnic diversity and profit margins.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/lineage/"
      ],
      "source": {
        "title": "McKinsey's Diversity Matters/Delivers/Wins Results Revisited",
        "url": "https://econjwatch.org/articles/mckinsey-s-diversity-matters-delivers-wins-results-revisited"
      },
      "checked": "2026-09-23"
    },
    {
      "id": "so-2025-almost-right",
      "kind": "external",
      "statement": "In Stack Overflow’s 2025 Developer Survey, 66% of the 31,476 people who answered the question on AI frustrations chose \"AI solutions that are almost right, but not quite\", the most chosen option.",
      "numbers": [
        "66%",
        "31,476"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Stack Overflow Developer Survey 2025, AI section",
        "url": "https://survey.stackoverflow.co/2025/ai"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "anthropic-81k-concerns",
      "kind": "external",
      "statement": "Anthropic interviewed 80,508 Claude users in December 2025; asked how AI could be developed against what they value, 16.3% raised cognitive atrophy, which Anthropic defines as over-reliance causing skill loss or a decline in critical thinking.",
      "numbers": [
        "80,508",
        "16.3%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "What 81,000 people want from AI",
        "url": "https://www.anthropic.com/features/81k-interviews"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "frontier-detection-obedience",
      "kind": "programme",
      "statement": "On one injected instruction placed as the last text in the prompt, Claude Sonnet 5 obeys 0.80 and detects 1.00; Opus 4.8 obeys 0.00 and detects 1.00; Haiku 4.5 obeys 0.00 and detects 0.59. On a nested ladder of neutral text after the instruction (13 August 2026), Sonnet 5 wrote the injected word in 66, 57, 52, 21, 0 and 1 of 80 attempts at none, 25, 50, 100, 200 and 400 characters, task-level rates 0.825, 0.713, 0.650, 0.263, 0.000 and 0.013; four same-prompt replications read 0.738, 0.787, 0.825 and 0.850.",
      "numbers": [
        "0.80",
        "1.00",
        "0.59",
        "0.713",
        "0.650",
        "0.263",
        "0.000",
        "0.738",
        "0.787",
        "0.825",
        "0.850",
        "0.013",
        "66 of 80",
        "57 of 80",
        "52 of 80",
        "21 of 80",
        "0 of 80",
        "1 of 80"
      ],
      "unit": "task; n = 20 tasks per model, four samples each, false alarms 0.00 for all three models; the trailing-text ladder is 20 tasks of four attempts per rung, 80 attempts, drawn attempt by attempt on /findings/.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "frontier-repro v1.1.0, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "oversight-channel",
      "kind": "programme",
      "statement": "Every monitor-side defence the lab tested that left attacker text in front of the monitor backfired or was broken; the one that held showed the monitor only the answer’s first integer.",
      "numbers": [
        "0.7151",
        "0.7300",
        "0.0030",
        "0.0059",
        "1.0000",
        "0.9970",
        "0.8160",
        "0.9941",
        "0.4184",
        "0.0089",
        "0.0742",
        "0.0623",
        "0.8991",
        "0.0208",
        "0.1039",
        "0.0178",
        "0.4807",
        "0.5163",
        "0.4955",
        "0.4748",
        "0.6973",
        "0.7181"
      ],
      "unit": "task; n = 337 per cell, the search’s screening cells n = 48; every result on two task pools unless marked one pool; monitors gemma4:12b, gemma4:31b and a majority panel adding qwen3.6:35b-a3b; arithmetic and lookup tasks.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "frontier-repro v1.1.0, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "monitor-label-correction",
      "kind": "programme",
      "statement": "frontier-repro v1.0.1 quoted a reviewing model’s discrimination falling from 0.335 to 0.107 under labelling, from cells run before a harness defect was fixed; the re-run in v1.1.0 reads 0.555 to 0.003.",
      "numbers": [
        "0.335",
        "0.107",
        "0.555",
        "0.003"
      ],
      "unit": "task; 337 items, gemma4:12b reviewing its own work (data/monitor_label_cells.json, python monitor_label.py).",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/evidence/"
      ],
      "source": {
        "title": "frontier-repro v1.1.0, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "frontier-selfreport-question",
      "kind": "programme",
      "statement": "Asked to audit a prompt, Claude Sonnet 5 reported the injected instruction every time; doing the task, it obeyed that instruction on 80% of tasks when it was the last text in the prompt.",
      "numbers": [
        "80%"
      ],
      "unit": "task; 20 tasks per model, four samples each, one vendor.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/questions/"
      ],
      "source": {
        "title": "frontier-repro v1.1.0, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "gemma-reserves",
      "kind": "programme",
      "statement": "Re-run with the template leak fixed, gemma4:12b’s unaided checking reads 0.042 (published 0.034), its reading with a correct peer reference 0.092 (published 0.072), and its pass rate on a false reference matching a corrupted answer 0.967 (published 0.969).",
      "numbers": [
        "0.042",
        "0.092",
        "0.967"
      ],
      "unit": "task; the same designs, tasks and cell sizes as the published July and August cells, served again after the 2026-09-17 fix to gemma4:12b’s template leak.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/",
        "https://bentleymoon.com/research/checking-ai/evidence/"
      ],
      "source": {
        "title": "verifier_frontier ledger, the re-served cells of 22 to 26 September 2026",
        "url": null
      },
      "checked": "2026-09-27"
    },
    {
      "id": "gemma-reserves-inversion",
      "kind": "programme",
      "statement": "Re-run with the template leak fixed, the inverted cell reads −0.620 (published −0.648), and the testable-predicate cell reads 0.536 as published.",
      "numbers": [
        "0.620"
      ],
      "unit": "task; the same designs, tasks and cell sizes as the published July and August cells, served again after the 2026-09-17 fix to gemma4:12b’s template leak.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/",
        "https://bentleymoon.com/research/checking-ai/evidence/"
      ],
      "source": {
        "title": "verifier_frontier ledger, the re-served cells of 22 to 26 September 2026",
        "url": null
      },
      "checked": "2026-09-27"
    },
    {
      "id": "gemma-reserves-ladder",
      "kind": "programme",
      "statement": "Re-run with the template leak fixed, gemma4:12b’s inversion crossing reads +0.250 at three digits and −0.743 at four (published +0.257 and −0.757).",
      "numbers": [
        "0.250",
        "0.743"
      ],
      "unit": "task; the same designs, tasks and cell sizes as the published July and August cells, served again after the 2026-09-17 fix to gemma4:12b’s template leak.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier ledger, the re-served cells of 22 to 26 September 2026",
        "url": null
      },
      "checked": "2026-09-27"
    },
    {
      "id": "panel-diversity-question",
      "kind": "programme",
      "statement": "A majority of three monitors from different families defeated a trigger aimed at one of them (0.0742 false alarms on correct work, against 0.8991 for gemma4:31b alone) and fell to a payload that fooled all three (0.9941).",
      "numbers": [
        "0.0742",
        "0.8991",
        "0.9941"
      ],
      "unit": "task; 337 items per cell, two task pools; monitors gemma4:12b, gemma4:31b and qwen3.6:35b-a3b.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/questions/"
      ],
      "source": {
        "title": "frontier-repro v1.1.0, public and re-derivable",
        "url": "https://github.com/BentleyMoon/frontier-repro"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "gemma-reserves-research",
      "kind": "programme",
      "statement": "Re-run with the template leak fixed on 2026-09-23, at the same n and seed, gemma4:12b’s arm of the which-model comparison reads 0.042 (published 0.034).",
      "numbers": [
        "0.042"
      ],
      "unit": "task; the same designs, tasks and cell sizes as the published July and August cells, served again after the 2026-09-17 fix to gemma4:12b’s template leak.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier ledger, the re-served cells of 22 to 26 September 2026",
        "url": null
      },
      "checked": "2026-09-27"
    },
    {
      "id": "inversion-ladder-drawn",
      "kind": "programme",
      "statement": "Signed discrimination by predicate width: gemma4:12b +0.257, −0.757, −0.752, −0.682, −0.722, −0.580 at 3 to 8 digits; gemma4:31b +0.884, +0.050, −0.417 at 3 to 5; qwen2.5-coder:32b +0.564, −0.116, −0.380; qwen2.5-coder:7b +0.002, +0.031, −0.068.",
      "numbers": [
        "0.752",
        "0.682",
        "0.722",
        "0.580",
        "0.417",
        "0.380",
        "0.002",
        "0.031",
        "0.068"
      ],
      "unit": "task; 424 tasks per rung, one predicate family, local models only.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/findings/"
      ],
      "source": {
        "title": "verifier_frontier ledger, the inversion ladder",
        "url": null
      },
      "checked": "2026-09-28"
    },
    {
      "id": "req-debate",
      "kind": "external",
      "statement": "Debate proposes two AI systems arguing before a judge; with optimal play it lets polynomial-time judges settle PSPACE questions, and whether it works in practice is an empirical question.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "AI safety via debate",
        "url": "https://arxiv.org/abs/1805.00899"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-prover-verifier",
      "kind": "external",
      "statement": "Legibility training against small verifiers increased checkability, and it transferred to time-constrained humans checking solutions.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Prover-Verifier Games improve legibility of LLM outputs",
        "url": "https://arxiv.org/abs/2407.13692"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-correlated-errors",
      "kind": "external",
      "statement": "Across more than 350 LLMs, models agree 60% of the time when both err on one leaderboard dataset; larger, more accurate models have highly correlated errors across architectures and providers.",
      "numbers": [
        "350",
        "60%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Correlated Errors in Large Language Models",
        "url": "https://arxiv.org/abs/2506.07962"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-great-models",
      "kind": "external",
      "statement": "LLM judges favour models similar to themselves, and model mistakes are becoming more similar as capability increases.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Great Models Think Alike and this Undermines AI Oversight",
        "url": "https://arxiv.org/abs/2502.04313"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-self-preference",
      "kind": "external",
      "statement": "LLM evaluators recognise their own generations and favour them, with a linear correlation between self-recognition and self-preference.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "LLM Evaluators Recognize and Favor Their Own Generations",
        "url": "https://arxiv.org/abs/2404.13076"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-ai-control",
      "kind": "external",
      "statement": "Safety protocols evaluated against an intentionally subversive model; trusted editing and untrusted monitoring improve substantially on simple baselines.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "AI Control: Improving Safety Despite Intentional Subversion",
        "url": "https://arxiv.org/abs/2312.06942"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-monitor-obfuscation",
      "kind": "external",
      "statement": "A weaker model monitoring chain of thought caught reward hacking; strong optimisation against the monitor produced obfuscated reward hacking, and the authors recommend a monitorability tax.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Monitoring Reasoning Models for Misbehavior and the Risks of Promoting Obfuscation",
        "url": "https://arxiv.org/abs/2503.11926"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-indirect-injection",
      "kind": "external",
      "statement": "Indirect prompt injection compromised real LLM-integrated applications including Bing GPT-4 chat; effective mitigations were lacking.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Not what you have signed up for: indirect prompt injection",
        "url": "https://arxiv.org/abs/2302.12173"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-instruction-hierarchy",
      "kind": "external",
      "statement": "An instruction hierarchy trained into GPT-3.5 drastically increases robustness with minimal degradation of capabilities.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "The Instruction Hierarchy",
        "url": "https://arxiv.org/abs/2404.13208"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-spotlighting",
      "kind": "external",
      "statement": "Spotlighting reduces indirect prompt injection attack success from greater than 50% to below 2% with GPT-family models.",
      "numbers": [
        "50%",
        "2%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Defending Against Indirect Prompt Injection Attacks With Spotlighting",
        "url": "https://arxiv.org/abs/2403.14720"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-camel",
      "kind": "external",
      "statement": "CaMeL separates control flow from untrusted data and solves 77% of AgentDojo tasks with provable security, against 84% undefended.",
      "numbers": [
        "77%",
        "84%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Defeating Prompt Injections by Design",
        "url": "https://arxiv.org/abs/2503.18813"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-hierarchy-breaks",
      "kind": "external",
      "statement": "Reasoning models can resolve an instruction conflict in reasoning and still produce a violating response; the strongest self-monitor reduces rule-following non-compliance by 81 to 99% across three models.",
      "numbers": [
        "81",
        "99%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Where Instruction Hierarchy Breaks",
        "url": "https://arxiv.org/abs/2606.07808"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-unfaithful-cot",
      "kind": "external",
      "statement": "Chain-of-thought explanations systematically misrepresent the reason for a prediction; accuracy drops by as much as 36% on 13 BIG-Bench Hard tasks.",
      "numbers": [
        "36%",
        "13"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Language Models Don’t Always Say What They Think",
        "url": "https://arxiv.org/abs/2305.04388"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-reasoning-faithfulness",
      "kind": "external",
      "statement": "Reasoning models reveal hint usage in at least 1% of cases but often below 20%; increased reward hacking does not increase verbalisation.",
      "numbers": [
        "20%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Reasoning Models Don’t Always Say What They Think",
        "url": "https://arxiv.org/abs/2505.05410"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-cot-monitorability",
      "kind": "external",
      "statement": "Chain-of-thought monitoring is a new and fragile opportunity for AI safety; like other known oversight methods it is imperfect.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Chain of Thought Monitorability: A New and Fragile Opportunity for AI Safety",
        "url": "https://arxiv.org/abs/2507.11473"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-metr-rct",
      "kind": "external",
      "statement": "In a randomised trial, 16 experienced developers completed 246 tasks; allowing AI increased completion time by 19% while the developers estimated it had reduced it by 20%.",
      "numbers": [
        "16",
        "246",
        "19%",
        "20%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity",
        "url": "https://arxiv.org/abs/2507.09089"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-so-less-confident",
      "kind": "external",
      "statement": "In Stack Overflow’s 2025 Developer Survey, 20% of respondents to the question on AI frustrations chose \"I’ve become less confident in my own problem-solving\".",
      "numbers": [
        "20%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Stack Overflow Developer Survey 2025, AI section",
        "url": "https://survey.stackoverflow.co/2025/ai"
      },
      "checked": "2026-09-27"
    },
    {
      "id": "req-fmti",
      "kind": "external",
      "statement": "The Foundation Model Transparency Index average fell from 58 of 100 in 2024 to 40 in 2025; companies are most opaque about training data, training compute and post-deployment usage and impact.",
      "numbers": [
        "58 of 100",
        "40 in 2025"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "The 2025 Foundation Model Transparency Index",
        "url": "https://arxiv.org/abs/2512.10169"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-compute-governance",
      "kind": "external",
      "statement": "AI-relevant compute is detectable, excludable and quantifiable; compute-based policies vary significantly in readiness for implementation.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Computing Power and the Governance of Artificial Intelligence",
        "url": "https://arxiv.org/abs/2402.08797"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "req-flexheg",
      "kind": "external",
      "statement": "Flexible hardware-enabled guarantees are proposed: an auditable guarantee processor beside AI accelerators; the first part of a three-part series.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "Flexible Hardware-Enabled Guarantees for AI Compute",
        "url": "https://arxiv.org/abs/2506.15093"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "requirements-lab",
      "kind": "programme",
      "statement": "The lab figures quoted under the requirements: the access table (1.000, 0.536, 0.034, 0.042 re-run), deference (0.969), which model (0.804), the monitor panel (0.0742, 0.8991, 0.9941), inversion (0.648, 0.620 re-run), labelling a reviewer (0.0059 to 1.0000) and Sonnet 5 obeying on 80% of tasks.",
      "numbers": [
        "1.000",
        "0.536",
        "0.034",
        "0.042",
        "0.969",
        "0.804",
        "0.0742",
        "0.8991",
        "0.9941",
        "0.648",
        "0.620",
        "0.0059",
        "1.0000",
        "80%"
      ],
      "unit": "task; each figure keeps the model, cell size and dates it carries on /findings/, where the same figures are registered with their units.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/directions/"
      ],
      "source": {
        "title": "The figures registered on /findings/ and in frontier-repro",
        "url": null
      },
      "checked": "2026-09-29"
    },
    {
      "id": "front-map",
      "kind": "programme",
      "statement": "The lab figures pinned to the front page’s drawing of one check: in the batch that went on to test text after it, Sonnet 5 followed an order planted last in its prompt in 66 of 80 attempts and in 0 of 80 with 200 characters after it (the headline run of the same prompt, 0.80, is on the findings page), and flagged it on all 20 tasks when asked; a reviewer’s first verdict under the untrusted label, 0.0059 to 1.0000 (0.0000 to 0.9970); the access table, 1.000 handed the value and 0.042 handed nothing (0.034 in July); inversion, 0.536 to 0.648 below zero (0.620 re-run); the three-monitor panel, 0.0742 against 0.8991 alone and 0.9941 under a shared payload.",
      "numbers": [
        "66 of 80",
        "0 of 80",
        "0.0059",
        "1.0000",
        "0.0000",
        "0.9970",
        "1.000",
        "0.042",
        "0.034",
        "0.536",
        "0.648",
        "0.620",
        "0.0742",
        "0.8991",
        "0.9941"
      ],
      "unit": "task or item; each figure keeps its model and cell size in the small print beside it (20 tasks, 337, 500 or 250 items), and the same figures are registered with their full units on /findings/.",
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "The figures registered on /findings/ and in frontier-repro",
        "url": null
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-iabied",
      "kind": "external",
      "statement": "The research overview opens on the problem as Eliezer Yudkowsky and Nate Soares state it in If Anyone Builds It, Everyone Dies, and paraphrases the book’s case in six steps: superhuman AI will be built, probably before long; today’s AI is grown, not crafted, and not understood inside; powerful AI behaves as if pursuing goals; current techniques cannot reliably give it the goals we want; by default its goals would be harmful to us; humanity could not defend itself.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "If Anyone Builds It, Everyone Dies: Why Superhuman AI Would Kill Us All",
        "url": "https://ifanyonebuildsit.com/"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-xpt",
      "kind": "external",
      "statement": "In the Existential Risk Persuasion Tournament (2022), median forecasts of human extinction by 2100 were: from AI, 3% by AI experts and 0.38% by superforecasters; from an engineered pathogen, 1% and 0.01%; from nuclear war, 0.55% and 0.074%; from a natural pathogen, 0.01% and 0.0018%; from non-anthropogenic causes, 0.004% and 0.0043%. 80 experts and 89 superforecasters took part.",
      "numbers": [
        "3%",
        "0.38%",
        "0.55%",
        "0.074%",
        "0.0018%",
        "0.004%",
        "0.0043%",
        "80 experts",
        "89 superforecasters"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "Forecasting Existential Risks: Evidence from a Long-Run Forecasting Tournament",
        "url": "https://forecastingresearch.org/xpt"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-ai-researchers",
      "kind": "external",
      "statement": "In a survey of 2,778 researchers who had published in top AI venues, conducted in autumn 2023, the median probability given to extremely bad outcomes such as human extinction from advanced AI was 5%.",
      "numbers": [
        "2,778",
        "5%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "Thousands of AI Authors on the Future of AI",
        "url": "https://arxiv.org/abs/2401.02843"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-hinton",
      "kind": "external",
      "statement": "Geoffrey Hinton said there is a 10% to 20% chance that AI leads to human extinction within three decades.",
      "numbers": [
        "10% to 20%"
      ],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "‘Godfather of AI’ shortens odds of the technology wiping out humanity over next 30 years",
        "url": "https://www.theguardian.com/technology/2024/dec/27/godfather-of-ai-raises-odds-of-the-technology-wiping-out-humanity-over-next-30-years"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-cais-statement",
      "kind": "external",
      "statement": "In 2023 the heads of OpenAI, Anthropic and Google DeepMind signed a statement that mitigating the risk of extinction from AI should be a global priority alongside other societal-scale risks such as pandemics and nuclear war; Geoffrey Hinton and Yoshua Bengio signed it too.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "Statement on AI Risk",
        "url": "https://www.safe.ai/work/statement-on-ai-risk"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-aisafety-com",
      "kind": "external",
      "statement": "AISafety.com describes itself as the hub for AI existential safety, with lists it keeps of events, communities, courses and more focused on preventing human extinction from AI, and a visual overview of the organizations in the field.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "AISafety.com",
        "url": "https://www.aisafety.com/"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-aisafety-info",
      "kind": "external",
      "statement": "AISafety.info is an AI safety FAQ with Stampy, a chatbot that points to its sources.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "AISafety.info",
        "url": "https://aisafety.info/"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-miri",
      "kind": "external",
      "statement": "MIRI says its researchers helped found the field of AI alignment, that superintelligence developed by anything remotely resembling current methods would most likely end in human extinction, and that survival depends on a globally coordinated moratorium.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "The Machine Intelligence Research Institute",
        "url": "https://intelligence.org/"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-metr",
      "kind": "external",
      "statement": "METR (Model Evaluation and Threat Research) informs the public about the risks and capabilities of advanced AI systems, including a time-horizon measure of how long a task they can complete.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "METR",
        "url": "https://metr.org/"
      },
      "checked": "2026-09-29"
    },
    {
      "id": "lai-lens",
      "kind": "external",
      "statement": "Lens Academy runs free, cohort-based AI safety courses from beginner to advanced, part-time or intensive, from understanding the biggest risks from advanced AI to doing something about them. Bentley has completed its courses and is now taking AI Control.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "Lens Academy",
        "url": "https://www.lensacademy.org/courses"
      },
      "checked": "2026-10-13"
    },
    {
      "id": "lai-book-forecast",
      "kind": "external",
      "statement": "If Anyone Builds It, Everyone Dies gives no percentage; its title states the forecast, that superhuman AI built with anything like current methods kills everyone.",
      "numbers": [],
      "unit": null,
      "pages": [
        "https://bentleymoon.com/research/checking-ai/"
      ],
      "source": {
        "title": "If Anyone Builds It, Everyone Dies: Why Superhuman AI Would Kill Us All",
        "url": "https://ifanyonebuildsit.com/"
      },
      "checked": "2026-09-29"
    }
  ],
  "withdrawn": [
    {
      "id": "correlated-verifier-shortfall",
      "statement": "Verifiers built to be separate share far more of their errors than their number implies, and the shortfall grows as systems get more capable.",
      "withdrawn": "2026-07-31",
      "reason": "No run in the lab produced it. It was presented under a heading reading “What was measured” and spread to four pages over two days before it was withdrawn. The arguments it supported now rest on measured results: requirement 1 and the first table on the findings page."
    }
  ],
  "corrections": [
    {
      "date": "2026-09-29",
      "label": "Corrected",
      "where": "legibleai.org, the findings and requirements pages",
      "was": "Labelling the reviewed text as untrusted took gemma4:31b’s first verdict from 0.0059 to 1.0000.",
      "now": "Those cells are gemma4:12b’s, reviewing with a yes-or-no verdict, as the lab’s own finding and the per-item record say. The figures stand; the model was wrong. Found while reading the released cells for the front page’s drawing."
    },
    {
      "date": "2026-09-26",
      "label": "Re-measured",
      "where": "the lab, for legibleai.org’s overview and findings",
      "was": "Every gemma4:12b figure on these pages, measured in July and August while that model was leaking hidden template text into its replies.",
      "now": "Re-run with the leak fixed between 22 and 26 September, each inside the interval the lab registered before the re-run: 0.034 reads 0.042, 0.072 reads 0.092, 0.969 reads 0.967 and −0.648 reads −0.620, while 1.000 and 0.536 read the same. Both readings stay on the pages."
    },
    {
      "date": "2026-09-22",
      "label": "Corrected",
      "where": "frontier-repro, from v1.0.1 to v1.1.0",
      "was": "Labelling the text a reviewing model reads took its discrimination from 0.335 to 0.107.",
      "now": "Those cells were run before a harness defect in the open-weight lane was fixed. The re-run points the same way with a larger effect, from 0.555 to 0.003, and is the figure the released record carries."
    },
    {
      "date": "2026-09-19",
      "label": "Corrected",
      "where": "legibleai.org, the overview",
      "was": "A 9B model out-verified a 12B by 0.77 on the same paired tasks, while a 31B of the 9B’s own family beat that same 12B by 0.92.",
      "now": "The 31B was gemma4:31b, from the 12B’s own family, so the sentence turned a result inside one family into one across two. The lab never claimed that arm: it rests on one cell. Only the pre-registered comparison stands."
    },
    {
      "date": "2026-09-19",
      "label": "Corrected",
      "where": "legibleai.org, the overview and the research map",
      "was": "Verification access: Measured.",
      "now": "Measured on one model, gemma4:12b, with each row’s n and tier on the page. The top row was reread the same day as deference to whatever the checker is told is correct, and that reading now sits beside the table."
    },
    {
      "date": "2026-09-19",
      "label": "Corrected",
      "where": "legibleai.org, the overview",
      "was": "A composition ladder testing exactly that is running now.",
      "now": "It had finished on 1 August, seven weeks earlier, and its results replaced the sentence. Since then no sentence here describes work as in flight; a result names the day it ran."
    }
  ]
}