{
 "tool": "nulltest",
 "version": "1.1",
 "n_perm": 600,
 "generated_from": "nulltest_audit_manifest.json",
 "rows": [
  {
   "name": "aperture/verifydomain-claims-147",
   "paper": "Aperture / Project Forty Two \u2014 the battery behind our own front-page headline, withdrawn",
   "label_kind": "true_vs_false_statement",
   "url": "https://www.honesty.tools/verifier/batteries/verifydomain-claims-147.json",
   "status": "scored",
   "sha256": "ed742e764a2f2e14f71ae6f4da5e7a88b2870529e75a6646ed804b709aa54d14",
   "bytes": 42247,
   "n": 147,
   "n_fake": 36,
   "n_real": 111,
   "cells": 3,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 0.967,
    "adj_p": 0.0017,
    "raw_p": 0.0017,
    "null": "lexicality",
    "stratum": "claim",
    "n": 147,
    "threshold": 0.654,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.967 in lexicality/claim (n=147), family-adjusted p = 0.0017 over 3 cells",
   "separating_cells": 3,
   "astar": 0.967,
   "strength": "severe"
  },
  {
   "name": "aperture/grounding-coordinate",
   "paper": "Aperture / Project Forty Two \u2014 published null-floor certificate (ours)",
   "label_kind": "real_vs_fabricated_entity",
   "url": "https://www.honesty.tools/verifier/batteries/grounding-coordinate.json",
   "status": "scored",
   "sha256": "3e1dcce261a7f9dd7b4a09bcd0943f130b2a1ef7c3b71d7139e3f2f035dd9608",
   "bytes": 25124,
   "n": 162,
   "n_fake": 81,
   "n_real": 81,
   "cells": 18,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 0.855,
    "adj_p": 0.03,
    "raw_p": 0.0033,
    "null": "length",
    "stratum": "eponymous_signs",
    "n": 28,
    "threshold": 0.708,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.855 in length/eponymous_signs (n=28), family-adjusted p = 0.03 over 18 cells",
   "separating_cells": 1,
   "astar": 0.855,
   "strength": "substantial"
  },
  {
   "name": "aperture/photon-entities",
   "paper": "Aperture / Project Forty Two \u2014 the battery we used for months (ours)",
   "label_kind": "real_vs_fabricated_entity",
   "url": "https://www.honesty.tools/verifier/batteries/photon-entities.json",
   "status": "scored",
   "sha256": "f97e87443535a036dcb2cd0240bbec6dec4025f8d618687d86f6cafc63a088cb",
   "bytes": 6527,
   "n": 103,
   "n_fake": 27,
   "n_real": 76,
   "cells": 12,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 1.0,
    "adj_p": 0.0033,
    "raw_p": 0.0033,
    "null": "length",
    "stratum": "astronomy",
    "n": 19,
    "threshold": 0.792,
    "underpowered": false
   },
   "headline": "worst cell A* = 1.0 in length/astronomy (n=19), family-adjusted p = 0.0033 over 12 cells",
   "separating_cells": 1,
   "astar": 1.0,
   "strength": "severe"
  },
  {
   "name": "aperture/prominence-people-v2",
   "paper": "Aperture / Project Forty Two \u2014 the first battery of ours that clears its own surface floor",
   "label_kind": "known_vs_unknown_real_entity",
   "url": "https://www.honesty.tools/verifier/batteries/prominence-people-v2.json",
   "status": "scored",
   "sha256": "d93ad2b7975d10f005507ef9c33dbc7afc1b0ce240a5d956bbad7d025b01bd91",
   "bytes": 35945,
   "n": 156,
   "n_fake": 78,
   "n_real": 78,
   "cells": 3,
   "verdict": "PASS",
   "worst_cell": {
    "astar": 0.554,
    "adj_p": 0.8186,
    "raw_p": 0.4043,
    "null": "lexicality",
    "stratum": "person",
    "n": 156,
    "threshold": 0.616,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.554 in lexicality/person (n=156), family-adjusted p = 0.8186 over 3 cells",
   "separating_cells": 0,
   "astar": 0.554,
   "strength": "\u2014"
  },
  {
   "name": "aperture/prominence-people-v1",
   "paper": "Aperture / Project Forty Two \u2014 a battery that PASSED this floor and was still broken",
   "label_kind": "known_vs_unknown_real_entity",
   "url": "https://www.honesty.tools/verifier/batteries/prominence-people-v1.json",
   "status": "scored",
   "sha256": "fd881034ccd02904dfae2296435ebc651ccce63b8d408333f018550ceb154322",
   "bytes": 38590,
   "n": 168,
   "n_fake": 84,
   "n_real": 84,
   "cells": 3,
   "verdict": "PASS",
   "worst_cell": {
    "astar": 0.602,
    "adj_p": 0.2246,
    "raw_p": 0.0815,
    "null": "shape",
    "stratum": "person",
    "n": 168,
    "threshold": 0.616,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.602 in shape/person (n=168), family-adjusted p = 0.2246 over 3 cells",
   "separating_cells": 0,
   "astar": 0.602,
   "strength": "\u2014"
  },
  {
   "name": "aperture/postcutoff-trials-v1",
   "paper": "Aperture / Project Forty Two \u2014 surface-blind by construction: BOTH classes real, split by registry date",
   "label_kind": "known_vs_unknown_real_entity",
   "url": "https://www.honesty.tools/verifier/batteries/postcutoff-trials-v1.json",
   "status": "scored",
   "sha256": "54a6dbd8569af454a81521f5916886e0abfb3c14e4f05bef9cb6ec2f9db236a3",
   "bytes": 131270,
   "n": 702,
   "n_fake": 351,
   "n_real": 351,
   "cells": 3,
   "verdict": "PASS",
   "worst_cell": {
    "astar": 0.526,
    "adj_p": 0.792,
    "raw_p": 0.4093,
    "null": "lexicality",
    "stratum": "trial_title",
    "n": 702,
    "threshold": 0.559,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.526 in lexicality/trial_title (n=702), family-adjusted p = 0.792 over 3 cells",
   "separating_cells": 0,
   "astar": 0.526,
   "strength": "\u2014"
  },
  {
   "name": "bielik-entity-familiarity-v2",
   "paper": "Graded Entity-Familiarity Readouts (arXiv:2607.13568, 2026)",
   "label_kind": "real_vs_fabricated_entity",
   "url": "https://raw.githubusercontent.com/agentGreg/bielik-entity-familiarity/main/data/entities.csv",
   "status": "scored",
   "sha256": "02546469b730cb173e16dabba926e08e6c5094a734cdb7efb383d2c1322d8573",
   "bytes": 180206,
   "n": 1440,
   "n_fake": 240,
   "n_real": 1200,
   "cells": 12,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 0.953,
    "adj_p": 0.0017,
    "raw_p": 0.0017,
    "null": "lexicality",
    "stratum": "cities",
    "n": 360,
    "threshold": 0.611,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.953 in lexicality/cities (n=360), family-adjusted p = 0.0017 over 12 cells",
   "separating_cells": 5,
   "astar": 0.953,
   "strength": "severe"
  },
  {
   "name": "hallucinated-references/gpt-4",
   "paper": "Do Language Models Know When They're Hallucinating References? (arXiv:2305.18248, EACL Findings 2024)",
   "label_kind": "real_vs_fabricated_entity",
   "url": "https://raw.githubusercontent.com/microsoft/hallucinated-references/main/data/gpt-4_results.csv",
   "status": "scored",
   "sha256": "3fd32d5e9dc1657a381b94f88817b64b4a0e7bb007107f4db18b3206cdc7a22f",
   "bytes": 2789567,
   "n": 1000,
   "n_fake": 468,
   "n_real": 532,
   "cells": 3,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 0.836,
    "adj_p": 0.0017,
    "raw_p": 0.0017,
    "null": "lexicality",
    "stratum": "_all",
    "n": 1000,
    "threshold": 0.553,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.836 in lexicality/_all (n=1000), family-adjusted p = 0.0017 over 3 cells",
   "separating_cells": 3,
   "astar": 0.836,
   "strength": "substantial"
  },
  {
   "name": "geometry-of-truth/cities",
   "paper": "The Geometry of Truth (arXiv:2310.06824, COLM 2024)",
   "label_kind": "true_vs_false_statement",
   "url": "https://raw.githubusercontent.com/saprmarks/geometry-of-truth/main/datasets/cities.csv",
   "status": "scored",
   "sha256": "f2560a0c1758c69935b50125111b574b71896bb75930dd87b1d8c973c1fa086e",
   "bytes": 95041,
   "n": 1496,
   "n_fake": 748,
   "n_real": 748,
   "cells": 3,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 0.765,
    "adj_p": 0.0017,
    "raw_p": 0.0017,
    "null": "lexicality",
    "stratum": "_all",
    "n": 1496,
    "threshold": 0.54,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.765 in lexicality/_all (n=1496), family-adjusted p = 0.0017 over 3 cells",
   "separating_cells": 1,
   "astar": 0.765,
   "strength": "substantial"
  },
  {
   "name": "geometry-of-truth/sp_en_trans",
   "paper": "The Geometry of Truth (arXiv:2310.06824, COLM 2024)",
   "label_kind": "true_vs_false_statement",
   "url": "https://raw.githubusercontent.com/saprmarks/geometry-of-truth/main/datasets/sp_en_trans.csv",
   "status": "scored",
   "sha256": "9afa97c1b8af539a05f47655a80a5c7832c6fdb914885fd24cb4bcccbe2795b5",
   "bytes": 15231,
   "n": 354,
   "n_fake": 177,
   "n_real": 177,
   "cells": 3,
   "verdict": "PASS",
   "worst_cell": {
    "astar": 0.519,
    "adj_p": 0.9052,
    "raw_p": 0.5374,
    "null": "length",
    "stratum": "_all",
    "n": 354,
    "threshold": 0.559,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.519 in length/_all (n=354), family-adjusted p = 0.9052 over 3 cells",
   "separating_cells": 0,
   "astar": 0.519,
   "strength": "\u2014"
  },
  {
   "name": "truth-is-universal/facts",
   "paper": "Truth is Universal (arXiv:2407.12831, NeurIPS 2024)",
   "label_kind": "true_vs_false_statement",
   "url": "https://raw.githubusercontent.com/sciai-lab/Truth_is_Universal/main/datasets/facts.csv",
   "status": "scored",
   "sha256": "303ec926c2710a13a6166e204a24ea013bb2991b8761e2899739fddcb923aee1",
   "bytes": 32912,
   "n": 561,
   "n_fake": 282,
   "n_real": 279,
   "cells": 3,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 0.565,
    "adj_p": 0.0216,
    "raw_p": 0.0083,
    "null": "length",
    "stratum": "_all",
    "n": 561,
    "threshold": 0.548,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.565 in length/_all (n=561), family-adjusted p = 0.0216 over 3 cells",
   "separating_cells": 1,
   "astar": 0.565,
   "strength": "slight"
  },
  {
   "name": "aperture/knowledge-prominence-v1",
   "paper": "Aperture / Project Forty Two",
   "label_kind": "known_vs_unknown_real_entity",
   "url": "https://www.honesty.tools/verifier/batteries/knowledge-prominence-v1.json",
   "status": "scored",
   "sha256": "955778ddef49b4c6fd3e330e0704add067413475772b082c55816048f5e6aeea",
   "bytes": 49942,
   "n": 324,
   "n_fake": 162,
   "n_real": 162,
   "cells": 15,
   "verdict": "PASS",
   "worst_cell": {
    "astar": 0.655,
    "adj_p": 0.228,
    "raw_p": 0.02,
    "null": "length",
    "stratum": "place",
    "n": 80,
    "threshold": 0.625,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.655 in length/place (n=80), family-adjusted p = 0.228 over 15 cells",
   "separating_cells": 0,
   "astar": 0.655,
   "strength": "\u2014"
  },
  {
   "name": "aperture/knowledge-prominence-v2",
   "paper": "Aperture / Project Forty Two",
   "label_kind": "known_vs_unknown_real_entity",
   "url": "https://www.honesty.tools/verifier/batteries/knowledge-prominence-v2.json",
   "status": "scored",
   "sha256": "be110cd56e479a469706f46f8dcd5fee4e8814d841c3e686cf30819796c5d49f",
   "bytes": 44695,
   "n": 296,
   "n_fake": 148,
   "n_real": 148,
   "cells": 15,
   "verdict": "PASS",
   "worst_cell": {
    "astar": 0.689,
    "adj_p": 0.0982,
    "raw_p": 0.0083,
    "null": "lexicality",
    "stratum": "place",
    "n": 80,
    "threshold": 0.647,
    "underpowered": false
   },
   "headline": "worst cell A* = 0.689 in lexicality/place (n=80), family-adjusted p = 0.0982 over 15 cells",
   "separating_cells": 0,
   "astar": 0.689,
   "strength": "\u2014"
  },
  {
   "name": "aperture/calibration-battery-v1",
   "paper": "Aperture / Project Forty Two \u2014 the battery we invited other labs to calibrate against",
   "label_kind": "real_vs_fabricated_entity",
   "url": "https://www.honesty.tools/verifier/batteries/calibration-battery-v1.json",
   "status": "scored",
   "sha256": "a31e55da3dc94f317f8e1886fcec47c6c60de4f9492e95c11aab1ed7297069d0",
   "bytes": 25937,
   "n": 168,
   "n_fake": 84,
   "n_real": 84,
   "cells": 18,
   "verdict": "DISQUALIFIED",
   "worst_cell": {
    "astar": 1.0,
    "adj_p": 0.0017,
    "raw_p": 0.0017,
    "null": "lexicality",
    "stratum": "book",
    "n": 28,
    "threshold": 0.762,
    "underpowered": false
   },
   "headline": "worst cell A* = 1.0 in lexicality/book (n=28), family-adjusted p = 0.0017 over 18 cells",
   "separating_cells": 16,
   "astar": 1.0,
   "strength": "severe"
  }
 ]
}