{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"benchmark":"TruthfulQA","configuration_scope":"Historical source-reported configuration; exact execution metadata unavailable","dataset_version":"v0","decoding":"greedy; temperature 0","grader":"Human evaluation by authors; separate v0 GPT-judge context column","judge_revision":null,"model_revision":null,"prompt":"default-qa","source_commit":"d71c110897f5d31c5d7f309e7bc316c152f6f031","source_method_sha256":"5833edb650c9412436c5823912e6252bc94ef7d39610c566b65788f13afbbae7","source_method_url":"https://arxiv.org/html/2109.07958v1","source_model_label":"GPT-2 1.5B","study_year":2021,"task":"generation"},"content_sha256":"b6a58e59f9b4c5924894fd1fd8b1696317b798ed0e752e33b52279b34305833a","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["factual_accuracy"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","evidence_support","pressure_resistance","uncertainty"],"id":"hm-e57af43af9886261cfbc","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"OpenAI","limitations":["Historical 2021 v0 generation baselines on misconception-focused questions. Truthfulness credits non-answers; informativeness must be read alongside it. Marginal truth/info rates do not establish the joint truthful-and-informative rate. Default QA prompting and UnifiedQA without a prefix are separate cohorts. GPT-judge is an overlapping automatic assessment, not an independent replication or a calibrated confidence estimate. The 2025 binary task and updated-reference baseline table are excluded. These are authors-reported aggregates, not current-model rankings."],"mark_status":"not_awarded","model_label":"GPT-2 1.5B","observations":[{"aggregation_blocker":"Historical 2021 v0 generation baselines on misconception-focused questions. Truthfulness credits non-answers; informativeness must be read alongside it. Marginal truth/info rates do not establish the joint truthful-and-informative rate. Default QA prompting and UnifiedQA without a prefix are separate cohorts. GPT-judge is an overlapping automatic assessment, not an independent replication or a calibrated confidence estimate. The 2025 binary task and updated-reference baseline table are excluded. These are authors-reported aggregates, not current-model rankings.","attribution":"Stephanie Lin, Jacob Hilton and Owain Evans; TruthfulQA authors","cohort":"truthfulqa:d71c110897f5d31c5d7f309e7bc316c152f6f031:v0-generation:default-qa","confidence_interval":null,"configuration":{"benchmark":"TruthfulQA","configuration_scope":"Historical source-reported configuration; exact execution metadata unavailable","dataset_version":"v0","decoding":"greedy; temperature 0","grader":"Human evaluation by authors; separate v0 GPT-judge context column","judge_revision":null,"model_revision":null,"prompt":"default-qa","source_commit":"d71c110897f5d31c5d7f309e7bc316c152f6f031","source_method_sha256":"5833edb650c9412436c5823912e6252bc94ef7d39610c566b65788f13afbbae7","source_method_url":"https://arxiv.org/html/2109.07958v1","source_model_label":"GPT-2 1.5B","study_year":2021,"task":"generation"},"configuration_id":"truthfulqa:d71c110897f5d31c5d7f309e7bc316c152f6f031:v0-generation:default-qa:GPT-2 1.5B","definition":"Source v0 GPT-judge percentage classified true on the same generation task. Shown as evaluator context; not an independent honesty vote or grader accuracy against humans.","denominator":null,"denominator_meaning":"Source benchmark has 817 questions; per-row realized judged counts not supplied in the table. No integer successes reconstructed from rounded rates.","direction":"context","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/sylinrl/TruthfulQA/blob/d71c110897f5d31c5d7f309e7bc316c152f6f031/LICENSE","metric":"gpt_judge_truthfulness","metric_label":"GPT-judge truthfulness (context)","model_label":"GPT-2 1.5B","nominal_benchmark_questions":817,"numerator":null,"overall_score":null,"pillar":"context","profile_id":"hm-e57af43af9886261cfbc","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"truthfulqa","source_sha256":"23834ca77c34e909cbdcceeaea89135c65b713a5d0c77be6bced7a637763ea44","source_url":"https://github.com/sylinrl/TruthfulQA/blob/d71c110897f5d31c5d7f309e7bc316c152f6f031/README.md#updates","split":"Historical v0 combined filtered and unfiltered questions","study_year":2021,"track":"honesty_evidence","uncertainty_note":"No per-row interval or response-level labels supplied; no interval reconstructed from rounded aggregates on an adversarial benchmark.","unit":"percent","value":29.87},{"aggregation_blocker":"Historical 2021 v0 generation baselines on misconception-focused questions. Truthfulness credits non-answers; informativeness must be read alongside it. Marginal truth/info rates do not establish the joint truthful-and-informative rate. Default QA prompting and UnifiedQA without a prefix are separate cohorts. GPT-judge is an overlapping automatic assessment, not an independent replication or a calibrated confidence estimate. The 2025 binary task and updated-reference baseline table are excluded. These are authors-reported aggregates, not current-model rankings.","attribution":"Stephanie Lin, Jacob Hilton and Owain Evans; TruthfulQA authors","cohort":"truthfulqa:d71c110897f5d31c5d7f309e7bc316c152f6f031:v0-generation:default-qa","confidence_interval":null,"configuration":{"benchmark":"TruthfulQA","configuration_scope":"Historical source-reported configuration; exact execution metadata unavailable","dataset_version":"v0","decoding":"greedy; temperature 0","grader":"Human evaluation by authors; separate v0 GPT-judge context column","judge_revision":null,"model_revision":null,"prompt":"default-qa","source_commit":"d71c110897f5d31c5d7f309e7bc316c152f6f031","source_method_sha256":"5833edb650c9412436c5823912e6252bc94ef7d39610c566b65788f13afbbae7","source_method_url":"https://arxiv.org/html/2109.07958v1","source_model_label":"GPT-2 1.5B","study_year":2021,"task":"generation"},"configuration_id":"truthfulqa:d71c110897f5d31c5d7f309e7bc316c152f6f031:v0-generation:default-qa:GPT-2 1.5B","definition":"Source human-rated percentage of informative answers. An informative answer may be false; this is not the joint truthful-and-informative rate.","denominator":null,"denominator_meaning":"Source benchmark has 817 questions; per-row realized judged counts not supplied in the table. No integer successes reconstructed from rounded rates.","direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/sylinrl/TruthfulQA/blob/d71c110897f5d31c5d7f309e7bc316c152f6f031/LICENSE","metric":"human_informativeness","metric_label":"Human-rated informativeness","model_label":"GPT-2 1.5B","nominal_benchmark_questions":817,"numerator":null,"overall_score":null,"pillar":"usefulness","profile_id":"hm-e57af43af9886261cfbc","publication_eligible":true,"rank":3,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"truthfulqa","source_sha256":"23834ca77c34e909cbdcceeaea89135c65b713a5d0c77be6bced7a637763ea44","source_url":"https://github.com/sylinrl/TruthfulQA/blob/d71c110897f5d31c5d7f309e7bc316c152f6f031/README.md#updates","split":"Historical v0 combined filtered and unfiltered questions","study_year":2021,"track":"honesty_evidence","uncertainty_note":"No per-row interval or response-level labels supplied; no interval reconstructed from rounded aggregates on an adversarial benchmark.","unit":"percent","value":89.84},{"aggregation_blocker":"Historical 2021 v0 generation baselines on misconception-focused questions. Truthfulness credits non-answers; informativeness must be read alongside it. Marginal truth/info rates do not establish the joint truthful-and-informative rate. Default QA prompting and UnifiedQA without a prefix are separate cohorts. GPT-judge is an overlapping automatic assessment, not an independent replication or a calibrated confidence estimate. The 2025 binary task and updated-reference baseline table are excluded. These are authors-reported aggregates, not current-model rankings.","attribution":"Stephanie Lin, Jacob Hilton and Owain Evans; TruthfulQA authors","cohort":"truthfulqa:d71c110897f5d31c5d7f309e7bc316c152f6f031:v0-generation:default-qa","confidence_interval":null,"configuration":{"benchmark":"TruthfulQA","configuration_scope":"Historical source-reported configuration; exact execution metadata unavailable","dataset_version":"v0","decoding":"greedy; temperature 0","grader":"Human evaluation by authors; separate v0 GPT-judge context column","judge_revision":null,"model_revision":null,"prompt":"default-qa","source_commit":"d71c110897f5d31c5d7f309e7bc316c152f6f031","source_method_sha256":"5833edb650c9412436c5823912e6252bc94ef7d39610c566b65788f13afbbae7","source_method_url":"https://arxiv.org/html/2109.07958v1","source_model_label":"GPT-2 1.5B","study_year":2021,"task":"generation"},"configuration_id":"truthfulqa:d71c110897f5d31c5d7f309e7bc316c152f6f031:v0-generation:default-qa:GPT-2 1.5B","definition":"Source human-rated percentage of answers avoiding false assertions. Noncommittal responses can count as true; this is not useful-answer accuracy.","denominator":null,"denominator_meaning":"Source benchmark has 817 questions; per-row realized judged counts not supplied in the table. No integer successes reconstructed from rounded rates.","direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/sylinrl/TruthfulQA/blob/d71c110897f5d31c5d7f309e7bc316c152f6f031/LICENSE","metric":"human_truthfulness","metric_label":"Human-rated truthfulness","model_label":"GPT-2 1.5B","nominal_benchmark_questions":817,"numerator":null,"overall_score":null,"pillar":"factual_accuracy","profile_id":"hm-e57af43af9886261cfbc","publication_eligible":true,"rank":1,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"truthfulqa","source_sha256":"23834ca77c34e909cbdcceeaea89135c65b713a5d0c77be6bced7a637763ea44","source_url":"https://github.com/sylinrl/TruthfulQA/blob/d71c110897f5d31c5d7f309e7bc316c152f6f031/README.md#updates","split":"Historical v0 combined filtered and unfiltered questions","study_year":2021,"track":"honesty_evidence","uncertainty_note":"No per-row interval or response-level labels supplied; no interval reconstructed from rounded aggregates on an adversarial benchmark.","unit":"percent","value":29.5}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","evidence_support","pressure_resistance","uncertainty"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: factual_accuracy.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"truthfulqa","sources":[{"comparison_caution":"Historical 2021 v0 generation baselines on misconception-focused questions. Truthfulness credits non-answers; informativeness must be read alongside it. Marginal truth/info rates do not establish the joint truthful-and-informative rate. Default QA prompting and UnifiedQA without a prefix are separate cohorts. GPT-judge is an overlapping automatic assessment, not an independent replication or a calibrated confidence estimate. The 2025 binary task and updated-reference baseline table are excluded. These are authors-reported aggregates, not current-model rankings.","id":"truthfulqa","import_status":"imported","metrics":["Human-rated truthfulness","Human-rated informativeness","GPT-judge truthfulness (context)"],"name":"TruthfulQA · historical v0 (2021)","organization":"TruthfulQA authors","pillars":["factual_accuracy"],"reviewed_on":"2026-10-11","url":"https://arxiv.org/abs/2109.07958"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}