{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"abstention_judge":"Llama 3.1 8B Instruct","averaging":"Paper-reported average across datasets; exact weighting not reconstructed","benchmark":"AbstentionBench","configuration_scope":"Source table label; no date alias expansion from another table or repository","model_revision":null,"paper_version":"2506.09038v1","provider":null,"source_method":"Appendix C: open models typically temperature0.8/top-p0.95/4k generation; API models defaults; o1 temperature1; exceptions apply","source_model_label":"Llama 3.1 8B Tulu 3 PPO RLVF","table":"Appendix D Table 4"},"content_sha256":"a4662f879f638d835d0efee1c0670568e8d8facb71a4c9467948b01aa1ff9cdc","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["factual_accuracy","uncertainty"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","evidence_support","pressure_resistance"],"id":"hm-970e5a9028f008a2a50e","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"Allen Institute for AI","limitations":["Historical paper Table4 averages, not the noncommercial repository result CSV. Accuracy on answerable questions and abstention recall on should-abstain questions have different populations. Source excludes invalid correctness-judge outputs; no pooled counts or joint utility reconstructed. Average weighting and per-model realized counts are not recoverable from this table. No overall ranking or Mark admission."],"mark_status":"not_awarded","model_label":"Llama 3.1 8B Tulu 3 PPO RLVF","observations":[{"aggregation_blocker":"Historical paper Table4 averages, not the noncommercial repository result CSV. Accuracy on answerable questions and abstention recall on should-abstain questions have different populations. Source excludes invalid correctness-judge outputs; no pooled counts or joint utility reconstructed. Average weighting and per-model realized counts are not recoverable from this table. No overall ranking or Mark admission.","attribution":"Polina Kirichenko, Mark Ibrahim, Kamalika Chaudhuri and Samuel J. Bell, AbstentionBench (2025), arXiv:2506.09038v1","cohort":"abstentionbench:paper:2506.09038v1","cohort_label":"AbstentionBench · 2025 paper v1 · Table 4 averages","confidence_interval":null,"configuration":{"abstention_judge":"Llama 3.1 8B Instruct","averaging":"Paper-reported average across datasets; exact weighting not reconstructed","benchmark":"AbstentionBench","configuration_scope":"Source table label; no date alias expansion from another table or repository","model_revision":null,"paper_version":"2506.09038v1","provider":null,"source_method":"Appendix C: open models typically temperature0.8/top-p0.95/4k generation; API models defaults; o1 temperature1; exceptions apply","source_model_label":"Llama 3.1 8B Tulu 3 PPO RLVF","table":"Appendix D Table 4"},"configuration_id":"abstentionbench:paper:2506.09038v1:Llama 3.1 8B Tulu 3 PPO RLVF","definition":"Paper Table4 average recall for identifying questions that should receive abstention; not precision, useful-answer accuracy, or the rate of abstaining on every request.","denominator":null,"denominator_meaning":"Source aggregate; no realized row counts or integer events reconstructed from rounded values.","direction":"context","evaluation_date":null,"evidence_class":"benchmark_authors_reported","identity_assurance":"source_reported_label","lab":"Allen Institute for AI","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","mark_admitted":false,"metric":"average_abstention_recall","metric_label":"Average abstention recall","model_label":"Llama 3.1 8B Tulu 3 PPO RLVF","native_direction":"higher","numerator":null,"overall_score":null,"pillar":"uncertainty","profile_id":"hm-970e5a9028f008a2a50e","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"abstentionbench","source_artifact":"abstention-table.html","source_locator":"Appendix D Table 4","source_paper_version":"2506.09038v1","source_sha256":"b60de6c1439013a759c2daf82f5ee5e1e36571d3427805ccb2b9c187b0147229","source_url":"https://arxiv.org/html/2506.09038v1","source_value_text":"0.51","source_value_unit":"fraction","study_year":2025,"track":"honesty_evidence","uncertainty_note":"Rounded source aggregate; no per-row interval supplied in the selected table or figure. No inferential interval reconstructed.","unit":"percent","value":51.0},{"aggregation_blocker":"Historical paper Table4 averages, not the noncommercial repository result CSV. Accuracy on answerable questions and abstention recall on should-abstain questions have different populations. Source excludes invalid correctness-judge outputs; no pooled counts or joint utility reconstructed. Average weighting and per-model realized counts are not recoverable from this table. No overall ranking or Mark admission.","attribution":"Polina Kirichenko, Mark Ibrahim, Kamalika Chaudhuri and Samuel J. Bell, AbstentionBench (2025), arXiv:2506.09038v1","cohort":"abstentionbench:paper:2506.09038v1","cohort_label":"AbstentionBench · 2025 paper v1 · Table 4 averages","confidence_interval":null,"configuration":{"abstention_judge":"Llama 3.1 8B Instruct","averaging":"Paper-reported average across datasets; exact weighting not reconstructed","benchmark":"AbstentionBench","configuration_scope":"Source table label; no date alias expansion from another table or repository","model_revision":null,"paper_version":"2506.09038v1","provider":null,"source_method":"Appendix C: open models typically temperature0.8/top-p0.95/4k generation; API models defaults; o1 temperature1; exceptions apply","source_model_label":"Llama 3.1 8B Tulu 3 PPO RLVF","table":"Appendix D Table 4"},"configuration_id":"abstentionbench:paper:2506.09038v1:Llama 3.1 8B Tulu 3 PPO RLVF","definition":"Paper Table4 average correctness on answerable/non-abstention questions where reference answers exist. Invalid correctness-judge outputs filtered; not all scheduled questions.","denominator":null,"denominator_meaning":"Source aggregate; no realized row counts or integer events reconstructed from rounded values.","direction":"context","evaluation_date":null,"evidence_class":"benchmark_authors_reported","identity_assurance":"source_reported_label","lab":"Allen Institute for AI","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","mark_admitted":false,"metric":"average_accuracy","metric_label":"Average answerable-question accuracy","model_label":"Llama 3.1 8B Tulu 3 PPO RLVF","native_direction":"higher","numerator":null,"overall_score":null,"pillar":"factual_accuracy","profile_id":"hm-970e5a9028f008a2a50e","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"abstentionbench","source_artifact":"abstention-table.html","source_locator":"Appendix D Table 4","source_paper_version":"2506.09038v1","source_sha256":"b60de6c1439013a759c2daf82f5ee5e1e36571d3427805ccb2b9c187b0147229","source_url":"https://arxiv.org/html/2506.09038v1","source_value_text":"0.75","source_value_unit":"fraction","study_year":2025,"track":"honesty_evidence","uncertainty_note":"Rounded source aggregate; no per-row interval supplied in the selected table or figure. No inferential interval reconstructed.","unit":"percent","value":75.0}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","evidence_support","pressure_resistance"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: factual_accuracy, uncertainty.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"abstentionbench","sources":[{"comparison_caution":"Historical paper Table4 averages, not the noncommercial repository result CSV. Accuracy on answerable questions and abstention recall on should-abstain questions have different populations. Source excludes invalid correctness-judge outputs; no pooled counts or joint utility reconstructed. Average weighting and per-model realized counts are not recoverable from this table. No overall ranking or Mark admission.","id":"abstentionbench","import_status":"imported","metrics":["Average abstention recall","Average answerable-question accuracy"],"name":"AbstentionBench · paper Table 4 (2025)","organization":"AbstentionBench authors / Meta research","pillars":["factual_accuracy","uncertainty"],"reviewed_on":"2026-10-13","url":"https://arxiv.org/html/2506.09038v1"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}