{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"benchmark":"AA-Omniscience","configuration_scope":"Paper chart labels, including visible effort labels; no exact endpoint attestation","judge":"Gemini 2.5 Flash Preview (09-2025), reasoning","model_revision":null,"nominal_questions":6000,"paper_version":"2511.13029v1","provider":null,"source_model_label":"Claude 4.5 Sonnet","source_prompt":"Paper Appendix A.1; instructed to answer only if confident","source_repetitions":"One full pass; separate repeatability experiment not pooled here","tools":"none"},"content_sha256":"5e4e07a94f999beb479a9792be19bbdbc319cc17f3fef650ad4b38c3144e26d8","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["factual_accuracy","uncertainty"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","evidence_support","pressure_resistance"],"id":"hm-38ec66165fcf5bf625b3","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"Anthropic","limitations":["Historical paper chart labels, not the current website leaderboard. Whole-number rounding retained. Index rewards correct answers and penalizes incorrect answers; always abstaining scores zero. Hallucination rate is incorrect/(partial+incorrect+not attempted), not incorrect/all or incorrect/attempted. Single-pass author-judged results, no calibration confidence claims, no Mark admission."],"mark_status":"not_awarded","model_label":"Claude 4.5 Sonnet","observations":[{"aggregation_blocker":"Historical paper chart labels, not the current website leaderboard. Whole-number rounding retained. Index rewards correct answers and penalizes incorrect answers; always abstaining scores zero. Hallucination rate is incorrect/(partial+incorrect+not attempted), not incorrect/all or incorrect/attempted. Single-pass author-judged results, no calibration confidence claims, no Mark admission.","attribution":"Declan Jackson, William Keating, George Cameron and Micah Hill-Smith, AA-Omniscience (2025), arXiv:2511.13029v1","cohort":"aa_omniscience:paper:2511.13029v1","cohort_label":"AA-Omniscience · 2025 paper v1 · Figures 1 and 7","confidence_interval":null,"configuration":{"benchmark":"AA-Omniscience","configuration_scope":"Paper chart labels, including visible effort labels; no exact endpoint attestation","judge":"Gemini 2.5 Flash Preview (09-2025), reasoning","model_revision":null,"nominal_questions":6000,"paper_version":"2511.13029v1","provider":null,"source_model_label":"Claude 4.5 Sonnet","source_prompt":"Paper Appendix A.1; instructed to answer only if confident","source_repetitions":"One full pass; separate repeatability experiment not pooled here","tools":"none"},"configuration_id":"aa_omniscience:paper:2511.13029v1:Claude 4.5 Sonnet","definition":"100*incorrect/(partial+incorrect+not_attempted). Denominator excludes correct answers; not the fraction incorrect among attempted or all questions.","denominator":null,"denominator_meaning":"Source aggregate; no realized row counts or integer events reconstructed from rounded values.","direction":"context","display_decimal_places":0,"evaluation_date":null,"evidence_class":"benchmark_authors_reported","identity_assurance":"source_reported_label","lab":"Anthropic","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","mark_admitted":false,"metric":"hallucination_rate","metric_label":"Hallucination rate (paper chart)","model_label":"Claude 4.5 Sonnet","native_direction":"lower","nominal_benchmark_questions":6000,"numerator":null,"overall_score":null,"pillar":"uncertainty","profile_id":"hm-38ec66165fcf5bf625b3","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"aa_omniscience","source_artifact":"Artificial_Analysis_Omniscience_Hallucination_Rate.png","source_locator":"Figure 7","source_paper_version":"2511.13029v1","source_rounding":"Whole-number chart label; do not infer decimal precision or integer event counts","source_sha256":"082b8c01c76027bd2e9b556b41592d1be355170520bd254eb3e792edfc9aa080","source_url":"https://arxiv.org/html/2511.13029v1","source_value_text":"48","study_year":2025,"track":"honesty_evidence","uncertainty_note":"Rounded source aggregate; no per-row interval supplied in the selected table or figure. No inferential interval reconstructed.","unit":"percent","value":48},{"aggregation_blocker":"Historical paper chart labels, not the current website leaderboard. Whole-number rounding retained. Index rewards correct answers and penalizes incorrect answers; always abstaining scores zero. Hallucination rate is incorrect/(partial+incorrect+not attempted), not incorrect/all or incorrect/attempted. Single-pass author-judged results, no calibration confidence claims, no Mark admission.","attribution":"Declan Jackson, William Keating, George Cameron and Micah Hill-Smith, AA-Omniscience (2025), arXiv:2511.13029v1","cohort":"aa_omniscience:paper:2511.13029v1","cohort_label":"AA-Omniscience · 2025 paper v1 · Figures 1 and 7","confidence_interval":null,"configuration":{"benchmark":"AA-Omniscience","configuration_scope":"Paper chart labels, including visible effort labels; no exact endpoint attestation","judge":"Gemini 2.5 Flash Preview (09-2025), reasoning","model_revision":null,"nominal_questions":6000,"paper_version":"2511.13029v1","provider":null,"source_model_label":"Claude 4.5 Sonnet","source_prompt":"Paper Appendix A.1; instructed to answer only if confident","source_repetitions":"One full pass; separate repeatability experiment not pooled here","tools":"none"},"configuration_id":"aa_omniscience:paper:2511.13029v1:Claude 4.5 Sonnet","definition":"100*(correct-incorrect)/(correct+partial+incorrect+not_attempted). Always abstaining yields0; no rescaling to honesty.","denominator":null,"denominator_meaning":"Source aggregate; no realized row counts or integer events reconstructed from rounded values.","direction":"context","display_decimal_places":0,"evaluation_date":null,"evidence_class":"benchmark_authors_reported","identity_assurance":"source_reported_label","lab":"Anthropic","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","mark_admitted":false,"metric":"omniscience_index","metric_label":"Omniscience Index (paper chart)","model_label":"Claude 4.5 Sonnet","native_direction":"higher","nominal_benchmark_questions":6000,"numerator":null,"overall_score":null,"pillar":"factual_accuracy","profile_id":"hm-38ec66165fcf5bf625b3","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scale_max":100,"scale_min":-100,"source":"aa_omniscience","source_artifact":"Artificial_Analysis_Omniscience_Index.png","source_locator":"Figure 1","source_paper_version":"2511.13029v1","source_rounding":"Whole-number chart label; do not infer decimal precision or integer event counts","source_sha256":"fd5ac0bb129c4f9843f259d6c5dddd1a996869e3ceea115e6965d829488a750f","source_url":"https://arxiv.org/html/2511.13029v1","source_value_text":"-2","study_year":2025,"track":"honesty_evidence","uncertainty_note":"Rounded source aggregate; no per-row interval supplied in the selected table or figure. No inferential interval reconstructed.","unit":"index_points","value":-2}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","evidence_support","pressure_resistance"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: factual_accuracy, uncertainty.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"aa_omniscience","sources":[{"comparison_caution":"Historical paper chart labels, not the current website leaderboard. Whole-number rounding retained. Index rewards correct answers and penalizes incorrect answers; always abstaining scores zero. Hallucination rate is incorrect/(partial+incorrect+not attempted), not incorrect/all or incorrect/attempted. Single-pass author-judged results, no calibration confidence claims, no Mark admission.","id":"aa_omniscience","import_status":"imported","metrics":["Hallucination rate (paper chart)","Omniscience Index (paper chart)"],"name":"AA-Omniscience · paper v1 (2025)","organization":"Artificial Analysis","pillars":["factual_accuracy","uncertainty"],"reviewed_on":"2026-10-13","url":"https://arxiv.org/html/2511.13029v1"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}