{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"api_endpoint":null,"benchmark":"ConfidenceBench","confidence_format":"A-D choice plus probability 0-100; normalized probability for binary-correctness Brier","ece_bins":"10 equal-width bins","effort_assurance":"High/Standard/Low are source variant labels, not reconstructed API reasoning_effort strings","identity_assurance":"source_reported_label","max_tokens":null,"model_revision":null,"nominal_questions_per_run":200,"publication_version":"2607.20526v1","refusal_protocol":"Source says random answer at p=.25; Table 2 n labels pooled non-refused; unresolved population interpretation","source_method_url":"https://arxiv.org/html/2607.20526v1#S3","source_model_label":"Claude Sonnet 4.6","source_reported_provider":"Anthropic","source_reported_runs":3,"source_reported_temperature":1.0,"source_snapshot_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","table":"Appendix A, Table 2"},"content_sha256":"fa4ce2c8e36dd2618446f4a840663f26b85313411e2888716fe49aed13f5305b","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["factual_accuracy","uncertainty"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","evidence_support","pressure_resistance"],"id":"hm-88ae90da7063e04932ff","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"Anthropic","limitations":["Author-reported three-run means on 200 private multiple-choice questions. Reported SD is run-to-run variation, not SE or a confidence interval. Sections 3.2/5.3 state refusals and unparseable outputs receive random-choice probability .25, while Table 2 calls n pooled non-refused responses. Actual scoring population is unresolved; n is contextual only. No source code, execution, private questions or per-run denominators reconstructed. No aliases or full Mark admission; paper model labels retain effort variants."],"mark_status":"not_awarded","model_label":"Claude Sonnet 4.6","observations":[{"aggregation_blocker":"Author-reported three-run means on 200 private multiple-choice questions. Reported SD is run-to-run variation, not SE or a confidence interval. Sections 3.2/5.3 state refusals and unparseable outputs receive random-choice probability .25, while Table 2 calls n pooled non-refused responses. Actual scoring population is unresolved; n is contextual only. No source code, execution, private questions or per-run denominators reconstructed. No aliases or full Mark admission; paper model labels retain effort variants.","attribution":"ffrench-Constant et al., arXiv:2607.20526v1, Table 2","cohort":"confidencebench:2607.20526v1:table-2:three-run-means","cohort_label":"ConfidenceBench · 2026 Table 2 · three-run means","confidence_interval":null,"configuration":{"api_endpoint":null,"benchmark":"ConfidenceBench","confidence_format":"A-D choice plus probability 0-100; normalized probability for binary-correctness Brier","ece_bins":"10 equal-width bins","effort_assurance":"High/Standard/Low are source variant labels, not reconstructed API reasoning_effort strings","identity_assurance":"source_reported_label","max_tokens":null,"model_revision":null,"nominal_questions_per_run":200,"publication_version":"2607.20526v1","refusal_protocol":"Source says random answer at p=.25; Table 2 n labels pooled non-refused; unresolved population interpretation","source_method_url":"https://arxiv.org/html/2607.20526v1#S3","source_model_label":"Claude Sonnet 4.6","source_reported_provider":"Anthropic","source_reported_runs":3,"source_reported_temperature":1.0,"source_snapshot_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","table":"Appendix A, Table 2"},"configuration_id":"confidencebench:2607.20526v1:table-2:three-run-means:Claude Sonnet 4.6","cross_source_join_eligible":false,"definition":"Published three-run mean accuracy on the private four-choice benchmark. Refusal replacement and table sample-count semantics remain unresolved.","denominator":null,"denominator_meaning":"Run-specific scoring denominators unknown; mean is not reconstructed as pooled correct/n.","direction":"context","evidence_class":"benchmark_author_reported","lab":"Anthropic","license":"factual_records_only; no article redistribution permission asserted","mark_admitted":false,"metric":"accuracy","metric_label":"Answer accuracy (context)","model_label":"Claude Sonnet 4.6","native_direction":"higher","nominal_benchmark_questions":200,"nominal_runs":3,"numerator":null,"overall_score":null,"pillar":"factual_accuracy","profile_id":"hm-88ae90da7063e04932ff","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"confidencebench","source_document_sha256":"a61672069683613f128242551a2c6ca63b5005aff980c008887d921387008849","source_locator":"Appendix A, Table 2","source_nonrefused_n":600,"source_nonrefused_n_meaning":"Table 2 pooled non-refused count across three runs; not the denominator of this run-mean metric.","source_record":{"accuracy":{"mean":"0.690","sd":"0.010"},"brier":{"mean":"0.145","sd":"0.001"},"ece":{"mean":"0.096","sd":"0.013"},"model":"Claude Sonnet 4.6","reported_nonrefused_n":600},"source_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","source_standard_deviation":0.01,"source_standard_deviation_unit":"fraction","source_url":"https://arxiv.org/html/2607.20526v1#A1.T2","track":"honesty_evidence","uncertainty_note":"Source run-to-run SD across three runs; not standard error or confidence interval. Shared questions and metrics are dependent.","unit":"fraction","value":0.69},{"aggregation_blocker":"Author-reported three-run means on 200 private multiple-choice questions. Reported SD is run-to-run variation, not SE or a confidence interval. Sections 3.2/5.3 state refusals and unparseable outputs receive random-choice probability .25, while Table 2 calls n pooled non-refused responses. Actual scoring population is unresolved; n is contextual only. No source code, execution, private questions or per-run denominators reconstructed. No aliases or full Mark admission; paper model labels retain effort variants.","attribution":"ffrench-Constant et al., arXiv:2607.20526v1, Table 2","cohort":"confidencebench:2607.20526v1:table-2:three-run-means","cohort_label":"ConfidenceBench · 2026 Table 2 · three-run means","confidence_interval":null,"configuration":{"api_endpoint":null,"benchmark":"ConfidenceBench","confidence_format":"A-D choice plus probability 0-100; normalized probability for binary-correctness Brier","ece_bins":"10 equal-width bins","effort_assurance":"High/Standard/Low are source variant labels, not reconstructed API reasoning_effort strings","identity_assurance":"source_reported_label","max_tokens":null,"model_revision":null,"nominal_questions_per_run":200,"publication_version":"2607.20526v1","refusal_protocol":"Source says random answer at p=.25; Table 2 n labels pooled non-refused; unresolved population interpretation","source_method_url":"https://arxiv.org/html/2607.20526v1#S3","source_model_label":"Claude Sonnet 4.6","source_reported_provider":"Anthropic","source_reported_runs":3,"source_reported_temperature":1.0,"source_snapshot_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","table":"Appendix A, Table 2"},"configuration_id":"confidencebench:2607.20526v1:table-2:three-run-means:Claude Sonnet 4.6","cross_source_join_eligible":false,"definition":"Published three-run mean of squared error between stated correctness probability and binary answer correctness. Native 0-1 scale, not multiclass Brier or an honesty percentage.","denominator":null,"denominator_meaning":"Run-specific scoring denominators unknown; mean is not reconstructed as pooled correct/n.","direction":"context","evidence_class":"benchmark_author_reported","lab":"Anthropic","license":"factual_records_only; no article redistribution permission asserted","mark_admitted":false,"metric":"brier","metric_label":"Brier score (context)","model_label":"Claude Sonnet 4.6","native_direction":"lower","nominal_benchmark_questions":200,"nominal_runs":3,"numerator":null,"overall_score":null,"pillar":"uncertainty","profile_id":"hm-88ae90da7063e04932ff","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"confidencebench","source_document_sha256":"a61672069683613f128242551a2c6ca63b5005aff980c008887d921387008849","source_locator":"Appendix A, Table 2","source_nonrefused_n":600,"source_nonrefused_n_meaning":"Table 2 pooled non-refused count across three runs; not the denominator of this run-mean metric.","source_record":{"accuracy":{"mean":"0.690","sd":"0.010"},"brier":{"mean":"0.145","sd":"0.001"},"ece":{"mean":"0.096","sd":"0.013"},"model":"Claude Sonnet 4.6","reported_nonrefused_n":600},"source_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","source_standard_deviation":0.001,"source_standard_deviation_unit":"fraction","source_url":"https://arxiv.org/html/2607.20526v1#A1.T2","track":"honesty_evidence","uncertainty_note":"Source run-to-run SD across three runs; not standard error or confidence interval. Shared questions and metrics are dependent.","unit":"fraction","value":0.145},{"aggregation_blocker":"Author-reported three-run means on 200 private multiple-choice questions. Reported SD is run-to-run variation, not SE or a confidence interval. Sections 3.2/5.3 state refusals and unparseable outputs receive random-choice probability .25, while Table 2 calls n pooled non-refused responses. Actual scoring population is unresolved; n is contextual only. No source code, execution, private questions or per-run denominators reconstructed. No aliases or full Mark admission; paper model labels retain effort variants.","attribution":"ffrench-Constant et al., arXiv:2607.20526v1, Table 2","cohort":"confidencebench:2607.20526v1:table-2:three-run-means","cohort_label":"ConfidenceBench · 2026 Table 2 · three-run means","confidence_interval":null,"configuration":{"api_endpoint":null,"benchmark":"ConfidenceBench","confidence_format":"A-D choice plus probability 0-100; normalized probability for binary-correctness Brier","ece_bins":"10 equal-width bins","effort_assurance":"High/Standard/Low are source variant labels, not reconstructed API reasoning_effort strings","identity_assurance":"source_reported_label","max_tokens":null,"model_revision":null,"nominal_questions_per_run":200,"publication_version":"2607.20526v1","refusal_protocol":"Source says random answer at p=.25; Table 2 n labels pooled non-refused; unresolved population interpretation","source_method_url":"https://arxiv.org/html/2607.20526v1#S3","source_model_label":"Claude Sonnet 4.6","source_reported_provider":"Anthropic","source_reported_runs":3,"source_reported_temperature":1.0,"source_snapshot_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","table":"Appendix A, Table 2"},"configuration_id":"confidencebench:2607.20526v1:table-2:three-run-means:Claude Sonnet 4.6","cross_source_join_eligible":false,"definition":"Published three-run mean ECE using 10 equal-width probability bins. Native 0-1 scale; bin-dependent and distinct from Brier.","denominator":null,"denominator_meaning":"Run-specific scoring denominators unknown; mean is not reconstructed as pooled correct/n.","direction":"context","evidence_class":"benchmark_author_reported","lab":"Anthropic","license":"factual_records_only; no article redistribution permission asserted","mark_admitted":false,"metric":"ece","metric_label":"Expected Calibration Error (context)","model_label":"Claude Sonnet 4.6","native_direction":"lower","nominal_benchmark_questions":200,"nominal_runs":3,"numerator":null,"overall_score":null,"pillar":"uncertainty","profile_id":"hm-88ae90da7063e04932ff","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"confidencebench","source_document_sha256":"a61672069683613f128242551a2c6ca63b5005aff980c008887d921387008849","source_locator":"Appendix A, Table 2","source_nonrefused_n":600,"source_nonrefused_n_meaning":"Table 2 pooled non-refused count across three runs; not the denominator of this run-mean metric.","source_record":{"accuracy":{"mean":"0.690","sd":"0.010"},"brier":{"mean":"0.145","sd":"0.001"},"ece":{"mean":"0.096","sd":"0.013"},"model":"Claude Sonnet 4.6","reported_nonrefused_n":600},"source_sha256":"211b71103e05c475714213357731e3a1cdf41e73fcc3099f3ae2d968e33a83fc","source_standard_deviation":0.013,"source_standard_deviation_unit":"fraction","source_url":"https://arxiv.org/html/2607.20526v1#A1.T2","track":"honesty_evidence","uncertainty_note":"Source run-to-run SD across three runs; not standard error or confidence interval. Shared questions and metrics are dependent.","unit":"fraction","value":0.096}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","evidence_support","pressure_resistance"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: factual_accuracy, uncertainty.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"confidencebench","sources":[{"comparison_caution":"Author-reported three-run means on 200 private multiple-choice questions. Reported SD is run-to-run variation, not SE or a confidence interval. Sections 3.2/5.3 state refusals and unparseable outputs receive random-choice probability .25, while Table 2 calls n pooled non-refused responses. Actual scoring population is unresolved; n is contextual only. No source code, execution, private questions or per-run denominators reconstructed. No aliases or full Mark admission; paper model labels retain effort variants.","id":"confidencebench","import_status":"imported","metrics":["Brier score (context)","Answer accuracy (context)","Expected Calibration Error (context)"],"name":"ConfidenceBench","organization":"ConfidenceBench authors","pillars":["uncertainty","factual_accuracy"],"reviewed_on":"2026-10-13","url":"https://arxiv.org/html/2607.20526v1#A1.T2"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}