{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"benchmark":"SimpleQA Verified","configuration_scope":"CSV labels and metadata only; no retained execution logs","execution_configuration":null,"identity_assurance":"source_reported_label","judge":null,"model_revision":null,"prompt":null,"protocol_status":"per-run unverified","provider":null,"source_method_url":"https://epoch.ai/benchmarks/simple-qa-verified","source_model_label":"gemini-3.6-flash_high","source_organization":"Google DeepMind","source_release_date":"2026-07-21","source_run_id":"F83pa7Gfsb4UNVttTnS5Hk","source_snapshot":"2026-10-13-b1bcc13013dd7c94","source_started_at":"2026-08-27T19:30:09.000Z"},"content_sha256":"05237bd92cc64a75f73ec37cf1788ecdeee0b45e39d4bb0793818fc6e2ea4509","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["factual_accuracy"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","evidence_support","pressure_resistance","uncertainty"],"id":"hm-ada123459c4984f7fd3e","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"Google DeepMind","limitations":["Independent publisher-reported SimpleQA Verified accuracy; aggregate context only. Distinct from original SimpleQA and Google F1. Epoch documents an anti-abstention prompt change on 2026-08-27, but the retained CSV includes earlier runs and does not bind each row to prompt, judge, provider, or execution configuration. Effort suffixes remain source labels. Scored denominators and incorrect/not-attempted counts are unknown. No cross-source identity join, honesty rank, or Mark admission."],"mark_status":"not_awarded","model_label":"gemini-3.6-flash_high","observations":[{"aggregation_blocker":"Independent publisher-reported SimpleQA Verified accuracy; aggregate context only. Distinct from original SimpleQA and Google F1. Epoch documents an anti-abstention prompt change on 2026-08-27, but the retained CSV includes earlier runs and does not bind each row to prompt, judge, provider, or execution configuration. Effort suffixes remain source labels. Scored denominators and incorrect/not-attempted counts are unknown. No cross-source identity join, honesty rank, or Mark admission.","attribution":"Epoch AI, Capabilities & benchmarking, accessed 2026-10-13","cohort":"epoch_simpleqa_verified:2026-10-13-b1bcc13013dd7c94:context","confidence_interval":null,"configuration":{"benchmark":"SimpleQA Verified","configuration_scope":"CSV labels and metadata only; no retained execution logs","execution_configuration":null,"identity_assurance":"source_reported_label","judge":null,"model_revision":null,"prompt":null,"protocol_status":"per-run unverified","provider":null,"source_method_url":"https://epoch.ai/benchmarks/simple-qa-verified","source_model_label":"gemini-3.6-flash_high","source_organization":"Google DeepMind","source_release_date":"2026-07-21","source_run_id":"F83pa7Gfsb4UNVttTnS5Hk","source_snapshot":"2026-10-13-b1bcc13013dd7c94","source_started_at":"2026-08-27T19:30:09.000Z"},"configuration_id":"epoch_simpleqa_verified:2026-10-13-b1bcc13013dd7c94:context:gemini-3.6-flash_high","definition":"Epoch-reported proportion receiving correct-answer credit, multiplied by 100. Not Google F1, original SimpleQA accuracy, or an overall honesty score.","denominator":null,"denominator_meaning":"Nominal benchmark size only; scored denominator unknown. Thirteen source rates are not multiples of 1/1000.","direction":"context","evaluation_date":"2026-08-27T19:30:09.000Z","evaluation_date_meaning":"Source Started at timestamp; not independently verified execution date","evidence_class":"independent_publisher_reported","lab":"Google DeepMind","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","mark_admitted":false,"metric":"simpleqa_verified_correct","metric_label":"SimpleQA Verified correct (context)","model_label":"gemini-3.6-flash_high","native_direction":"higher","nominal_benchmark_questions":1000,"numerator":null,"overall_score":null,"pillar":"factual_accuracy","profile_id":"hm-ada123459c4984f7fd3e","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"epoch_simpleqa_verified","source_log_url":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/F83pa7Gfsb4UNVttTnS5Hk.eval","source_log_viewer_url":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF83pa7Gfsb4UNVttTnS5Hk.eval","source_record":{"Best score (across scorers)":"0.662","Country":"United States of America","Log viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF83pa7Gfsb4UNVttTnS5Hk.eval","Logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/F83pa7Gfsb4UNVttTnS5Hk.eval","Model version":"gemini-3.6-flash_high","Organization":"Google DeepMind","Release date":"2026-07-21","Started at":"2026-08-27T19:30:09.000Z","Training compute (FLOP)":"","Training compute notes":"","id":"F83pa7Gfsb4UNVttTnS5Hk","mean_score":"0.662","stderr":"0.014965960710224383"},"source_run_id":"F83pa7Gfsb4UNVttTnS5Hk","source_sha256":"b1bcc13013dd7c94036dc780cf39a712775d196866f7b505ad2057a5b76ad6e1","source_standard_error":1.4965960710224382,"source_standard_error_unit":"percentage_points","source_url":"https://epoch.ai/data/benchmark_data.zip","track":"honesty_evidence","uncertainty_note":"Source-reported standard error, not a confidence interval; computation and realized denominator are not verified from CSV. No interval or integer count reconstructed.","unit":"percent","value":66.2}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","evidence_support","pressure_resistance","uncertainty"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: factual_accuracy.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The source reports standard errors for some metrics. No cross-benchmark or award uncertainty is established.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"epoch_simpleqa_verified","sources":[{"comparison_caution":"Independent publisher-reported SimpleQA Verified accuracy; aggregate context only. Distinct from original SimpleQA and Google F1. Epoch documents an anti-abstention prompt change on 2026-08-27, but the retained CSV includes earlier runs and does not bind each row to prompt, judge, provider, or execution configuration. Effort suffixes remain source labels. Scored denominators and incorrect/not-attempted counts are unknown. No cross-source identity join, honesty rank, or Mark admission.","id":"epoch_simpleqa_verified","import_status":"imported","metrics":["Source-reported correct-answer rate (context)"],"name":"Epoch AI · SimpleQA Verified","organization":"Epoch AI","pillars":["factual_accuracy"],"reviewed_on":"2026-10-13","url":"https://epoch.ai/benchmarks/simple-qa-verified"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}