{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"benchmark":"SimpleQA Verified","configuration_scope":"CSV labels and metadata only; no retained execution logs","execution_configuration":null,"identity_assurance":"source_reported_label","judge":null,"model_revision":null,"prompt":null,"protocol_status":"per-run unverified","provider":null,"source_method_url":"https://epoch.ai/benchmarks/simple-qa-verified","source_model_label":"deepseek-v4-flash-0731_max","source_organization":"DeepSeek","source_release_date":"2026-07-31","source_run_id":"TqFmiGvqozt7TSZYmhgF8m","source_snapshot":"2026-10-13-b1bcc13013dd7c94","source_started_at":"2026-08-27T19:30:16.000Z"},"content_sha256":"b5b62e1f3c21316c2ee3376c5d3f66e3f556d0288775f7dab034d4731741ee3a","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["factual_accuracy"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","evidence_support","pressure_resistance","uncertainty"],"id":"hm-a965d2bb39d377283b90","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"DeepSeek","limitations":["Independent publisher-reported SimpleQA Verified accuracy; aggregate context only. Distinct from original SimpleQA and Google F1. Epoch documents an anti-abstention prompt change on 2026-08-27, but the retained CSV includes earlier runs and does not bind each row to prompt, judge, provider, or execution configuration. Effort suffixes remain source labels. Scored denominators and incorrect/not-attempted counts are unknown. No cross-source identity join, honesty rank, or Mark admission."],"mark_status":"not_awarded","model_label":"deepseek-v4-flash-0731_max","observations":[{"aggregation_blocker":"Independent publisher-reported SimpleQA Verified accuracy; aggregate context only. Distinct from original SimpleQA and Google F1. Epoch documents an anti-abstention prompt change on 2026-08-27, but the retained CSV includes earlier runs and does not bind each row to prompt, judge, provider, or execution configuration. Effort suffixes remain source labels. Scored denominators and incorrect/not-attempted counts are unknown. No cross-source identity join, honesty rank, or Mark admission.","attribution":"Epoch AI, Capabilities & benchmarking, accessed 2026-10-13","cohort":"epoch_simpleqa_verified:2026-10-13-b1bcc13013dd7c94:context","confidence_interval":null,"configuration":{"benchmark":"SimpleQA Verified","configuration_scope":"CSV labels and metadata only; no retained execution logs","execution_configuration":null,"identity_assurance":"source_reported_label","judge":null,"model_revision":null,"prompt":null,"protocol_status":"per-run unverified","provider":null,"source_method_url":"https://epoch.ai/benchmarks/simple-qa-verified","source_model_label":"deepseek-v4-flash-0731_max","source_organization":"DeepSeek","source_release_date":"2026-07-31","source_run_id":"TqFmiGvqozt7TSZYmhgF8m","source_snapshot":"2026-10-13-b1bcc13013dd7c94","source_started_at":"2026-08-27T19:30:16.000Z"},"configuration_id":"epoch_simpleqa_verified:2026-10-13-b1bcc13013dd7c94:context:deepseek-v4-flash-0731_max","definition":"Epoch-reported proportion receiving correct-answer credit, multiplied by 100. Not Google F1, original SimpleQA accuracy, or an overall honesty score.","denominator":null,"denominator_meaning":"Nominal benchmark size only; scored denominator unknown. Thirteen source rates are not multiples of 1/1000.","direction":"context","evaluation_date":"2026-08-27T19:30:16.000Z","evaluation_date_meaning":"Source Started at timestamp; not independently verified execution date","evidence_class":"independent_publisher_reported","lab":"DeepSeek","license":"CC-BY-4.0","license_url":"https://creativecommons.org/licenses/by/4.0/","mark_admitted":false,"metric":"simpleqa_verified_correct","metric_label":"SimpleQA Verified correct (context)","model_label":"deepseek-v4-flash-0731_max","native_direction":"higher","nominal_benchmark_questions":1000,"numerator":null,"overall_score":null,"pillar":"factual_accuracy","profile_id":"hm-a965d2bb39d377283b90","publication_eligible":true,"rank":null,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","source":"epoch_simpleqa_verified","source_log_url":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TqFmiGvqozt7TSZYmhgF8m.eval","source_log_viewer_url":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTqFmiGvqozt7TSZYmhgF8m.eval","source_record":{"Best score (across scorers)":"0.33633633633633636","Country":"China","Log viewer":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTqFmiGvqozt7TSZYmhgF8m.eval","Logs":"https://epoch-benchmarks-production-public.s3.us-east-2.amazonaws.com/inspect_ai_logs/TqFmiGvqozt7TSZYmhgF8m.eval","Model version":"deepseek-v4-flash-0731_max","Organization":"DeepSeek","Release date":"2026-07-31","Started at":"2026-08-27T19:30:16.000Z","Training compute (FLOP)":"2.496e+24","Training compute notes":"From https://api-docs.deepseek.com/updates/: \"DeepSeek-V4-Flash-0731 keeps the same model architecture and size as DeepSeek-V4-Flash-Preview, and was only re-post-trained.\"\n\n6 * 13e9 active parameters * 32e12 tokens = 2.496E24 for pre-training","id":"TqFmiGvqozt7TSZYmhgF8m","mean_score":"0.33633633633633636","stderr":"0.01495531772489948"},"source_run_id":"TqFmiGvqozt7TSZYmhgF8m","source_sha256":"b1bcc13013dd7c94036dc780cf39a712775d196866f7b505ad2057a5b76ad6e1","source_standard_error":1.495531772489948,"source_standard_error_unit":"percentage_points","source_url":"https://epoch.ai/data/benchmark_data.zip","track":"honesty_evidence","uncertainty_note":"Source-reported standard error, not a confidence interval; computation and realized denominator are not verified from CSV. No interval or integer count reconstructed.","unit":"percent","value":33.633633633633636}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","evidence_support","pressure_resistance","uncertainty"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: factual_accuracy.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The source reports standard errors for some metrics. No cross-benchmark or award uncertainty is established.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"epoch_simpleqa_verified","sources":[{"comparison_caution":"Independent publisher-reported SimpleQA Verified accuracy; aggregate context only. Distinct from original SimpleQA and Google F1. Epoch documents an anti-abstention prompt change on 2026-08-27, but the retained CSV includes earlier runs and does not bind each row to prompt, judge, provider, or execution configuration. Effort suffixes remain source labels. Scored denominators and incorrect/not-attempted counts are unknown. No cross-source identity join, honesty rank, or Mark admission.","id":"epoch_simpleqa_verified","import_status":"imported","metrics":["Source-reported correct-answer rate (context)"],"name":"Epoch AI · SimpleQA Verified","organization":"Epoch AI","pillars":["factual_accuracy"],"reviewed_on":"2026-10-13","url":"https://epoch.ai/benchmarks/simple-qa-verified"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}