{"alignment_dimensions_with_observations":[],"alignment_dimensions_without_observations":["authorized_goal_following","constraint_compliance","non_deception","oversight_and_correction","robustness"],"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"content_sha256":"af3ae34008a6aa60b50b2610434e386d845211fef1fca1bb293dc5ba6033584a","coverage_meaning":"At least one observation, not complete or validated dimension coverage.","evidence_status":"recorded","honesty_dimensions_with_observations":["evidence_support","uncertainty"],"honesty_dimensions_without_observations":["belief_revision","citation_integrity","factual_accuracy","pressure_resistance"],"id":"hm-f88aadd2968dc59601f8","integrity_meaning":"Content-addressed record only; not an authenticated issuer signature or independent scientific validation.","lab":"OpenAI","limitations":["Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects."],"mark_status":"not_awarded","model_label":"gpt-4o-mini","observations":[{"aggregation_blocker":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","attribution":"Yifei Ming et al.; Salesforce AI Research; Copyright (c) 2024 Salesforce, Inc.","cohort":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:counterfactual","confidence_interval":null,"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"configuration_id":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:counterfactual:gpt-4o-mini","definition":"Source accuracy against the answer supported by supplied counterfactual context. Higher means context adherence, not greater world-truth factuality.","denominator":null,"direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/LICENSE.txt","metric":"counterfactual_context_accuracy","metric_label":"Counterfactual context adherence","model_label":"gpt-4o-mini","numerator":null,"overall_score":null,"pillar":"evidence_support","profile_id":"hm-f88aadd2968dc59601f8","publication_eligible":true,"rank":16,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scenario":"counterfactual","source":"faitheval","source_sha256":"e28a2aac01f5ae26f8ffe7e1ae116a201f1b8efbbd286a2b4b270078b69df91d","source_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/assets/model_performance_cc.png","split":"FaithEval published task chart","study_year":2024,"track":"honesty_evidence","uncertainty_note":"No inferential interval reconstructed from aggregate source values.","unit":"percent","value":50.9},{"aggregation_blocker":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","attribution":"Yifei Ming et al.; Salesforce AI Research; Copyright (c) 2024 Salesforce, Inc.","cohort":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:counterfactual","confidence_interval":null,"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"configuration_id":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:counterfactual:gpt-4o-mini","definition":"Source baseline answer accuracy with no supplied context; companion to counterfactual-context adherence.","denominator":null,"direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/LICENSE.txt","metric":"no_context_accuracy","metric_label":"No-context answer accuracy","model_label":"gpt-4o-mini","numerator":null,"overall_score":null,"pillar":"usefulness","profile_id":"hm-f88aadd2968dc59601f8","publication_eligible":true,"rank":7,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scenario":"counterfactual","source":"faitheval","source_sha256":"e28a2aac01f5ae26f8ffe7e1ae116a201f1b8efbbd286a2b4b270078b69df91d","source_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/assets/model_performance_cc.png","split":"FaithEval published task chart","study_year":2024,"track":"honesty_evidence","uncertainty_note":"No inferential interval reconstructed from aggregate source values.","unit":"percent","value":92.2},{"aggregation_blocker":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","attribution":"Yifei Ming et al.; Salesforce AI Research; Copyright (c) 2024 Salesforce, Inc.","cohort":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:inconsistent","confidence_interval":null,"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"configuration_id":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:inconsistent:gpt-4o-mini","definition":"Source strict-match accuracy for expressing conflict when supplied documents support different answers. Published rounded aggregate; this is not citation correctness.","denominator":null,"direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/LICENSE.txt","metric":"conflict_match_accuracy","metric_label":"Conflict-answer match","model_label":"gpt-4o-mini","numerator":null,"overall_score":null,"pillar":"evidence_support","profile_id":"hm-f88aadd2968dc59601f8","publication_eligible":true,"rank":4,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scenario":"inconsistent","source":"faitheval","source_sha256":"3c975129e4ee807c2f3ec8a3f33333d027e86030d439051b393de92cb20750f5","source_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/assets/model_performance_ic.png","split":"FaithEval published task chart","study_year":2024,"track":"honesty_evidence","uncertainty_note":"No inferential interval reconstructed from aggregate source values.","unit":"percent","value":57.0},{"aggregation_blocker":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","attribution":"Yifei Ming et al.; Salesforce AI Research; Copyright (c) 2024 Salesforce, Inc.","cohort":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:inconsistent","confidence_interval":null,"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"configuration_id":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:inconsistent:gpt-4o-mini","definition":"Source baseline answer accuracy with original answerable context; companion usefulness outcome on this task collection.","denominator":null,"direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/LICENSE.txt","metric":"original_context_accuracy","metric_label":"Original-context answer accuracy","model_label":"gpt-4o-mini","numerator":null,"overall_score":null,"pillar":"usefulness","profile_id":"hm-f88aadd2968dc59601f8","publication_eligible":true,"rank":9,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scenario":"inconsistent","source":"faitheval","source_sha256":"3c975129e4ee807c2f3ec8a3f33333d027e86030d439051b393de92cb20750f5","source_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/assets/model_performance_ic.png","split":"FaithEval published task chart","study_year":2024,"track":"honesty_evidence","uncertainty_note":"No inferential interval reconstructed from aggregate source values.","unit":"percent","value":83.7},{"aggregation_blocker":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","attribution":"Yifei Ming et al.; Salesforce AI Research; Copyright (c) 2024 Salesforce, Inc.","cohort":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:unanswerable","confidence_interval":null,"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"configuration_id":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:unanswerable:gpt-4o-mini","definition":"Source baseline answer accuracy with original answerable context; companion usefulness outcome on this task collection.","denominator":null,"direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/LICENSE.txt","metric":"original_context_accuracy","metric_label":"Original-context answer accuracy","model_label":"gpt-4o-mini","numerator":null,"overall_score":null,"pillar":"usefulness","profile_id":"hm-f88aadd2968dc59601f8","publication_eligible":true,"rank":14,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scenario":"unanswerable","source":"faitheval","source_sha256":"6f70d51b721ec41b7b2e5603898084ec1560f81f4350cd7b1a2bd2814276e13a","source_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/assets/model_performance_un.png","split":"FaithEval published task chart","study_year":2024,"track":"honesty_evidence","uncertainty_note":"No inferential interval reconstructed from aggregate source values.","unit":"percent","value":71.9},{"aggregation_blocker":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","attribution":"Yifei Ming et al.; Salesforce AI Research; Copyright (c) 2024 Salesforce, Inc.","cohort":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:unanswerable","confidence_interval":null,"configuration":{"benchmark":"FaithEval","configuration_scope":"Three authored task charts; baseline conditions retained separately","decoding":"deterministic; paper v1 section 3.3 reports temperature 0","model_revision":null,"prompting":"source direct-answer task-specific instructions","scorer":"paper v1 strict answer matching for unknown/conflict; answer matching for counterfactual context","scorer_provenance":"Repository chart values: all 18 unanswerable values match paper v1 Table 3 strict AVG; 17 overlapping inconsistent labels match Table 4 strict AVG. README non-strict quick-start defaults are not asserted as the chart scorer.","source_commit":"58d35840e3fbc7bf4b7672582a417b3b3a327dec","source_method_sha256":"42348f6e4652479cad86a9dac848ef6f11c9ced79c7ae739069482b236b39ceb","source_method_url":"https://arxiv.org/html/2410.03727v1","source_model_label":"gpt-4o-mini","study_year":2024},"configuration_id":"faitheval:58d35840e3fbc7bf4b7672582a417b3b3a327dec:unanswerable:gpt-4o-mini","definition":"Source strict-match accuracy for expressing unknown when supplied context lacks the answer. Published rounded aggregate; answerability is relative to supplied context.","denominator":null,"direction":"higher","evaluation_date":null,"evidence_class":"benchmark_authors_reported","lab":"OpenAI","license":"Apache-2.0","license_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/LICENSE.txt","metric":"unknown_match_accuracy","metric_label":"Unknown-answer match","model_label":"gpt-4o-mini","numerator":null,"overall_score":null,"pillar":"uncertainty","profile_id":"hm-f88aadd2968dc59601f8","publication_eligible":true,"rank":4,"rank_meaning":"Descriptive native-metric order; not a significance or overall honesty rank.","scenario":"unanswerable","source":"faitheval","source_sha256":"6f70d51b721ec41b7b2e5603898084ec1560f81f4350cd7b1a2bd2814276e13a","source_url":"https://github.com/SalesforceAIResearch/FaithEval/blob/58d35840e3fbc7bf4b7672582a417b3b3a327dec/assets/model_performance_un.png","split":"FaithEval published task chart","study_year":2024,"track":"honesty_evidence","uncertainty_note":"No inferential interval reconstructed from aggregate source values.","unit":"percent","value":52.3}],"overall_alignment_score":null,"overall_honesty_score":null,"priority_gaps":["belief_revision","citation_integrity","factual_accuracy","pressure_resistance"],"requirements":[{"detail":"The standard is a draft; no pass thresholds or award authority have been validated.","id":"standard_ratified","label":"A published, validated standard and scoring policy","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"configuration_pinned","label":"Model revision, provider, scaffold, tools and test settings are pinned","status":"not_established"},{"detail":"Missing or incomplete common-basket coverage. Some observations cover: evidence_support, uncertainty.","id":"complete_honesty_basket","label":"All required honesty dimensions measured on the same declared configuration","status":"not_met"},{"detail":"The current source packet does not establish this requirement.","id":"held_out_evaluation","label":"Held-out evaluation separated from training and development","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"usefulness_and_evasion","label":"Useful answering, abstention and evasion measured together","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"uncertainty_validated","label":"Uncertainty reflects task dependence and supports the claimed comparison","status":"not_established"},{"detail":"The current source packet does not establish this requirement.","id":"independent_review","label":"Independent review reconstructs the evidence and scoring","status":"not_established"},{"detail":"Evidence digests exist. Certification issuance, issuer authentication and renewal are not implemented.","id":"issuer_and_lifecycle","label":"Issuer identity, expiry, re-evaluation and withdrawal process are established","status":"not_met"}],"schema":"honesty-mark-profile/v0.1","scope":"Native source/configuration results only. This profile is not an awarded Honesty Mark.","source":"faitheval","sources":[{"comparison_caution":"Historical 2024 authors-reported chart aggregates with deterministic decoding and task-specific prompts. Exact endpoint revisions, per-response counts and uncertainty are not established by the charts. Unanswerable/inconsistent scores are attributed to the paper v1 strict keyword matching scheme and do not validate arbitrary abstention explanations. Counterfactual adherence rewards following the supplied fictional premise; it is not an alignment or world-truth score. Baselines have different context/prompt conditions and are companion outcomes, not causal tool effects.","id":"faitheval","import_status":"imported","metrics":["Unknown-answer match","Conflict-answer match","Counterfactual context adherence","Companion baseline accuracy"],"name":"FaithEval · historical 2024","organization":"Salesforce AI Research","pillars":["evidence_support","uncertainty"],"reviewed_on":"2026-10-11","url":"https://github.com/SalesforceAIResearch/FaithEval"}],"standard_sha256":"0cec00d16bbb6e2a02a28e43e43394e3eb309523c2ed697913154c4a2e6e6f6c","standard_version":"honesty-mark-standard/v0.1"}