{ "$schema": "adaptorch.benchmark.evidence/v1", "brand": "AdaptOrch", "title": "AdaptOrch verifier-gate evidence", "generated_at": "2026-07-24T22:17:03.598424+00:00", "generated_by": "scripts/build_adaptorch_benchmark_json.py", "reproduce": { "validate_without_api": "python scripts/run_hard_v3_arm_c.py --validate-only", "paired_run": "python scripts/run_hard_v3_arm_c.py --baseline-config artifacts/benchmarks/hard_v3_arm_c/ledger_only/ledger_only_paper_n30.json --arm-c-config artifacts/benchmarks/hard_v3_arm_c/ledger_only/ledger_only_arm_c_n30.json --repeats 1", "deterministic_seeds": true }, "claim_boundary": { "scope": "local synthetic regression evidence", "is_official_benchmark": false, "is_publication": false, "is_release_go_input": false, "full50_go": "NO_GO", "statement": "Internal reproducible regression evidence on a synthetic task suite. Not an official benchmark, not a publication, and not a release gate input." }, "experiments": { "verifier_gate_ledger": { "id": "verifier_gate_ledger", "question": "Does gating selection on an independent verifier decide better than not gating?", "family": "ledger_reconciliation_v3", "family_label": "Ledger reconciliation", "n_units": 30, "repeats": 1, "model": "gemma-4-31b", "provider": "Cerebras", "arms": { "A": { "name": "paper", "accuracy_pct": 73.3, "run_id": "20260724_003533_e9ead96990ef" }, "B": { "name": "robust-no-verifier", "accuracy_pct": 83.3, "run_id": "20260724_004200_a09cc27fe5ac" }, "C": { "name": "robust-verifier", "accuracy_pct": 96.7, "run_id": "20260724_040855_d757f7b9e0ab" }, "D": { "name": "robust-judge", "accuracy_pct": 90.0, "run_id": "20260724_012313_9be8d686609b" } }, "headline": { "caught_by_verifier": 8, "caught_unit_ids": [ "h3L3", "h3L4", "h3L5", "h3L11", "h3L17", "h3L21", "h3L25", "h3L26" ], "regressed_by_verifier": 1, "regressed_unit_ids": [ "h3L1" ], "net_units_gained": 7 }, "units": [ { "id": "h3L1", "unverified_pass": true, "verified_pass": false }, { "id": "h3L2", "unverified_pass": true, "verified_pass": true }, { "id": "h3L3", "unverified_pass": false, "verified_pass": true }, { "id": "h3L4", "unverified_pass": false, "verified_pass": true }, { "id": "h3L5", "unverified_pass": false, "verified_pass": true }, { "id": "h3L6", "unverified_pass": true, "verified_pass": true }, { "id": "h3L7", "unverified_pass": true, "verified_pass": true }, { "id": "h3L8", "unverified_pass": true, "verified_pass": true }, { "id": "h3L9", "unverified_pass": true, "verified_pass": true }, { "id": "h3L10", "unverified_pass": true, "verified_pass": true }, { "id": "h3L11", "unverified_pass": false, "verified_pass": true }, { "id": "h3L12", "unverified_pass": true, "verified_pass": true }, { "id": "h3L13", "unverified_pass": true, "verified_pass": true }, { "id": "h3L14", "unverified_pass": true, "verified_pass": true }, { "id": "h3L15", "unverified_pass": true, "verified_pass": true }, { "id": "h3L16", "unverified_pass": true, "verified_pass": true }, { "id": "h3L17", "unverified_pass": false, "verified_pass": true }, { "id": "h3L18", "unverified_pass": true, "verified_pass": true }, { "id": "h3L19", "unverified_pass": true, "verified_pass": true }, { "id": "h3L20", "unverified_pass": true, "verified_pass": true }, { "id": "h3L21", "unverified_pass": false, "verified_pass": true }, { "id": "h3L22", "unverified_pass": true, "verified_pass": true }, { "id": "h3L23", "unverified_pass": true, "verified_pass": true }, { "id": "h3L24", "unverified_pass": true, "verified_pass": true }, { "id": "h3L25", "unverified_pass": false, "verified_pass": true }, { "id": "h3L26", "unverified_pass": false, "verified_pass": true }, { "id": "h3L27", "unverified_pass": true, "verified_pass": true }, { "id": "h3L28", "unverified_pass": true, "verified_pass": true }, { "id": "h3L29", "unverified_pass": true, "verified_pass": true }, { "id": "h3L30", "unverified_pass": true, "verified_pass": true } ], "claim_scope": "Single family (ledger), n=30, one run per arm. Absolute per-arm values drift between runs; the arm ordering is the durable signal." }, "paired_confirmatory_ledger": { "id": "paired_confirmatory_ledger", "question": "On identical tasks inside one run, how much does the verifier gate add?", "family": "ledger_reconciliation_v3", "family_label": "Ledger reconciliation", "n_units": 30, "repeats": 1, "model": "gemma-4-31b", "provider": "Cerebras", "session_id": "hard-v3-arm-c-20260724T211511085282Z", "baseline_pct": 86.7, "verified_pct": 100.0, "paired_delta_pp": 13.3, "ci95_pp": [ 3.3, 26.7 ], "significant": true, "design": "paired \u2014 both arms run the same units, same seeds, same session", "claim_scope": "This paired delta is the durable claim: it cancels run-to-run drift because both arms see identical units inside the same session." }, "family_saturation": { "id": "family_saturation", "question": "Where can the verifier even be measured?", "families": [ { "family": "dependency_reasoning_dag_v3", "label": "Dependency DAG", "baseline_pct": 100.0, "delta_pp": 0.0, "has_headroom": false }, { "family": "ledger_reconciliation_v3", "label": "Ledger reconciliation", "baseline_pct": 60.0, "delta_pp": 40.0, "has_headroom": true }, { "family": "python_semantics_mcq_v3", "label": "Python semantics", "baseline_pct": 100.0, "delta_pp": 0.0, "has_headroom": false } ], "gate_passed": false, "gate_reason": "only 1 families have headroom (min required for gate: 3); gate inconclusive", "honest_note": "Two of three families are already solved by the baseline, so no gating method can show a difference there. We only claim on the family that still had room to measure." }, "direct_arm_calibration": { "id": "direct_arm_calibration", "question": "How hard are these tasks for the model with no help at all?", "model": "gemma-4-31b", "target_band_pct": [ 30, 70 ], "families": [ { "family": "ledger_reconciliation_v3", "label": "Ledger reconciliation", "pass_rate_pct": 83.3, "n": 24 }, { "family": "python_semantics_mcq_v3", "label": "Python semantics", "pass_rate_pct": 87.5, "n": 24 }, { "family": "dependency_reasoning_dag_v3", "label": "Dependency DAG", "pass_rate_pct": 45.8, "n": 24 } ] } } }