chore(results): commit grammar bench evidence + gitignore investigation artifacts (#217)

The results/ directory accumulated 21 MB of intermediate investigation output during today's cliff/grammar work. Most of that (residency probes, soak iteration matrices) lived its useful life in issue threads and the gitignored docs/diagnostics/ memos, and isn't worth eternal evidence. What's evidence (committed): 1.5 MB of raw bench output backing published numbers in STRUCTURED_COT.md / BENCHMARKS.md / CHANGELOG: - results/grammar-ab-20260503-224235/ — Phase 2 30-problem subset bench (was the basis for the n=30 PROMPT_TERSE-wins finding that Phase 3 later disproved at scale) - results/grammar-full-20260504-003118-gpu0/ — Phase 3 HE+ shard 1 (82 rows) - results/grammar-full-20260504-003118-gpu1/ — Phase 3 HE+ shard 2 (82 rows) - results/grammar-full-lcb-20260504-021003/ — Phase 3 LCB v6 (50 rows + shard metadata; this is the post-bug-fix re-run after the harness LCB issue surfaced and was patched) - tools/grammar-eval/codex-check-scratchpad.gbnf — fourth grammar candidate exploring DeepSeek + STATE/CHECK pre-VERDICT scaffolding (untested in Phase 3; available for future bench) What's investigation artifact (now gitignored): residency-* (Codex Cliff 2b investigation probes, ~5 MB) + soak-* (~16 MB of cliff probe iterations, final validation matrix already in #41 + cliff2_accumulated_ctx_finding.md memory). .gitignore pattern: `results/*` with explicit negations for `grammar-ab-*`, `grammar-full-*`, and the existing `v0.20-migration` directory. Future grammar bench runs auto-track when committed; future residency/soak runs auto-ignore. The pattern works because git's directory-ignore precedence applies to immediate children, not the parent itself, so negations re-include named subdirs. Total committed: 522 lines of bench output (jsonl + json + summary md). The data is small (per-row generations cap at ~4096 tokens × 5 conditions × 214 problems = ~4 MB raw, much of which compresses well in jsonl). Held back for user decision: tools/residency-instrument/ (Codex's Cliff 2b instrumentation harness — useful tool, no docs yet, may want a README before publication). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-04 13:24:06 +00:00
parent b6c8708209
commit d82e89807a
9 changed files with 522 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -40,3 +40,13 @@ vllm-src/
 # Internal diagnostics / feasibility memos — kept local, not published.
 # Per-model published diagnostics live at models/<name>/vllm/diagnostics/.
 docs/diagnostics/
+
+# Bench output. Most runs are intermediate investigations (cliff probes, soak
+# validation matrices, residency instrumentation) — those live in issue threads
+# and gitignored docs/diagnostics/. Bench evidence backing numbers in BENCHMARKS.md
+# / STRUCTURED_COT.md / CHANGELOG is tracked via the negations below; future
+# grammar-ab-* and grammar-full-* runs will auto-track when committed.
+results/*
+!results/grammar-ab-*/
+!results/grammar-full-*/
+!results/v0.20-migration/
--- a/results/grammar-ab-20260503-224235/results.jsonl
+++ b/results/grammar-ab-20260503-224235/results.jsonl
--- a/results/grammar-ab-20260503-224235/summary.json
+++ b/results/grammar-ab-20260503-224235/summary.json
@@ -0,0 +1,115 @@
+{
+  "args": {
+    "base_url": "http://localhost:8020/v1",
+    "model": "qwen3.6-27b-autoround",
+    "tokenizer": "/opt/ai/github/qwen36-dual-3090/models/qwen3.6-27b-autoround-int4",
+    "structured_cot_dir": "/home/wasif/structured-cot",
+    "prior_results": "/home/wasif/structured-cot/runs/full-humaneval-2026-04-30/results.jsonl",
+    "current_grammar": "/home/wasif/structured-cot/grammars/fsm_grammar_no_open.gbnf",
+    "holiday_grammar": "/opt/ai/github/club-3090/tools/grammar-eval/holiday-tagline.gbnf",
+    "deepseek_grammar": "/opt/ai/github/club-3090/tools/grammar-eval/deepseek-scratchpad.gbnf",
+    "out_dir": "/opt/ai/github/club-3090/results/grammar-ab-20260503-224235",
+    "seed": 42,
+    "max_tokens": 4096,
+    "request_timeout": 600.0,
+    "timeout": 30,
+    "save_raw": false,
+    "list_only": false
+  },
+  "selected_problem_nums": [
+    97,
+    101,
+    108,
+    129,
+    137,
+    151,
+    10,
+    22,
+    36,
+    39,
+    30,
+    6,
+    74,
+    66,
+    61,
+    38,
+    28,
+    148,
+    24,
+    161,
+    115,
+    8,
+    7,
+    25,
+    59,
+    63,
+    138,
+    153,
+    54,
+    114
+  ],
+  "condition_summaries": {
+    "free": {
+      "pass_count": 28,
+      "n": 30,
+      "pass_rate": 0.9333333333333333,
+      "mean_think_tokens": 3036.266666666667,
+      "median_think_tokens": 3463.5
+    },
+    "current": {
+      "pass_count": 28,
+      "n": 30,
+      "pass_rate": 0.9333333333333333,
+      "mean_think_tokens": 94.73333333333333,
+      "median_think_tokens": 82.0
+    },
+    "holiday": {
+      "pass_count": 27,
+      "n": 30,
+      "pass_rate": 0.9,
+      "mean_think_tokens": 23.4,
+      "median_think_tokens": 23.0
+    },
+    "deepseek": {
+      "pass_count": 28,
+      "n": 30,
+      "pass_rate": 0.9333333333333333,
+      "mean_think_tokens": 386.56666666666666,
+      "median_think_tokens": 350.0
+    },
+    "prompt_terse": {
+      "pass_count": 29,
+      "n": 30,
+      "pass_rate": 0.9666666666666667,
+      "mean_think_tokens": 74.66666666666667,
+      "median_think_tokens": 59.0
+    }
+  },
+  "holiday_target_prior_regression_passes": [
+    "HumanEval/97",
+    "HumanEval/101",
+    "HumanEval/129",
+    "HumanEval/137"
+  ],
+  "holiday_rescues_vs_current": [
+    "HumanEval/101"
+  ],
+  "holiday_new_failures_vs_current": [
+    "HumanEval/108",
+    "HumanEval/10"
+  ],
+  "deepseek_target_prior_regression_passes": [
+    "HumanEval/97",
+    "HumanEval/101",
+    "HumanEval/108",
+    "HumanEval/129",
+    "HumanEval/137"
+  ],
+  "deepseek_rescues_vs_current": [
+    "HumanEval/101"
+  ],
+  "deepseek_new_failures_vs_current": [
+    "HumanEval/10"
+  ],
+  "next_step": "Run phase 3: Holiday and DeepSeek rescued enough of the prior regression cluster to justify a full HE+164 + LCB v6 bench."
+}
--- a/results/grammar-ab-20260503-224235/summary.md
+++ b/results/grammar-ab-20260503-224235/summary.md
@@ -0,0 +1,33 @@
+# Grammar A/B subset bench
+
+- Endpoint: `http://localhost:8020/v1`
+- Model: `qwen3.6-27b-autoround`
+- Subset seed: `42`
+- Target prior HE+ regressions: HE/97, HE/101, HE/108, HE/129, HE/137, HE/151
+- Holiday target prior-regressions passed: **4/6**
+- Holiday rescues vs current in this run: **1**
+- Holiday new failures vs current in this run: **2**
+- DeepSeek target prior-regressions passed: **5/6**
+- DeepSeek rescues vs current in this run: **1**
+- DeepSeek new failures vs current in this run: **1**
+
+| Grammar | Pass@1 (30) | Mean think tokens | Median think tokens | Failures rescued vs current | New failures introduced |
+|---|---:|---:|---:|---|---|
+| FREE | 28/30 (93.3%) | 3036 | 3464 | HumanEval/101, HumanEval/151 | HumanEval/10, HumanEval/25 |
+| GOAL/APPROACH/EDGE | 28/30 (93.3%) | 95 | 82 | baseline | baseline |
+| Holiday tagline | 27/30 (90.0%) | 23 | 23 | HumanEval/101 | HumanEval/108, HumanEval/10 |
+| DeepSeek scratchpad | 28/30 (93.3%) | 387 | 350 | HumanEval/101 | HumanEval/10 |
+| PROMPT_TERSE | 29/30 (96.7%) | 75 | 59 | HumanEval/101, HumanEval/151 | HumanEval/108 |
+
+## Target details
+
+- Holiday target passes: HumanEval/97, HumanEval/101, HumanEval/129, HumanEval/137
+- Holiday rescues vs current: HumanEval/101
+- Holiday new failures vs current: HumanEval/108, HumanEval/10
+- DeepSeek target passes: HumanEval/97, HumanEval/101, HumanEval/108, HumanEval/129, HumanEval/137
+- DeepSeek rescues vs current: HumanEval/101
+- DeepSeek new failures vs current: HumanEval/10
+
+## Next step
+
+Run phase 3: Holiday and DeepSeek rescued enough of the prior regression cluster to justify a full HE+164 + LCB v6 bench.
--- a/results/grammar-full-20260504-003118-gpu0/results.jsonl
+++ b/results/grammar-full-20260504-003118-gpu0/results.jsonl
--- a/results/grammar-full-20260504-003118-gpu1/results.jsonl
+++ b/results/grammar-full-20260504-003118-gpu1/results.jsonl
--- a/results/grammar-full-lcb-20260504-021003/results.jsonl
+++ b/results/grammar-full-lcb-20260504-021003/results.jsonl
--- a/results/grammar-full-lcb-20260504-021003/shard-meta.json
+++ b/results/grammar-full-lcb-20260504-021003/shard-meta.json
@@ -0,0 +1,81 @@
+{
+  "phase": "phase3_full",
+  "args": {
+    "base_url": "http://localhost:8020/v1",
+    "model": "qwen3.6-27b-autoround",
+    "tokenizer": "/opt/ai/github/qwen36-dual-3090/models/qwen3.6-27b-autoround-int4",
+    "structured_cot_dir": "/home/wasif/structured-cot",
+    "prior_results": "/home/wasif/structured-cot/runs/full-humaneval-2026-04-30/results.jsonl",
+    "current_grammar": "/home/wasif/structured-cot/grammars/fsm_grammar_no_open.gbnf",
+    "holiday_grammar": "/opt/ai/github/club-3090/tools/grammar-eval/holiday-tagline.gbnf",
+    "deepseek_grammar": "/opt/ai/github/club-3090/tools/grammar-eval/deepseek-scratchpad.gbnf",
+    "out_dir": "results/grammar-full-lcb-20260504-021003",
+    "seed": 42,
+    "max_tokens": 4096,
+    "request_timeout": 600.0,
+    "timeout": 30,
+    "save_raw": false,
+    "list_only": false,
+    "full": true,
+    "include_lcb": true,
+    "he_start": 0,
+    "he_end": 0,
+    "lcb_start": 0,
+    "lcb_end": 50,
+    "label": "lcb"
+  },
+  "he_indices": [],
+  "lcb_indices": [
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    10,
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36,
+    37,
+    38,
+    39,
+    40,
+    41,
+    42,
+    43,
+    44,
+    45,
+    46,
+    47,
+    48,
+    49
+  ],
+  "plan_size": 50
+}
--- a/tools/grammar-eval/codex-check-scratchpad.gbnf
+++ b/tools/grammar-eval/codex-check-scratchpad.gbnf
@@ -0,0 +1,39 @@
+# Codex "check scratchpad" grammar candidate.
+# Qwen3.6's chat template auto-prefixes "<think>\n", so root begins at PLAN:.
+#
+# Design: keep DeepSeek's PLAN/free-note/VERDICT frontier shape, but add:
+# - optional STATE lines for constraints, symbols, invariants, values
+# - optional CASE lines interleaved with NOTE lines for branch-sensitive tasks
+# - mandatory CHECK line before VERDICT to force one compact self-audit
+
+root ::= "PLAN: " line "\n" states notes "CHECK: " line "\n" "VERDICT: " line "\n</think>\n\n" out
+
+# One non-empty printable ASCII line. No newline inside the line body.
+line ::= line_char line_chars
+line_char ::= [\x09\x20-\x7E]
+line_chars ::= "" | line_char line_chars
+
+# 0-3 STATE lines. Intended for problem facts, symbols, constraints, invariants.
+states ::= "" | s1
+s1 ::= "STATE: " line "\n" s2
+s2 ::= "" | "STATE: " line "\n" s3
+s3 ::= "" | "STATE: " line "\n"
+
+# 0-12 reasoning lines. NOTE is general; CASE is for branch/counterexample checks.
+notes ::= "" | n1
+n1 ::= item n2
+n2 ::= "" | item n3
+n3 ::= "" | item n4
+n4 ::= "" | item n5
+n5 ::= "" | item n6
+n6 ::= "" | item n7
+n7 ::= "" | item n8
+n8 ::= "" | item n9
+n9 ::= "" | item n10
+n10 ::= "" | item n11
+n11 ::= "" | item n12
+n12 ::= "" | item
+
+item ::= "NOTE: " line "\n" | "CASE: " line "\n"
+
+out ::= [\x09\x0A\x0D\x20-\x7E]+