diff --git a/docs/DUAL_CARD.md b/docs/DUAL_CARD.md index 258128d8..f088307d 100644 --- a/docs/DUAL_CARD.md +++ b/docs/DUAL_CARD.md @@ -169,7 +169,16 @@ For variance, AL / accept rates, per-config row docstrings: see each compose YAM | `dual-dflash.yml` | 185K | 82 / 125 | ~140 ms | 1 | ✅ | code + vision | | `dual-dflash-noviz.yml` | 200K | 78 / 127 | ~145 ms | 1 | ❌ | pure text code | -`dual-turbo` numbers re-benched 2026-05-01 on the v0.20 + Genesis v7.65 substrate (full PROD env-var stack, +9% TPS over the prior dev205 + v7.64 measurements). Other dual variants pending re-bench on v0.20. All bench runs: 3 warmup + 5 measured per prompt. Run-by-run + CV in `models/qwen3.6-27b/CHANGELOG.md` "Dual-card re-bench" entry. +All four dual variants re-benched 2026-05-01 PM on the v0.20 + Genesis v7.65 dev tip substrate (n=5 measured + 3 warmup per prompt): + +| Variant | Narr / Code wall_TPS (CV) | vs prior chart | +|---|---|---| +| `dual.yml` | 68.61 / 90.71 (CV 1.8% both) | flat (within noise) | +| `dual-turbo.yml` | 58.33 / 76.01 (n=1) · 269 TPS aggregate at n=4 streams | matches prior | +| `dual-dflash.yml` | 77.12 / 125.97 (CV 2-4%) | code flat, narr -5.9% (slight) | +| `dual-dflash-noviz.yml` | 78.94 / 123.18 (CV 2-3%) | flat (within noise) | + +Code TPS held within bench variance across all 4 variants — no v0.20 regression on fp8 / FP16 paths. Run-by-run + per-config summaries in [`results/v0.20-migration/`](https://github.com/noonghunna/club-3090/tree/master/results/v0.20-migration). --- diff --git a/results/v0.20-migration/dual-bench.txt b/results/v0.20-migration/dual-bench.txt new file mode 100644 index 00000000..f34b5ffc --- /dev/null +++ b/results/v0.20-migration/dual-bench.txt @@ -0,0 +1,45 @@ + +========== NARRATIVE (prompt=65 chars, max_tokens=1000) ========== +=== warmups (3) === + warm-1 wall= 42.93s ttft= 27642ms toks=1000 wall_TPS= 23.30 decode_TPS= 65.43 + warm-2 wall= 14.27s ttft= 118ms toks=1000 wall_TPS= 70.08 decode_TPS= 70.66 + warm-3 wall= 14.80s ttft= 143ms toks= 999 wall_TPS= 67.50 decode_TPS= 68.16 + +=== measured (5) === + run-1 wall= 14.79s ttft= 146ms toks=1000 wall_TPS= 67.62 decode_TPS= 68.29 + run-2 wall= 14.54s ttft= 150ms toks=1000 wall_TPS= 68.77 decode_TPS= 69.49 + run-3 wall= 14.33s ttft= 143ms toks=1000 wall_TPS= 69.77 decode_TPS= 70.47 + run-4 wall= 14.32s ttft= 121ms toks=1000 wall_TPS= 69.83 decode_TPS= 70.42 + run-5 wall= 14.91s ttft= 118ms toks=1000 wall_TPS= 67.05 decode_TPS= 67.58 + +=== summary [narrative] (n=5) === + wall_TPS mean= 68.61 std= 1.25 CV= 1.8% min=67.05 max=69.83 + decode_TPS mean= 69.25 std= 1.29 CV= 1.9% min=67.58 max=70.47 + TTFT mean= 136ms std= 15ms min=118ms max=150ms + +========== CODE (prompt=78 chars, max_tokens=800) ========== +=== warmups (3) === + warm-1 wall= 9.63s ttft= 154ms toks= 800 wall_TPS= 83.07 decode_TPS= 84.42 + warm-2 wall= 9.13s ttft= 117ms toks= 800 wall_TPS= 87.66 decode_TPS= 88.79 + warm-3 wall= 6.99s ttft= 153ms toks= 626 wall_TPS= 89.52 decode_TPS= 91.52 + +=== measured (5) === + run-1 wall= 5.21s ttft= 144ms toks= 483 wall_TPS= 92.79 decode_TPS= 95.43 + run-2 wall= 8.85s ttft= 148ms toks= 800 wall_TPS= 90.41 decode_TPS= 91.95 + run-3 wall= 8.08s ttft= 144ms toks= 743 wall_TPS= 91.94 decode_TPS= 93.60 + run-4 wall= 8.98s ttft= 147ms toks= 800 wall_TPS= 89.04 decode_TPS= 90.52 + run-5 wall= 8.95s ttft= 146ms toks= 800 wall_TPS= 89.37 decode_TPS= 90.85 + +=== summary [code] (n=5) === + wall_TPS mean= 90.71 std= 1.62 CV= 1.8% min=89.04 max=92.79 + decode_TPS mean= 92.47 std= 2.05 CV= 2.2% min=90.52 max=95.43 + TTFT mean= 146ms std= 2ms min=144ms max=148ms + +=== GPU state === +0, 66 %, 22246 MiB, 24576 MiB, 228.71 W, 66 +1, 86 %, 22246 MiB, 24576 MiB, 229.33 W, 63 + +=== Last 3 SpecDecoding metrics === +(APIServer pid=1) INFO 05-01 19:15:26 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 3.44, Accepted throughput: 64.40 tokens/s, Drafted throughput: 79.19 tokens/s, Accepted: 644 tokens, Drafted: 792 tokens, Per-position acceptance rate: 0.932, 0.822, 0.686, Avg Draft acceptance rate: 81.3% +(APIServer pid=1) INFO 05-01 19:15:36 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 3.51, Accepted throughput: 66.59 tokens/s, Drafted throughput: 79.49 tokens/s, Accepted: 666 tokens, Drafted: 795 tokens, Per-position acceptance rate: 0.955, 0.857, 0.702, Avg Draft acceptance rate: 83.8% +(APIServer pid=1) INFO 05-01 19:15:46 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 3.49, Accepted throughput: 63.19 tokens/s, Drafted throughput: 76.19 tokens/s, Accepted: 632 tokens, Drafted: 762 tokens, Per-position acceptance rate: 0.961, 0.843, 0.685, Avg Draft acceptance rate: 82.9% diff --git a/results/v0.20-migration/dual-dflash-bench.txt b/results/v0.20-migration/dual-dflash-bench.txt new file mode 100644 index 00000000..2976e83c --- /dev/null +++ b/results/v0.20-migration/dual-dflash-bench.txt @@ -0,0 +1,45 @@ + +========== NARRATIVE (prompt=65 chars, max_tokens=1000) ========== +=== warmups (3) === + warm-1 wall= 16.24s ttft= 1740ms toks=1000 wall_TPS= 61.58 decode_TPS= 68.97 + warm-2 wall= 12.53s ttft= 145ms toks=1000 wall_TPS= 79.83 decode_TPS= 80.77 + warm-3 wall= 12.06s ttft= 143ms toks=1000 wall_TPS= 82.93 decode_TPS= 83.93 + +=== measured (5) === + run-1 wall= 13.26s ttft= 142ms toks=1000 wall_TPS= 75.43 decode_TPS= 76.24 + run-2 wall= 13.17s ttft= 118ms toks=1000 wall_TPS= 75.93 decode_TPS= 76.61 + run-3 wall= 13.04s ttft= 118ms toks=1000 wall_TPS= 76.68 decode_TPS= 77.38 + run-4 wall= 12.54s ttft= 140ms toks=1000 wall_TPS= 79.74 decode_TPS= 80.64 + run-5 wall= 12.85s ttft= 117ms toks=1000 wall_TPS= 77.83 decode_TPS= 78.55 + +=== summary [narrative] (n=5) === + wall_TPS mean= 77.12 std= 1.72 CV= 2.2% min=75.43 max=79.74 + decode_TPS mean= 77.88 std= 1.78 CV= 2.3% min=76.24 max=80.64 + TTFT mean= 127ms std= 13ms min=117ms max=142ms + +========== CODE (prompt=78 chars, max_tokens=800) ========== +=== warmups (3) === + warm-1 wall= 5.29s ttft= 153ms toks= 715 wall_TPS=135.14 decode_TPS=139.15 + warm-2 wall= 6.51s ttft= 140ms toks= 800 wall_TPS=122.89 decode_TPS=125.60 + warm-3 wall= 5.67s ttft= 140ms toks= 717 wall_TPS=126.53 decode_TPS=129.74 + +=== measured (5) === + run-1 wall= 6.55s ttft= 142ms toks= 800 wall_TPS=122.11 decode_TPS=124.82 + run-2 wall= 5.74s ttft= 148ms toks= 706 wall_TPS=122.99 decode_TPS=126.24 + run-3 wall= 3.27s ttft= 141ms toks= 433 wall_TPS=132.60 decode_TPS=138.57 + run-4 wall= 6.39s ttft= 146ms toks= 792 wall_TPS=123.89 decode_TPS=126.78 + run-5 wall= 6.24s ttft= 140ms toks= 800 wall_TPS=128.27 decode_TPS=131.22 + +=== summary [code] (n=5) === + wall_TPS mean= 125.97 std= 4.40 CV= 3.5% min=122.11 max=132.60 + decode_TPS mean= 129.53 std= 5.59 CV= 4.3% min=124.82 max=138.57 + TTFT mean= 143ms std= 3ms min=140ms max=148ms + +=== GPU state === +0, 84 %, 21720 MiB, 24576 MiB, 229.07 W, 66 +1, 85 %, 21720 MiB, 24576 MiB, 226.38 W, 62 + +=== Last 3 SpecDecoding metrics === +(APIServer pid=1) INFO 05-01 19:24:24 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 4.28, Accepted throughput: 95.80 tokens/s, Drafted throughput: 146.00 tokens/s, Accepted: 958 tokens, Drafted: 1460 tokens, Per-position acceptance rate: 0.880, 0.788, 0.644, 0.534, 0.435, Avg Draft acceptance rate: 65.6% +(APIServer pid=1) INFO 05-01 19:24:34 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 4.23, Accepted throughput: 94.39 tokens/s, Drafted throughput: 145.99 tokens/s, Accepted: 944 tokens, Drafted: 1460 tokens, Per-position acceptance rate: 0.870, 0.760, 0.620, 0.534, 0.449, Avg Draft acceptance rate: 64.7% +(APIServer pid=1) INFO 05-01 19:24:44 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 4.33, Accepted throughput: 97.10 tokens/s, Drafted throughput: 146.00 tokens/s, Accepted: 971 tokens, Drafted: 1460 tokens, Per-position acceptance rate: 0.908, 0.771, 0.654, 0.531, 0.462, Avg Draft acceptance rate: 66.5% diff --git a/results/v0.20-migration/dual-dflash-noviz-bench.txt b/results/v0.20-migration/dual-dflash-noviz-bench.txt new file mode 100644 index 00000000..bbed5562 --- /dev/null +++ b/results/v0.20-migration/dual-dflash-noviz-bench.txt @@ -0,0 +1,45 @@ + +========== NARRATIVE (prompt=65 chars, max_tokens=1000) ========== +=== warmups (3) === + warm-1 wall= 15.52s ttft= 1408ms toks= 962 wall_TPS= 61.97 decode_TPS= 68.16 + warm-2 wall= 12.00s ttft= 144ms toks=1000 wall_TPS= 83.36 decode_TPS= 84.37 + warm-3 wall= 13.11s ttft= 142ms toks=1000 wall_TPS= 76.28 decode_TPS= 77.11 + +=== measured (5) === + run-1 wall= 12.24s ttft= 147ms toks=1000 wall_TPS= 81.68 decode_TPS= 82.67 + run-2 wall= 12.97s ttft= 145ms toks=1000 wall_TPS= 77.12 decode_TPS= 77.99 + run-3 wall= 12.84s ttft= 149ms toks=1000 wall_TPS= 77.86 decode_TPS= 78.77 + run-4 wall= 12.23s ttft= 121ms toks=1000 wall_TPS= 81.77 decode_TPS= 82.59 + run-5 wall= 12.45s ttft= 120ms toks= 949 wall_TPS= 76.25 decode_TPS= 76.99 + +=== summary [narrative] (n=5) === + wall_TPS mean= 78.94 std= 2.61 CV= 3.3% min=76.25 max=81.77 + decode_TPS mean= 79.80 std= 2.66 CV= 3.3% min=76.99 max=82.67 + TTFT mean= 136ms std= 15ms min=120ms max=149ms + +========== CODE (prompt=78 chars, max_tokens=800) ========== +=== warmups (3) === + warm-1 wall= 5.96s ttft= 144ms toks= 800 wall_TPS=134.20 decode_TPS=137.52 + warm-2 wall= 5.92s ttft= 143ms toks= 753 wall_TPS=127.15 decode_TPS=130.29 + warm-3 wall= 5.55s ttft= 146ms toks= 725 wall_TPS=130.74 decode_TPS=134.28 + +=== measured (5) === + run-1 wall= 6.29s ttft= 145ms toks= 800 wall_TPS=127.21 decode_TPS=130.20 + run-2 wall= 6.58s ttft= 146ms toks= 800 wall_TPS=121.57 decode_TPS=124.32 + run-3 wall= 6.52s ttft= 144ms toks= 800 wall_TPS=122.61 decode_TPS=125.38 + run-4 wall= 3.36s ttft= 145ms toks= 399 wall_TPS=118.58 decode_TPS=123.92 + run-5 wall= 6.28s ttft= 144ms toks= 791 wall_TPS=125.93 decode_TPS=128.89 + +=== summary [code] (n=5) === + wall_TPS mean= 123.18 std= 3.46 CV= 2.8% min=118.58 max=127.21 + decode_TPS mean= 126.54 std= 2.83 CV= 2.2% min=123.92 max=130.20 + TTFT mean= 145ms std= 1ms min=144ms max=146ms + +=== GPU state === +0, 82 %, 22060 MiB, 24576 MiB, 228.59 W, 66 +1, 84 %, 22060 MiB, 24576 MiB, 226.59 W, 62 + +=== Last 3 SpecDecoding metrics === +(APIServer pid=1) INFO 05-01 19:32:33 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 4.38, Accepted throughput: 99.09 tokens/s, Drafted throughput: 146.49 tokens/s, Accepted: 991 tokens, Drafted: 1465 tokens, Per-position acceptance rate: 0.918, 0.799, 0.659, 0.543, 0.464, Avg Draft acceptance rate: 67.6% +(APIServer pid=1) INFO 05-01 19:32:43 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 4.27, Accepted throughput: 95.29 tokens/s, Drafted throughput: 145.48 tokens/s, Accepted: 953 tokens, Drafted: 1455 tokens, Per-position acceptance rate: 0.876, 0.787, 0.639, 0.543, 0.430, Avg Draft acceptance rate: 65.5% +(APIServer pid=1) INFO 05-01 19:32:53 [metrics.py:101] SpecDecoding metrics: Mean acceptance length: 4.14, Accepted throughput: 91.80 tokens/s, Drafted throughput: 146.00 tokens/s, Accepted: 918 tokens, Drafted: 1460 tokens, Per-position acceptance rate: 0.890, 0.747, 0.613, 0.497, 0.397, Avg Draft acceptance rate: 62.9% diff --git a/results/v0.20-migration/dual-dflash-noviz.summary b/results/v0.20-migration/dual-dflash-noviz.summary new file mode 100644 index 00000000..e75f4fd7 --- /dev/null +++ b/results/v0.20-migration/dual-dflash-noviz.summary @@ -0,0 +1,26 @@ +variant: dual-dflash-noviz.yml +date: 2026-05-01 PM +context: 200000 +gpu_mem_util: tbd-from-compose +kv_dtype: auto (FP16 forced by DFlash) +tp: 2 +spec_decode: dflash_n5 +language_model_only: true +narr_wall_tps: 78.94 +narr_wall_cv: 3.3 +narr_decode_tps: 79.80 +code_wall_tps: 123.18 +code_wall_cv: 2.8 +code_decode_tps: 126.54 +ttft_narr_ms: 136 +ttft_code_ms: 145 +dflash_al: 4.14-4.38 +dflash_avg_accept: 0.629-0.676 +chart_value_narr: 78.19 +chart_value_code: 126.99 +delta_vs_chart_narr: +1.0% +delta_vs_chart_code: -3.0% +notes: | + No env-var changes vs dev205. Image bump only. Both narr + code within + bench variance of chart values. Confirms no v0.20 regression on the + text-only DFlash path. diff --git a/results/v0.20-migration/dual-dflash.summary b/results/v0.20-migration/dual-dflash.summary new file mode 100644 index 00000000..463ac8ef --- /dev/null +++ b/results/v0.20-migration/dual-dflash.summary @@ -0,0 +1,31 @@ +variant: dual-dflash.yml +date: 2026-05-01 PM +context: 185000 +gpu_mem_util: tbd-from-compose +kv_dtype: auto (FP16 forced by DFlash non-causal head_size=256) +tp: 2 +spec_decode: dflash_n5 +narr_wall_tps: 77.12 +narr_wall_cv: 2.2 +narr_decode_tps: 77.88 +code_wall_tps: 125.97 +code_wall_cv: 3.5 +code_decode_tps: 129.53 +ttft_narr_ms: 127 +ttft_code_ms: 143 +dflash_al: 4.23-4.33 +dflash_avg_accept: 0.65-0.67 +vram_per_card_mib: 21720 +kv_pool_tokens: 66096 +max_concurrency: 1.02 +chart_value_narr: 81.94 +chart_value_code: 124.93 +delta_vs_chart_narr: -5.9% +delta_vs_chart_code: +0.8% +notes: | + No env-var changes vs dev205 (dual-dflash is Genesis-less). Image bump + only. Code TPS holds. Narrative -5.9% — slightly outside CV band; could + be substrate (different driver / power state) or real microregression + in DFlash's spec-decode draft path on v0.20. Chart value (82) still + within ±5pp of measured (77); not large enough to update charts but + worth flagging if reproducible across reruns. diff --git a/results/v0.20-migration/dual-yml.summary b/results/v0.20-migration/dual-yml.summary new file mode 100644 index 00000000..051dbe44 --- /dev/null +++ b/results/v0.20-migration/dual-yml.summary @@ -0,0 +1,27 @@ +variant: dual.yml +date: 2026-05-01 PM +context: 262144 +gpu_mem_util: 0.92 +kv_dtype: fp8_e5m2 +tp: 2 +spec_decode: mtp_n3 +narr_wall_tps: 68.61 +narr_wall_cv: 1.8 +narr_decode_tps: 69.25 +code_wall_tps: 90.71 +code_wall_cv: 1.8 +code_decode_tps: 92.47 +ttft_ms: 146 +mtp_al: 3.44-3.51 +mtp_avg_accept: 0.813-0.838 +vram_per_card_mib: 22246 +kv_pool_tokens: 169600 +max_concurrency: 2.38 +chart_value_narr: 69.05 +chart_value_code: 88.58 +delta_vs_chart_narr: -0.6% +delta_vs_chart_code: +2.4% +notes: | + No env-var changes vs dev205 (dual.yml is "Genesis-less by design"). + v0.20 image bump only. TPS within bench variance of chart values. + Confirms no v0.20 regression on the fp8 + MTP path.