diff --git a/docs/HARDWARE.md b/docs/HARDWARE.md index 793019d0..c8769a05 100644 --- a/docs/HARDWARE.md +++ b/docs/HARDWARE.md @@ -22,12 +22,18 @@ The recipes are written against 3090 specifically but should work on: |---|---|---|---| | RTX 3090 | 24 GB | sm_86 | **Tested. Default target.** | | RTX 3090 Ti | 24 GB | sm_86 | Should work; same VRAM, slightly higher TPS expected | +| **2× RTX 3080 modded 20 GB** | 20 GB / card (40 GB combined) | sm_86 | **Tested 2026-05-02 by [@troymroberts](https://github.com/troymroberts) ([#25](https://github.com/noonghunna/club-3090/discussions/25#discussioncomment-16787782))** at 200W/card power limit. `dual.yml` (TQ k8v4 KV + MTP K=3) boots at full 262K target with `gpu-memory-utilization=0.82` (down from shipped 0.95 — see note below). Available KV pool 5.2 GB/card, max concurrency 1.43×. verify-full 10/10 pass; bench 49 TPS wall single-stream, 210 TPS aggregate at n=8. First published SM86 / 40 GB combined data point outside the 3090 family. | | RTX 4090 | 24 GB | sm_89 | Should work; ~30% faster decode (newer SMs); same memory characteristics | | RTX 5090 | 32 GB | sm_120 | Untested; more VRAM relaxes the prefill cliffs but kernel paths might differ | +| RTX A5000 | 24 GB | sm_86 | **Sander's PROD class** for [genesis-vllm-patches](https://github.com/Sandermage/genesis-vllm-patches). Identical SM and VRAM to 3090; should run identically. | | RTX A6000 | 48 GB | sm_86 | Should work; double VRAM lets you skip the cliff workarounds (use Sandermage's reference defaults) | | H100 SXM | 80 GB | sm_90 | Different beast; flash-attn 3 paths available; not what these recipes target | -**Won't work:** anything with <24 GB VRAM (3060, 3070, 3080 12 GB). The 27B model in INT4 is ~18 GB — KV pool + activations push past 24 GB on smaller cards even with aggressive quantization. +**Won't work:** anything with <20 GB VRAM (3060, 3070, stock 3080, 3080 Ti). The 27B model in INT4 is ~18 GB — KV pool + activations push past 24 GB on smaller cards even with aggressive quantization. **Modded 20 GB 3080s do work** (see row above) — the mod gives them enough headroom for the 27B + TQ K8V4 KV path on TP=2, with `mem-util=0.82` to absorb cudagraph profiling overhead. + +### Note for sub-24 GB cards + +On 20 GB cards (modded 3080) the cudagraph-profiling overhead is a meaningful slice of available VRAM. Drop `--gpu-memory-utilization` to **0.82** (vs shipped 0.95 for 24 GB). vLLM nightly's `gpu_worker.py` reports the equivalent effective KV size in the boot log; tune to keep activation headroom for the ~15K tool-prefill peak (verify-full check 8). Credit: [@troymroberts](https://github.com/troymroberts). --- diff --git a/scripts/verify-full.sh b/scripts/verify-full.sh index b6edd20b..602f307f 100755 --- a/scripts/verify-full.sh +++ b/scripts/verify-full.sh @@ -93,15 +93,25 @@ check_patches() { skip "container '${CONTAINER}' not found" return 0 fi + # Anchors updated 2026-05-02 for Genesis v7.14+ logging conventions (the old + # "[OK] Qwen3 tool_call fix" string is no longer emitted; markers are now + # "[Genesis] applied:" per patch + "apply_all elapsed" at the end + "FAILED:" + # for any patch that errored). The `|| true` guards against grep returning 1 + # under `set -euo pipefail` when no anchor matches yet (e.g. early boot). + # Reported by @troymroberts in club-3090#25. local logs - logs="$(docker logs "${CONTAINER}" 2>&1 | grep -E "Qwen3 tool_call fix|\[FAILED\]" | tail -5)" - if echo "$logs" | grep -q "\[OK\] Qwen3 tool_call fix"; then - pass "Genesis Qwen3 tool_call fix applied" - elif echo "$logs" | grep -q "\[FAILED\] Qwen3 tool_call fix"; then - fail "Genesis Qwen3 tool_call fix [FAILED]" \ - "vLLM image drifted past patch anchor. Pin sha256:9bba4628a3b9 in compose." + logs="$(docker logs "${CONTAINER}" 2>&1 \ + | { grep -E "apply_all elapsed|\[Genesis\] FAILED|\[Genesis Unified Patch|\[Genesis\] applied:" || true; } \ + | tail -10)" + if echo "$logs" | grep -q "\[Genesis\] FAILED"; then + fail "Genesis apply_all reported FAILED patch(es)" \ + "Inspect: docker logs ${CONTAINER} 2>&1 | grep -E 'Genesis.*FAILED' | head" + elif echo "$logs" | grep -q "apply_all elapsed"; then + pass "Genesis patches applied (apply_all completed clean)" + elif echo "$logs" | grep -q "\[Genesis\] applied:"; then + pass "Genesis patches applied (partial log — apply_all may still be running)" else - skip "no Genesis marker in logs" + skip "no Genesis marker in logs (container restarted, or Genesis not loaded)" fi } run_check "patches" check_patches diff --git a/scripts/verify.sh b/scripts/verify.sh index 9557fa0c..cba74569 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -41,22 +41,32 @@ else fi # -------------------------------------------------------------------- -# 2. Genesis patches applied cleanly — especially the fragile tool_call one +# 2. Genesis patches applied cleanly # -------------------------------------------------------------------- -echo "[2/4] Genesis patches applied (Qwen3 tool_call fix in particular) ..." +# Anchors updated 2026-05-02 for Genesis v7.14+ logging conventions. The +# pre-v7.14 "[OK] Qwen3 tool_call fix" marker is no longer emitted; v7.14+ +# logs "[Genesis] applied:" per patch and "apply_all elapsed:" once at the +# end. The `|| true` after grep prevents `set -euo pipefail` from killing the +# pipeline when no anchor matches yet (e.g. mid-boot). Reported by +# @troymroberts in club-3090#25. +echo "[2/4] Genesis patches applied ..." if ! command -v docker >/dev/null 2>&1; then echo " (skipped — docker not in PATH, cannot read container logs)" elif ! docker inspect "${CONTAINER}" >/dev/null 2>&1; then echo " (skipped — container '${CONTAINER}' not found; if your container has a different name, set CONTAINER=...)" else - logs="$(docker logs "${CONTAINER}" 2>&1 | grep -E "Qwen3 tool_call fix|\[FAILED\]" | tail -5)" - if echo "$logs" | grep -q "\[OK\] Qwen3 tool_call fix"; then - pass "Genesis Qwen3 tool_call fix applied" - elif echo "$logs" | grep -q "\[FAILED\] Qwen3 tool_call fix"; then - fail "Genesis Qwen3 tool_call fix [FAILED]" \ - "Your vLLM image drifted past the patch anchor. Pin to sha256:9bba4628a3b9... in compose/docker-compose.yml (already pinned by default). Re-pull if you bumped it manually." + logs="$(docker logs "${CONTAINER}" 2>&1 \ + | { grep -E "apply_all elapsed|\[Genesis\] FAILED|\[Genesis\] applied:" || true; } \ + | tail -10)" + if echo "$logs" | grep -q "\[Genesis\] FAILED"; then + fail "Genesis apply_all reported FAILED patch(es)" \ + "Inspect: docker logs ${CONTAINER} 2>&1 | grep -E 'Genesis.*FAILED' | head" + elif echo "$logs" | grep -q "apply_all elapsed"; then + pass "Genesis patches applied (apply_all completed clean)" + elif echo "$logs" | grep -q "\[Genesis\] applied:"; then + pass "Genesis patches applied (apply_all may still be running)" else - echo " (warn — no Genesis OK/FAILED marker for tool_call in logs; container may have been restarted. Continuing.)" + echo " (warn — no Genesis marker in logs; container may have been restarted. Continuing.)" fi fi