diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..782823ae --- /dev/null +++ b/.env.example @@ -0,0 +1,87 @@ +# club-3090 environment overrides +# +# Copy to `.env` (or export inline) and edit only the values you want to +# change. Every variable below has a sensible default in the compose files +# and scripts — `.env` is purely for opt-in overrides. +# +# cp .env.example .env +# $EDITOR .env +# +# Variables are read by `docker compose ...` automatically when run from the +# directory that contains a `.env`. For scripts/setup.sh you can either +# `export` them in your shell or prefix the command: +# +# MODEL_DIR=/scratch/models bash scripts/setup.sh qwen3.6-27b + + +# ----------------------------------------------------------------------------- +# Model storage +# ----------------------------------------------------------------------------- + +# Where weights live. Default is /models-cache. Override if you keep +# weights on a separate disk (e.g. /mnt/models). Path is resolved as-is — +# absolute paths recommended. +# MODEL_DIR=/mnt/models + +# HuggingFace token. Only needed for gated/private repos. The Lorbus +# AutoRound INT4 weights and unsloth GGUFs we use are public, so this is +# usually blank. +# HF_TOKEN=hf_xxxxxxxxxxxxxxxxxxxxxxxxx + + +# ----------------------------------------------------------------------------- +# GPU selection +# ----------------------------------------------------------------------------- + +# Which GPUs to expose to docker. Single-card composes use just `0`, +# dual-card composes use `0,1`. The compose files set this themselves; +# override here if your physical layout differs (e.g. you run dual-card +# on cards 2,3). +# CUDA_VISIBLE_DEVICES=0,1 + + +# ----------------------------------------------------------------------------- +# vLLM tuning knobs +# ----------------------------------------------------------------------------- + +# GPU memory utilization fraction. The shipped composes use: +# single-card default (48K): 0.92 +# single-card long-vision/text: 0.98 +# dual-card: 0.92 +# Lowering this leaves more headroom for prefill spikes — see +# docs/INTERNALS.md "prefill cliffs". Raising above 0.95 is risky on +# the 48K config but fine on long-* configs because the model + KV +# fully consume the card anyway. +# MEM_UTIL=0.92 + +# Max sequence length. Shipped defaults: 48K (chat-safe), 192K (long-vision), +# 205K (long-text). Going beyond these on a 24GB card needs --mem-fraction +# tuning; see docs/USE_CASES.md for the curve. +# MAX_MODEL_LEN=48000 + + +# ----------------------------------------------------------------------------- +# Genesis patch tree (vLLM only) +# ----------------------------------------------------------------------------- + +# Pin Sandermage/genesis-vllm-patches to a specific ref. Default is the exact +# commit our published TPS numbers were measured against. Bumping requires +# re-running scripts/verify-full.sh to confirm your new ref still passes. +# GENESIS_PIN=bf667c7 + +# Skip cloning Genesis (e.g. you only intend to run llama.cpp / SGLang). +# SKIP_GENESIS=1 + + +# ----------------------------------------------------------------------------- +# verify-full.sh / verify-stress.sh +# ----------------------------------------------------------------------------- + +# Endpoint to test against. Default localhost:8020 matches every shipped +# compose's host port. +# URL=http://localhost:8020 + +# Number of warmup + measured runs in the bench script. Defaults are +# 3 warmup + 5 measured. Lower these for a faster smoke test. +# WARMUPS=3 +# RUNS=5 diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000..135b2551 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,113 @@ +name: Bug report +description: Something isn't working — server crashes, OOM, wrong output, weird TPS regression, etc. +title: "[bug] " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + Thanks for filing a bug! The data we ask for below has saved hours + of back-and-forth on every previous report. Please don't skip the + log/`verify-full.sh`/`nvidia-smi` blocks — they're how we tell a + local-config issue from a real upstream regression. + + - type: textarea + id: what-happened + attributes: + label: What happened? + description: One-paragraph summary. What did you expect, what did you see instead? + placeholder: | + I started `dual-turbo.yml` and it OOMs after ~30 seconds of prefill on a 32K + message. Expected: stable serving like the README says. Actual: the + attached log line. + validations: + required: true + + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: Exact commands. The `compose up` line, the curl/openai-client request, etc. + placeholder: | + cd models/qwen3.6-27b/vllm/compose + docker compose -f docker-compose.dual-turbo.yml up -d + # then this curl request: + curl -s http://localhost:8020/v1/chat/completions -d '...' + render: bash + validations: + required: true + + - type: textarea + id: docker-logs + attributes: + label: docker logs (last 100 lines) + description: | + Output of `docker logs --tail 100 2>&1`. The container + name is the `container_name:` field in the compose file you're running. + placeholder: | + docker logs --tail 100 vllm-qwen36-27b 2>&1 + # paste the output here + render: text + validations: + required: true + + - type: textarea + id: verify-full + attributes: + label: verify-full.sh output + description: | + Run `bash scripts/verify-full.sh` (or `verify-stress.sh` if that's + what's failing) against your running container and paste the full output. + This tells us which check fails on your rig vs. ours. + placeholder: | + bash scripts/verify-full.sh + # paste full output + render: text + validations: + required: true + + - type: textarea + id: nvidia-smi + attributes: + label: nvidia-smi output + description: | + `nvidia-smi` output captured while the container is running. We need to + see VRAM use per card, driver version, and CUDA version. + placeholder: | + nvidia-smi + # paste full output + render: text + validations: + required: true + + - type: input + id: gpu-count + attributes: + label: GPU configuration + placeholder: "1× RTX 3090, or 2× RTX 3090 (PCIe, no NVLink), etc." + validations: + required: true + + - type: input + id: compose-variant + attributes: + label: Compose variant + description: Which `docker-compose.*.yml` file are you running? + placeholder: "docker-compose.yml (default 48K) / dual-turbo.yml / long-text.yml / ..." + validations: + required: true + + - type: input + id: commit + attributes: + label: club-3090 commit + description: Output of `git -C /path/to/club-3090 rev-parse --short HEAD`. + placeholder: "e.g. 5060e22" + validations: + required: true + + - type: textarea + id: extras + attributes: + label: Anything else? + description: Custom environment overrides (`.env`), non-default `MEM_UTIL`, distro/kernel oddities, anything that might matter. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..9095d878 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: General Q&A / discussion + url: https://github.com/noonghunna/club-3090/discussions + about: Questions about setup, hardware, or "should I tune X" — please use Discussions instead of opening an issue. diff --git a/.github/ISSUE_TEMPLATE/numbers-from-your-rig.yml b/.github/ISSUE_TEMPLATE/numbers-from-your-rig.yml new file mode 100644 index 00000000..0696b202 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/numbers-from-your-rig.yml @@ -0,0 +1,78 @@ +name: Numbers from your rig +description: Cross-rig TPS / context-ceiling contribution. We collect these and add them to BENCHMARKS. +title: "[bench] " +labels: ["bench-contribution"] +body: + - type: markdown + attributes: + value: | + Thanks for benching! Your numbers help us tell which results are + rig-dependent vs. universal. We'll add high-quality contributions to + our benchmarks table with attribution. + + Please use the canonical bench prompt so your numbers are comparable + to the ones in the repo: + + > **Narrative prompt:** "Write a detailed 800-word essay explaining transformer attention." + > `max_tokens=1000, temperature=0.6, top_p=0.95, top_k=20, enable_thinking=false` + > + > **Code prompt:** "Write a Python implementation of quicksort with comments explaining each step." + > `max_tokens=800`, same sampling. + + The shipped `scripts/bench.sh` runs both with these exact settings. + + - type: input + id: rig + attributes: + label: Rig — GPUs + interconnect + placeholder: "1× RTX 3090, or 2× RTX 3090 (PCIe Gen4 x16, no NVLink), or 1× RTX 4090, ..." + validations: + required: true + + - type: input + id: cpu-ram + attributes: + label: CPU + system RAM + placeholder: "Ryzen 9 5950X / 64 GB DDR4-3600" + validations: + required: true + + - type: input + id: compose-variant + attributes: + label: Compose variant + commit + description: Which compose file did you run, and at which club-3090 commit? + placeholder: "docker-compose.yml @ 5060e22" + validations: + required: true + + - type: textarea + id: bench-output + attributes: + label: scripts/bench.sh output + description: Full output of `bash scripts/bench.sh`. Includes both narrative + code TPS, AL, and per-run numbers. + render: text + validations: + required: true + + - type: input + id: vram + attributes: + label: VRAM used (per card, peak) + description: From `nvidia-smi --query-gpu=memory.used --format=csv` while the bench is running. + placeholder: "23.6 GB / 24 GB" + validations: + required: true + + - type: input + id: max-ctx + attributes: + label: Max context that boots cleanly + description: If you pushed `--max-model-len` past the shipped default, what's the highest value that boots and survives `verify-full.sh`? + placeholder: "48K (shipped default), or 90K, or 'didn't try'" + + - type: textarea + id: notes + attributes: + label: Notes + description: Anything else worth knowing — driver version, distro, custom flags, surprising failures. diff --git a/.gitignore b/.gitignore index 295359fc..2d9ea1e6 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,8 @@ models-cache/ # Genesis patches fetched by setup.sh (upstream, not ours to vendor). # Lives at models//vllm/patches/genesis/ in the new layout. -models/*/vllm/patches/genesis/ +# No trailing slash — also matches local symlinks pointing at out-of-tree clones. +models/*/vllm/patches/genesis # Python __pycache__/ diff --git a/scripts/setup.sh b/scripts/setup.sh index 9652af0e..44982ddd 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -73,25 +73,29 @@ echo "Model dir: ${MODEL_DIR}" # between v7.13 (monolithic patch_genesis_unified.py shim) and v7.14 (modular # vllm/_genesis package + per-patch env opts). Newer composes mount the package; # the legacy compose still references the v7.13 shim. -# Pin Genesis to a tested tag — addresses the "out-of-sync genesis dir vs -# compose variant" concern from issue #8 in the predecessor repo. We test -# composes against this specific tag; bumping requires re-running -# verify-full.sh to confirm the new tag works on your config. -GENESIS_PIN="${GENESIS_PIN:-v7.51-stable-2026-04-27}" +# Pin Genesis to the exact commit our published numbers were measured against. +# This is bf667c7 (Genesis HEAD as of 2026-04-27 — semver "v7.54", untagged). +# Tagged stable v7.51-stable-2026-04-27 is one minor older; we ship the exact +# tested commit instead so reproducibility doesn't depend on the upstream tag +# moving or being garbage-collected. Bumping requires re-running verify-full.sh +# to confirm the new commit works on your config. +GENESIS_PIN="${GENESIS_PIN:-bf667c7}" if [[ "${SKIP_GENESIS:-0}" != "1" ]]; then if [[ -d "${GENESIS_DIR}/.git" ]]; then echo "[genesis] Already cloned at ${GENESIS_DIR} — fetching + checking out ${GENESIS_PIN} ..." - (cd "${GENESIS_DIR}" && git fetch --tags origin && git checkout "${GENESIS_PIN}" 2>&1 | tail -3) + (cd "${GENESIS_DIR}" && git fetch origin && git checkout "${GENESIS_PIN}" 2>&1 | tail -3) else echo "[genesis] Cloning Sandermage/genesis-vllm-patches at ${GENESIS_PIN} ..." - git clone --branch "${GENESIS_PIN}" --depth 1 https://github.com/Sandermage/genesis-vllm-patches.git "${GENESIS_DIR}" + # Full clone (commit SHAs aren't reachable via --branch + --depth 1). + git clone https://github.com/Sandermage/genesis-vllm-patches.git "${GENESIS_DIR}" + (cd "${GENESIS_DIR}" && git checkout "${GENESIS_PIN}") fi - # v7.14 layout sanity check + # v7.14+ layout sanity check if [[ ! -d "${GENESIS_DIR}/vllm/_genesis" ]]; then echo "ERROR: genesis tree at ${GENESIS_PIN} missing vllm/_genesis package." >&2 - echo " Re-run with GENESIS_PIN= to try a different version." >&2 + echo " Re-run with GENESIS_PIN= to try a different version." >&2 exit 1 fi echo "[genesis] Pinned to ${GENESIS_PIN} ($(cd "${GENESIS_DIR}" && git rev-parse --short HEAD))"