diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..f53786f2 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +.git +.github + +models-cache +results + +**/cache +**/.cache +**/__pycache__ +**/*.pyc + +*.safetensors +*.gguf +*.bin +*.pt +*.pth + +.env +.env.* diff --git a/.github/workflows/build-vllm-image.yml b/.github/workflows/build-vllm-image.yml new file mode 100644 index 00000000..ecc83f00 --- /dev/null +++ b/.github/workflows/build-vllm-image.yml @@ -0,0 +1,397 @@ +name: Build vLLM Club3090 Image + +on: + workflow_dispatch: + inputs: + upstream_image: + description: "Upstream vLLM image to vendor overlays into" + required: false + default: "vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657" + smoke: + description: "GPU smoke behavior" + required: false + default: "auto" + type: choice + options: + - auto + - skip + - required + schedule: + - cron: "0 0 * * 0" + push: + branches: + - master + tags: + - "v0.7.*" + - "v0.[8-9].*" + - "v[1-9]*" + paths: + - ".github/workflows/build-vllm-image.yml" + - "docker/vllm-club3090/**" + - "models/qwen3.6-27b/vllm/patches/**" + - "models/gemma-4-31b/vllm/patches/**" + +concurrency: + group: build-vllm-club3090-image + cancel-in-progress: false + +permissions: + actions: read + contents: read + packages: write + +env: + IMAGE_NAME: ghcr.io/noonghunna/vllm-club3090 + DEFAULT_VLLM_BASE_IMAGE: vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657 + CANONICAL_COMPOSE: models/qwen3.6-27b/vllm/compose/dual/docker-compose.yml + +jobs: + build: + name: Build and push dated image + runs-on: ubuntu-latest + outputs: + image_ref: ${{ steps.meta.outputs.image_ref }} + image_tag: ${{ steps.meta.outputs.image_tag }} + tags: ${{ steps.meta.outputs.tags }} + upstream_image: ${{ steps.meta.outputs.upstream_image }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Compute image metadata + id: meta + shell: bash + env: + INPUT_UPSTREAM_IMAGE: ${{ inputs.upstream_image }} + run: | + set -euo pipefail + + upstream_image="${INPUT_UPSTREAM_IMAGE:-${DEFAULT_VLLM_BASE_IMAGE}}" + date_tag="$(date -u +%Y%m%d)" + counter="$(printf "%04d" "${GITHUB_RUN_NUMBER}")" + image_tag="nightly-${date_tag}-club${counter}" + image_ref="${IMAGE_NAME}:${image_tag}" + + { + echo "upstream_image=${upstream_image}" + echo "image_tag=${image_tag}" + echo "image_ref=${image_ref}" + echo "tags<> "${GITHUB_OUTPUT}" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push dated image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/vllm-club3090/Dockerfile + platforms: linux/amd64 + pull: true + push: true + build-args: | + VLLM_BASE_IMAGE=${{ steps.meta.outputs.upstream_image }} + tags: ${{ steps.meta.outputs.tags }} + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ steps.meta.outputs.image_tag }} + club3090.upstream_vllm_image=${{ steps.meta.outputs.upstream_image }} + + detect-smoke-runner: + name: Detect self-hosted GPU runner + needs: build + runs-on: ubuntu-latest + outputs: + available: ${{ steps.detect.outputs.available }} + smoke_mode: ${{ steps.detect.outputs.smoke_mode }} + steps: + - name: Detect online gpu-labeled runner + id: detect + uses: actions/github-script@v7 + env: + SMOKE_MODE: ${{ inputs.smoke || 'auto' }} + with: + script: | + const mode = process.env.SMOKE_MODE || "auto"; + core.setOutput("smoke_mode", mode); + + if (mode === "skip") { + core.notice("Smoke explicitly skipped. Dated image was pushed; aliases will not move."); + core.setOutput("available", "false"); + return; + } + + const runners = await github.paginate( + github.rest.actions.listSelfHostedRunnersForRepo, + { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100, + }, + ); + + const available = runners.some((runner) => { + const labels = runner.labels.map((label) => label.name.toLowerCase()); + return runner.status === "online" && + labels.includes("self-hosted") && + labels.includes("gpu"); + }); + + core.setOutput("available", available ? "true" : "false"); + if (!available) { + const message = "No online self-hosted runner with label 'gpu' was found. Dated image was pushed; latest/nightly-stable were not moved."; + if (mode === "required") { + core.setFailed(message); + } else { + core.notice(message); + } + } + + smoke: + name: GPU smoke and alias promotion + needs: + - build + - detect-smoke-runner + if: needs.detect-smoke-runner.outputs.available == 'true' + runs-on: + - self-hosted + - linux + - x64 + - gpu + timeout-minutes: 120 + env: + IMAGE_REF: ${{ needs.build.outputs.image_ref }} + IMAGE_NAME: ghcr.io/noonghunna/vllm-club3090 + COMPOSE_PROJECT_NAME: club3090-ci-vllm-dual + COMPOSE_OVERRIDE: /tmp/club3090-ci-vllm-image.override.yml + URL: http://localhost:8010 + MODEL: qwen3.6-27b-autoround + CONTAINER: vllm-qwen36-27b-dual + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare canonical compose override + shell: bash + run: | + set -euo pipefail + docker pull "${IMAGE_REF}" + cat > "${COMPOSE_OVERRIDE}" </dev/null; then + exit 0 + fi + sleep 10 + done + docker compose \ + -f "${CANONICAL_COMPOSE}" \ + -f "${COMPOSE_OVERRIDE}" \ + -p "${COMPOSE_PROJECT_NAME}" \ + logs --tail=200 + exit 1 + + - name: Run verify-full + shell: bash + run: | + set -euo pipefail + URL="${URL}" MODEL="${MODEL}" CONTAINER="${CONTAINER}" bash scripts/verify-full.sh + + - name: Run 3-prompt smoke bench + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import json + import time + import urllib.request + + url = "http://localhost:8010" + model = "qwen3.6-27b-autoround" + prompts = [ + ("narrative", "Write a concise paragraph explaining transformer attention.", 96), + ("code", "Write a small Python function that returns the nth Fibonacci number.", 96), + ("reasoning", "A train leaves at 08:00 traveling 60 km/h. Another leaves at 09:00 traveling 90 km/h. When does the second catch up?", 96), + ] + + for label, prompt, max_tokens in prompts: + body = json.dumps({ + "model": model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": 0.3, + "stream": False, + "chat_template_kwargs": {"enable_thinking": False}, + }).encode() + req = urllib.request.Request( + f"{url}/v1/chat/completions", + data=body, + headers={"Content-Type": "application/json"}, + ) + start = time.time() + with urllib.request.urlopen(req, timeout=300) as response: + data = json.load(response) + wall = time.time() - start + usage = data.get("usage") or {} + tokens = usage.get("completion_tokens") or 0 + text = data["choices"][0]["message"].get("content") or "" + if not text.strip(): + raise SystemExit(f"{label}: empty completion") + tps = tokens / wall if wall > 0 else 0 + print(f"{label}: wall={wall:.2f}s completion_tokens={tokens} wall_TPS={tps:.2f}") + PY + + - name: Promote latest aliases + shell: bash + run: | + set -euo pipefail + docker tag "${IMAGE_REF}" "${IMAGE_NAME}:latest" + docker tag "${IMAGE_REF}" "${IMAGE_NAME}:nightly-stable" + docker push "${IMAGE_NAME}:latest" + docker push "${IMAGE_NAME}:nightly-stable" + + - name: Cleanup canonical compose + if: always() + shell: bash + run: | + docker compose \ + -f "${CANONICAL_COMPOSE}" \ + -f "${COMPOSE_OVERRIDE}" \ + -p "${COMPOSE_PROJECT_NAME}" \ + down --remove-orphans || true + + retention: + name: Retain four weeks of dated nightlies + needs: build + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + steps: + - name: Delete old dated nightly package versions + uses: actions/github-script@v7 + with: + script: | + const packageType = "container"; + const packageName = "vllm-club3090"; + const cutoff = Date.now() - 28 * 24 * 60 * 60 * 1000; + + async function listVersions(scope) { + if (scope === "org") { + return github.paginate("GET /orgs/{org}/packages/{package_type}/{package_name}/versions", { + org: context.repo.owner, + package_type: packageType, + package_name: packageName, + per_page: 100, + }); + } + return github.paginate("GET /users/{username}/packages/{package_type}/{package_name}/versions", { + username: context.repo.owner, + package_type: packageType, + package_name: packageName, + per_page: 100, + }); + } + + async function deleteVersion(scope, id) { + if (scope === "org") { + return github.request("DELETE /orgs/{org}/packages/{package_type}/{package_name}/versions/{package_version_id}", { + org: context.repo.owner, + package_type: packageType, + package_name: packageName, + package_version_id: id, + }); + } + return github.request("DELETE /users/{username}/packages/{package_type}/{package_name}/versions/{package_version_id}", { + username: context.repo.owner, + package_type: packageType, + package_name: packageName, + package_version_id: id, + }); + } + + let scope = "org"; + let versions = []; + try { + versions = await listVersions(scope); + } catch (error) { + if (error.status !== 404) throw error; + scope = "user"; + try { + versions = await listVersions(scope); + } catch (userError) { + if (userError.status === 404) { + core.notice(`Package ${packageName} does not exist yet; retention skipped.`); + return; + } + throw userError; + } + } + + for (const version of versions) { + const tags = version.metadata?.container?.tags || []; + if (tags.length === 0) continue; + + const protectedTag = tags.some((tag) => + tag === "latest" || + tag === "nightly-stable" || + tag.startsWith("club-v") + ); + const datedNightly = tags.some((tag) => /^nightly-\d{8}-club\d{4,}$/.test(tag)); + const updatedAt = new Date(version.updated_at).getTime(); + + if (datedNightly && !protectedTag && updatedAt < cutoff) { + core.notice(`Deleting old ${packageName} package version ${version.id}: ${tags.join(", ")}`); + await deleteVersion(scope, version.id); + } + } diff --git a/docker/vllm-club3090/Dockerfile b/docker/vllm-club3090/Dockerfile new file mode 100644 index 00000000..87d341a0 --- /dev/null +++ b/docker/vllm-club3090/Dockerfile @@ -0,0 +1,46 @@ +ARG VLLM_BASE_IMAGE=vllm/vllm-openai:nightly-1acd67a795ebccdf9b9db7697ae9082058301657 +FROM ${VLLM_BASE_IMAGE} + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +LABEL org.opencontainers.image.title="club-3090 vLLM" +LABEL org.opencontainers.image.description="vLLM nightly with club-3090 vendored overlays baked into site-packages" +LABEL org.opencontainers.image.source="https://github.com/noonghunna/club-3090" +LABEL org.opencontainers.image.vendor="club-3090" + +COPY models/qwen3.6-27b/vllm/patches/vllm-marlin-pad/marlin.py /tmp/club3090-overlays/qwen-marlin-pad/model_executor/kernels/linear/mixed_precision/marlin.py +COPY models/qwen3.6-27b/vllm/patches/vllm-marlin-pad/MPLinearKernel.py /tmp/club3090-overlays/qwen-marlin-pad/model_executor/kernels/linear/mixed_precision/MPLinearKernel.py +COPY models/qwen3.6-27b/vllm/patches/vllm-pr35936-required-fallback/vllm/ /tmp/club3090-overlays/qwen-pr35936/ +COPY models/gemma-4-31b/vllm/patches/vllm-gemma4-dflash-int8/ /tmp/club3090-overlays/gemma-dflash-int8/ + +RUN set -eux; \ + VLLM_SITE="$(python3 -c 'import importlib.util, pathlib; spec = importlib.util.find_spec("vllm"); print(pathlib.Path(spec.submodule_search_locations[0]) if spec and spec.submodule_search_locations else "")')"; \ + test -n "${VLLM_SITE}"; \ + test -d "${VLLM_SITE}"; \ + echo "Applying club-3090 overlays into ${VLLM_SITE}"; \ + find /tmp/club3090-overlays/qwen-marlin-pad -type f -name '*.py' -print0 \ + | while IFS= read -r -d '' src; do \ + rel="${src#/tmp/club3090-overlays/qwen-marlin-pad/}"; \ + install -D -m 0644 "$src" "${VLLM_SITE}/${rel}"; \ + done; \ + find /tmp/club3090-overlays/qwen-pr35936 -type f -name '*.py' -print0 \ + | while IFS= read -r -d '' src; do \ + rel="${src#/tmp/club3090-overlays/qwen-pr35936/}"; \ + install -D -m 0644 "$src" "${VLLM_SITE}/${rel}"; \ + done; \ + find /tmp/club3090-overlays/gemma-dflash-int8 -type f -name '*.py' ! -path '*/_pre-pr42102-historical/*' -print0 \ + | while IFS= read -r -d '' src; do \ + rel="${src#/tmp/club3090-overlays/gemma-dflash-int8/}"; \ + install -D -m 0644 "$src" "${VLLM_SITE}/${rel}"; \ + done; \ + mkdir -p /etc/club3090; \ + { \ + echo "vllm base image: ${VLLM_BASE_IMAGE}"; \ + echo "vendored overlays:"; \ + echo " - vLLM PR #40361: Marlin pad-sub-tile-n for AutoRound INT4 TP shards"; \ + echo " - vLLM PR #35936: Qwen3-Coder required-tool fallback serving patch"; \ + echo " - vLLM PR #41703: Gemma 4 DFlash drafter"; \ + echo " - vLLM PR #42102: DFlash + INT8 per-token-head KV coexistence"; \ + echo "source repo: https://github.com/noonghunna/club-3090"; \ + } > /etc/club3090/vllm-overlays.txt; \ + rm -rf /tmp/club3090-overlays diff --git a/docs/CI_RUNNER_SETUP.md b/docs/CI_RUNNER_SETUP.md new file mode 100644 index 00000000..72c42f10 --- /dev/null +++ b/docs/CI_RUNNER_SETUP.md @@ -0,0 +1,112 @@ +# Club-3090 CI GPU Runner Setup + +The vLLM image workflow builds on GitHub-hosted Ubuntu, then optionally smokes the +fresh image on a self-hosted GPU runner. If no runner is registered, the workflow +still pushes the dated `nightly-YYYYMMDD-clubXXXX` image and leaves `latest` / +`nightly-stable` untouched. + +## Runner Requirements + +- Linux x86_64 host with Docker Engine and Docker Compose v2. +- NVIDIA driver and NVIDIA Container Toolkit installed. +- At least two 24 GB NVIDIA GPUs for the canonical `qwen3.6-27b/vllm/dual` + smoke. The production validation target is 2x RTX 3090. +- Enough local storage for the vLLM image, model cache, Docker layers, and + compile caches. Plan for at least 250 GB free. +- A dedicated runner host. Do not run untrusted pull-request jobs on this + machine. + +## Labels + +Register the runner with the normal self-hosted labels plus `gpu`: + +```text +self-hosted +linux +x64 +gpu +``` + +The workflow checks for an online runner with `self-hosted` and `gpu`; the smoke +job itself targets `[self-hosted, linux, x64, gpu]`. + +## Registration + +1. Open the GitHub repository. +2. Go to **Settings -> Actions -> Runners -> New self-hosted runner**. +3. Choose Linux x64 and follow GitHub's generated commands. +4. Add the `gpu` label during configuration, or add it later from the runner UI. +5. Install the runner as a service: + +```bash +sudo ./svc.sh install +sudo ./svc.sh start +``` + +The runner user must be able to run Docker commands. On a typical Ubuntu host: + +```bash +sudo usermod -aG docker "$USER" +newgrp docker +``` + +Restart the runner service after changing group membership. + +## Host Preflight + +Run these on the runner host before enabling the smoke job: + +```bash +nvidia-smi +docker compose version +docker run --rm --gpus all nvidia/cuda:12.8.0-base-ubuntu24.04 nvidia-smi +``` + +Then clone this repository at the path used by the runner workspace once and +make sure the model cache is present or mounted at the compose default: + +```bash +ls -ld models-cache +``` + +If your cache lives elsewhere, set `MODEL_DIR` in the runner service +environment. The canonical compose reads `${MODEL_DIR:-../../../../../models-cache}`. + +## What The Smoke Job Does + +On a green build, the workflow: + +1. Pulls `ghcr.io/noonghunna/vllm-club3090:nightly-YYYYMMDD-clubXXXX`. +2. Boots `models/qwen3.6-27b/vllm/compose/dual/docker-compose.yml` with a + temporary compose override that points at the dated image. +3. Waits for `http://localhost:8010/v1/models`. +4. Runs `bash scripts/verify-full.sh`. +5. Runs a three-prompt OpenAI-compatible smoke bench. +6. Only then retags the image as `latest` and `nightly-stable`. + +If any smoke step fails, the dated image remains available for debugging and the +rolling aliases do not move. + +## Existing Containers + +The runner should be dedicated to CI. Before booting the canonical compose, the +workflow tears down the default club-3090 estate if `~/.club3090/estate.yml` +exists, then runs `docker compose down` for the CI project name. Avoid running +manual workloads on the same host while the workflow is active. + +## Registry Permissions + +The workflow uses `GITHUB_TOKEN` with `packages: write` to push GHCR images and +move aliases. No personal access token is required for the repository-owned +package. + +## Retention + +The scheduled workflow keeps: + +- `latest` +- `nightly-stable` +- every `club-v*` release tag +- dated `nightly-YYYYMMDD-clubXXXX` tags from the last four weeks + +Older dated nightly package versions are deleted by the retention job.