club-3090/scripts/bench.sh

#!/usr/bin/env bash
#
# Canonical bench against the running vLLM service.
#   - Runs both the canonical narrative AND code prompts in one invocation.
#     This matches the README's narrative/code TPS pairing.
#   - 3 warmup + N measured runs per prompt (default 5 narrative + 5 code).
#   - per-run: wall time, TTFT (via streaming), completion tokens,
#     wall_TPS (= comp / wall), decode_TPS (= comp / (wall - TTFT))
#   - per-prompt summary: mean / std / CV for both TPS metrics + mean TTFT
#     + prompt-processing throughput (`PP tok/s`)
#   - shows MTP SpecDecoding metrics from docker logs at the end
#
# Why two TPS metrics:
#   - wall_TPS  = "user-perceived speed" (includes prefill cost)
#   - decode_TPS = "model decode rate" (excludes prefill)
#   For long prompts the two can differ a lot. For short prompts they
#   converge. Reporting both keeps comparisons honest across configs.
#
# Why narrative + code:
#   MTP acceptance varies wildly by prompt structure. Code (repetitive,
#   token-predictable) accepts at ~80% per position; prose (semantically
#   rich) at ~50%. Reporting only one half is misleading. README claims
#   like "66 / 85 TPS" pair them; bench should too.
#
# Prereq: stack is running and reports "Application startup complete".
#
# Env vars:
#   URL                Endpoint. Default: http://localhost:8020
#   MODEL              Served model name. Default: auto-detected from
#                      /v1/models, else qwen3.6-27b
#   CONTAINER          Container for log scraping. Default: vllm-qwen36-27b
#   RUNS               Measured runs per prompt. Default: 5
#   WARMUPS            Warm-up runs (shared across both). Default: 3
#   PROMPT_NARR        Override narrative prompt
#   PROMPT_CODE        Override code prompt
#   MAX_TOKENS_NARR    Default: 1000
#   MAX_TOKENS_CODE    Default: 800
#   ONLY               Set to "narr" or "code" to skip the other. Default: both
#   QUIET              Set to 1 to skip per-run lines (just print summary)
#   PP                 Set to 1 to add the long-prompt PP fallback probe.
#                      llama.cpp containers enable this automatically.
#   PP_FALLBACK_TOKENS Approximate filler-token target for PP=1. Default: 10000
#   PP_MAX_TOKENS      Completion cap for the PP fallback request. Default: 16
#   PREFILL_PROBE      Set to 0 to skip the canonical two-depth prefill/TTFT
#                      probe (catalog-baselines 2c). Default: 1. Each request
#                      uses a FRESH salted haystack — composes serve
#                      enable_prefix_caching, so identical prompts re-measure
#                      the CACHE HIT, not prefill. Depths exceeding the served
#                      context are skipped with a note.
#   PREFILL_DEPTHS     CSV of prompt-token depths. Default: 10000,90000 (the
#                      shallow/deep warm anchors; 90K sits inside the DeltaNet
#                      degradation regime and pairs with the verify-stress
#                      ladder's ~94K rung for anchor calibration).
#   PREFILL_RUNS       Measured runs per depth. Default: 3
#   ENABLE_THINKING    Set to 1 to send chat_template_kwargs.enable_thinking=true
#                      in bench requests. Default: 0.
#   FORCE_TOKENS       Force EXACTLY this many output tokens per run (sets
#                      max_tokens + min_tokens + ignore_eos), overriding
#                      MAX_TOKENS_NARR/CODE. Use to bench at a fixed / larger output
#                      size. Essential for DIFFUSION LMs: they self-terminate early
#                      (~1-2K words), so raising the cap alone won't lengthen the
#                      generation — you must force the length to measure sustained
#                      throughput. vLLM-oriented (ignore_eos/min_tokens). 0 = off
#                      (model decides length). Default: 0.
#
# Usage:
#   bash scripts/bench.sh
#   ONLY=code bash scripts/bench.sh
#   PP=1 bash scripts/bench.sh
#   RUNS=10 bash scripts/bench.sh
#   FORCE_TOKENS=4000 bash scripts/bench.sh   # fixed 4000-tok output (diffusion / sustained-TPS)

set -euo pipefail

# Auto-detect running container + port (URL/CONTAINER env vars still win).
# See scripts/preflight.sh::preflight_autodetect_endpoint.
ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
if [[ -f "${ROOT_DIR}/scripts/preflight.sh" ]]; then
  # shellcheck source=preflight.sh
  source "${ROOT_DIR}/scripts/preflight.sh"
  preflight_autodetect_endpoint || true
fi
URL="${URL:-http://localhost:8020}"
# Resolve the served model from /v1/models when MODEL is unset (#372). The qwen
# literal below is only a last resort if detection no-ops (endpoint unreachable).
declare -F preflight_autodetect_model >/dev/null && preflight_autodetect_model
MODEL="${MODEL:-qwen3.6-27b}"
CONTAINER="${CONTAINER:-vllm-qwen36-27b}"
RUNS="${RUNS:-5}"
WARMUPS="${WARMUPS:-3}"
MAX_TOKENS_NARR="${MAX_TOKENS_NARR:-1000}"
MAX_TOKENS_CODE="${MAX_TOKENS_CODE:-800}"
PROMPT_NARR="${PROMPT_NARR:-Write a detailed 800-word essay explaining transformer attention.}"
PROMPT_CODE="${PROMPT_CODE:-Write a Python implementation of quicksort with comments explaining each step.}"
ONLY="${ONLY:-both}"
QUIET="${QUIET:-0}"
PP="${PP:-0}"
PP_FALLBACK_TOKENS="${PP_FALLBACK_TOKENS:-10000}"
PP_MAX_TOKENS="${PP_MAX_TOKENS:-16}"
PREFILL_PROBE="${PREFILL_PROBE:-1}"
PREFILL_DEPTHS="${PREFILL_DEPTHS:-10000,90000}"
PREFILL_RUNS="${PREFILL_RUNS:-3}"
# The probe knobs reach the python heredoc via the environment (the argv tuple
# is full); export them here.
export PREFILL_PROBE PREFILL_DEPTHS PREFILL_RUNS
ENABLE_THINKING="${ENABLE_THINKING:-0}"
FORCE_TOKENS="${FORCE_TOKENS:-0}"

need() {
  command -v "$1" >/dev/null 2>&1 || { echo "ERROR: '$1' not in PATH." >&2; exit 1; }
}
need curl
need python3

ENGINE_KIND="${ENGINE_KIND:-unknown}"
if [[ "$ENGINE_KIND" == "unknown" && "${CONTAINER:-}" != "none" ]] && command -v docker >/dev/null 2>&1 && docker inspect "${CONTAINER}" >/dev/null 2>&1; then
  container_image="$(docker inspect --format '{{.Config.Image}}' "${CONTAINER}" 2>/dev/null || true)"
  container_name="$(docker inspect --format '{{.Name}}' "${CONTAINER}" 2>/dev/null || true)"
  if [[ "${container_image} ${container_name}" == *"llama.cpp"* || "${container_image} ${container_name}" == *"llama-cpp"* ]]; then
    ENGINE_KIND="llamacpp"
  elif [[ "${container_image} ${container_name}" == *"vllm"* ]]; then
    ENGINE_KIND="vllm"
  fi
fi

PP_MODE="log"
if [[ "$PP" == "1" || "$ENGINE_KIND" == "llamacpp" ]]; then
  PP_MODE="fallback"
fi

if [[ "$ENABLE_THINKING" == "1" ]]; then
  echo "[bench] thinking: enabled (request chat_template_kwargs.enable_thinking=true)" >&2
fi

if [[ "${BENCH_MOCK:-0}" == "1" ]]; then
  if [[ "$PP_MODE" == "fallback" ]]; then
    cat <<'EOF'

========== PROMPT-PROCESSING (fallback target=10000 prompt tokens, max_tokens=16) ==========
=== measured (1) ===
  run-1      wall=  3.20s  ttft= 2500ms  prompt_toks=  9876  PP_tok/s=3950.40

=== summary [prompt-processing] (n=1) ===
  PP tok/s       mean=3950.40   std=  0.00   CV= 0.0%   min=3950.40   max=3950.40
  TTFT          mean=  2500ms  std=    0ms  min=2500ms  max=2500ms
EOF
  else
    cat <<'EOF'

========== NARRATIVE (prompt=61 chars, max_tokens=1000) ==========
=== measured (1) ===
  run-1      wall=  4.20s  ttft=   120ms  toks=1000  wall_TPS=238.10  decode_TPS=245.10

=== summary [narrative] (n=1) ===
  wall_TPS       mean= 238.10   std=  0.00   CV= 0.0%   min=238.10   max=238.10
  decode_TPS     mean= 245.10   std=  0.00   CV= 0.0%   min=245.10   max=245.10
  TTFT          mean=   120ms  std=    0ms  min=120ms  max=120ms
  PP tok/s       mean=2843.21   std=  0.00   CV= 0.0%   min=2843.21   max=2843.21
EOF
  fi
  exit 0
fi

if ! curl -sf "${URL}/v1/models" >/dev/null; then
  echo "ERROR: service not reachable at ${URL}/v1/models" >&2
  echo "  Start with: cd compose && docker compose up -d" >&2
  exit 1
fi

server_reasoning_on() {
  if curl -sf -m 3 "${URL}/props" 2>/dev/null | python3 -c '
import json, sys
try:
    obj = json.load(sys.stdin)
except Exception:
    sys.exit(1)

def walk(x):
    if isinstance(x, dict):
        for k, v in x.items():
            lk = str(k).lower()
            if lk in {"reasoning", "enable_reasoning"}:
                if v is True or str(v).lower() in {"1", "true", "on", "yes"}:
                    return True
            if walk(v):
                return True
    elif isinstance(x, list):
        return any(walk(v) for v in x)
    return False
sys.exit(0 if walk(obj) else 1)
' >/dev/null 2>&1; then
    return 0
  fi
  if [[ -n "${CONTAINER:-}" && "${CONTAINER:-}" != "none" ]] \
     && command -v docker >/dev/null 2>&1 \
     && docker inspect "$CONTAINER" >/dev/null 2>&1; then
    docker inspect "$CONTAINER" 2>/dev/null \
      | grep -Eq -- '(--reasoning[= ]+on|"--reasoning"[[:space:]]*,[[:space:]]*"on")' && return 0
  fi
  return 1
}

if [[ "$ENABLE_THINKING" != "1" ]] && server_reasoning_on; then
  echo "[bench] WARN: server appears to have reasoning enabled, but bench requests send enable_thinking=false. Use ENABLE_THINKING=1 for reasoning-on TPS." >&2
fi

python3 - "$URL" "$MODEL" "$WARMUPS" "$RUNS" "$QUIET" "$ONLY" \
            "$CONTAINER" "$PP_MODE" "$PP_FALLBACK_TOKENS" "$PP_MAX_TOKENS" \
            "$ENABLE_THINKING" "$FORCE_TOKENS" \
            "$PROMPT_NARR" "$MAX_TOKENS_NARR" \
            "$PROMPT_CODE" "$MAX_TOKENS_CODE" << 'PYEOF'
import json, os, random, re, shutil, string, subprocess, sys, time, urllib.request, statistics as s

(URL, MODEL, WARMUPS, RUNS, QUIET, ONLY,
 CONTAINER, PP_MODE, PP_FALLBACK_TOKENS, PP_MAX_TOKENS,
 ENABLE_THINKING, FORCE, PROMPT_NARR, MAX_NARR, PROMPT_CODE, MAX_CODE) = sys.argv[1:]
WARMUPS = int(WARMUPS); RUNS = int(RUNS); QUIET = int(QUIET) == 1
MAX_NARR = int(MAX_NARR); MAX_CODE = int(MAX_CODE)
PP_FALLBACK_TOKENS = int(PP_FALLBACK_TOKENS); PP_MAX_TOKENS = int(PP_MAX_TOKENS)
ENABLE_THINKING = ENABLE_THINKING == "1"
FORCE = int(FORCE)   # >0: force EXACTLY this many output tokens (max+min+ignore_eos)

def run_once(prompt, max_tokens):
    # FORCE>0: force EXACTLY FORCE output tokens (overrides the per-prompt cap) so the
    # model can't self-terminate early — required to measure sustained throughput at a
    # chosen output size on diffusion LMs (which stop ~1-2K words otherwise).
    mt = FORCE if FORCE > 0 else max_tokens
    req_body = {
        "model": MODEL,
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": mt,
        "temperature": 0.6,
        "top_p": 0.95,
        "stream": True,
        "stream_options": {"include_usage": True},
        "chat_template_kwargs": {"enable_thinking": ENABLE_THINKING},
    }
    if FORCE > 0:
        req_body["min_tokens"] = FORCE
        req_body["ignore_eos"] = True
    body = json.dumps(req_body).encode()
    req = urllib.request.Request(f"{URL}/v1/chat/completions", data=body,
                                 headers={"Content-Type": "application/json"})
    t_send = time.time()
    ttft = None
    completion_tokens = 0
    prompt_tokens = 0
    with urllib.request.urlopen(req, timeout=600) as r:
        for line in r:
            line = line.decode("utf-8", errors="ignore").rstrip()
            if not line.startswith("data: "):
                continue
            payload = line[6:]
            if payload == "[DONE]":
                break
            try:
                chunk = json.loads(payload)
            except json.JSONDecodeError:
                continue
            choices = chunk.get("choices") or []
            if choices:
                delta = choices[0].get("delta", {})
                content = delta.get("content") or delta.get("reasoning_content")
                if content and ttft is None:
                    ttft = time.time() - t_send
            usage = chunk.get("usage")
            if usage:
                completion_tokens = usage.get("completion_tokens", completion_tokens)
                prompt_tokens = usage.get("prompt_tokens", prompt_tokens)
    t_end = time.time()
    wall = t_end - t_send
    if ttft is None:
        ttft = wall
    if not prompt_tokens:
        prompt_tokens = max(1, len(prompt.split()))
    return wall, ttft, completion_tokens, prompt_tokens

def fmt(label, wall, ttft, toks):
    decode_t = max(wall - ttft, 1e-6)
    wtps = toks / wall if wall > 0 else 0
    dtps = toks / decode_t
    line = f"  {label:<10s} wall={wall:6.2f}s  ttft={ttft*1000:6.0f}ms  toks={toks:>4d}  wall_TPS={wtps:6.2f}  decode_TPS={dtps:6.2f}"
    return wtps, dtps, ttft, line

def fmt_pp(label, wall, ttft, prompt_tokens):
    pp = prompt_tokens / max(ttft, 1e-6)
    line = f"  {label:<10s} wall={wall:6.2f}s  ttft={ttft*1000:6.0f}ms  prompt_toks={prompt_tokens:>6d}  PP_tok/s={pp:7.2f}"
    return pp, ttft, line

def stats(name, xs, unit=""):
    m = s.mean(xs)
    sd = s.stdev(xs) if len(xs) > 1 else 0
    cv = (sd / m * 100) if m > 0 else 0
    return f"  {name:<14s} mean={m:7.2f}{unit}   std={sd:6.2f}   CV={cv:4.1f}%   min={min(xs):.2f}   max={max(xs):.2f}"

def scrape_prompt_throughput(container, n):
    if not container or container == "none" or shutil.which("docker") is None:
        return []
    try:
        proc = subprocess.run(
            ["docker", "logs", container],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT,
            text=True,
            errors="replace",
            timeout=10,
            check=False,
        )
    except Exception:
        return []
    vals = [
        float(m.group(1))
        for m in re.finditer(r"Avg prompt throughput:\s*([0-9]+(?:\.[0-9]+)?)\s*tokens/s", proc.stdout)
    ]
    return vals[-max(n, 1):]

def run_set(label, prompt, max_tokens):
    print(f"\n========== {label.upper()} (prompt={len(prompt)} chars, max_tokens={max_tokens}) ==========")
    print(f"=== warmups ({WARMUPS}) ===")
    for i in range(WARMUPS):
        try:
            w, t, k, _ = run_once(prompt, max_tokens)
            _, _, _, line = fmt(f"warm-{i+1}", w, t, k)
            if not QUIET:
                print(line)
        except Exception as e:
            print(f"  warm-{i+1}  FAIL: {e}")
    print(f"\n=== measured ({RUNS}) ===")
    walls, decodes, ttfts = [], [], []
    for i in range(RUNS):
        try:
            w, t, k, _ = run_once(prompt, max_tokens)
            wtps, dtps, ttft, line = fmt(f"run-{i+1}", w, t, k)
            if not QUIET:
                print(line)
            walls.append(wtps); decodes.append(dtps); ttfts.append(ttft)
        except Exception as e:
            print(f"  run-{i+1}  FAIL: {e}")
    if walls:
        print(f"\n=== summary [{label}] (n={len(walls)}) ===")
        print(stats("wall_TPS",   walls))
        print(stats("decode_TPS", decodes))
        print(f"  TTFT          mean={s.mean(ttfts)*1000:6.0f}ms  std={s.stdev(ttfts)*1000 if len(ttfts) > 1 else 0:5.0f}ms  min={min(ttfts)*1000:.0f}ms  max={max(ttfts)*1000:.0f}ms")
        if PP_MODE == "log":
            pp_vals = scrape_prompt_throughput(CONTAINER, len(walls))
            if pp_vals:
                print(stats("PP tok/s", pp_vals))
            else:
                print("  PP tok/s       n/a (vLLM log scrape unavailable; use PP=1 for long-prompt fallback)")
        else:
            print("  PP tok/s       n/a (long-prompt fallback below)")

def long_prompt(target_tokens):
    filler = (
        "club3090 prompt processing calibration filler with stable token shape. "
        "This sentence is intentionally plain so tokenizer variance stays modest. "
    )
    words_per_chunk = max(len(filler.split()), 1)
    chunks = max(1, target_tokens // words_per_chunk)
    return (
        "Read the following calibration text. Reply with one concise sentence summarizing its purpose.\n\n"
        + filler * chunks
    )

def run_pp_fallback():
    prompt = long_prompt(PP_FALLBACK_TOKENS)
    print(
        f"\n========== PROMPT-PROCESSING "
        f"(fallback target={PP_FALLBACK_TOKENS} prompt tokens, max_tokens={PP_MAX_TOKENS}) =========="
    )
    print("=== measured (1) ===")
    pp_vals, ttfts = [], []
    try:
        w, t, _k, prompt_tokens = run_once(prompt, PP_MAX_TOKENS)
        pp, ttft, line = fmt_pp("run-1", w, t, prompt_tokens)
        print(line)
        pp_vals.append(pp); ttfts.append(ttft)
    except Exception as e:
        print(f"  run-1      FAIL: {e}")
    if pp_vals:
        print("\n=== summary [prompt-processing] (n=1) ===")
        print(stats("PP tok/s", pp_vals))
        print(f"  TTFT          mean={s.mean(ttfts)*1000:6.0f}ms  std=    0ms  min={min(ttfts)*1000:.0f}ms  max={max(ttfts)*1000:.0f}ms")

def prefill_prompt(target_tokens, salt):
    """Cache-busted long prompt (catalog-baselines 2c).

    Composes serve ``enable_prefix_caching=True`` — an identical prompt
    re-measures the PREFIX-CACHE HIT, not prefill.  vLLM's prefix cache is
    block-chained, so a unique FIRST line breaks the whole chain; the salt is
    also woven through the filler as belt-and-braces against block-aligned
    partial hits."""
    filler = (
        f"calibration {salt} segment with stable token shape for the prefill probe. "
        "This sentence is intentionally plain so tokenizer variance stays modest. "
    )
    words_per_chunk = max(len(filler.split()), 1)
    chunks = max(1, target_tokens // words_per_chunk)
    return (
        f"[probe {salt}] Read the following calibration text. Reply with the single word DONE.\n\n"
        + filler * chunks
    )

def run_prefill_probe():
    """Canonical two-depth prefill/TTFT probe (catalog-baselines 2c): warm
    engine, FRESH salted haystack per request, prefill tok/s = prompt_tokens/TTFT.
    The deep anchor pairs with the verify-stress ladder's nearest rung for
    anchor calibration (agreement certifies the ladder's whole depth curve).
    A depth the served context can't hold is SKIPPED with a note."""
    depths = [int(x) for x in os.environ.get("PREFILL_DEPTHS", "10000,90000").split(",") if x.strip()]
    n = max(1, int(os.environ.get("PREFILL_RUNS", "3")))

    def salt():
        return "".join(random.choices(string.ascii_lowercase, k=8))

    for target in depths:
        label = f"prefill-{target // 1000}k"
        print(
            f"\n========== {label.upper()} (target={target} prompt tokens, "
            f"max_tokens={PP_MAX_TOKENS}, cache-busted: fresh haystack per run) =========="
        )
        print("=== warmups (1) ===")
        # The warmup doubles as the TOKEN CALIBRATOR: the word-count heuristic
        # overshoots (salted filler tokenizes ~1.3 tok/word), so the measured
        # runs scale their request by the warmup's actual/target ratio — the
        # depth label stays honest to within a few percent.
        request_tokens = target
        try:
            w, t, _k, ptoks = run_once(prefill_prompt(target, salt()), PP_MAX_TOKENS)
            _pp, _t, line = fmt_pp("warm-1", w, t, ptoks)
            if not QUIET:
                print(line)
            if ptoks > 0:
                request_tokens = max(200, int(target * target / ptoks))
                print(f"  [calibrated: {ptoks} tok at request={target} → measured runs request {request_tokens}]")
        except Exception as e:
            msg = str(e)
            if any(x in msg for x in ("400", "maximum context", "max_model_len", "context length", "context window")):
                print(f"  SKIP: {target} tokens exceeds the served context ({msg[:100]})")
                continue
            print(f"  warm-1     FAIL: {e}")
        print(f"\n=== measured ({n}) ===")
        pps, ttfts = [], []
        for i in range(n):
            try:
                w, t, _k, ptoks = run_once(prefill_prompt(request_tokens, salt()), PP_MAX_TOKENS)
                pp, ttft, line = fmt_pp(f"run-{i+1}", w, t, ptoks)
                if not QUIET:
                    print(line)
                pps.append(pp)
                ttfts.append(ttft)
            except Exception as e:
                print(f"  run-{i+1}    FAIL: {e}")
        if pps:
            print(f"\n=== summary [{label}] (n={len(pps)}) ===")
            # prompt_tokens/TTFT = the CLIENT-OBSERVED rate (includes
            # tokenization + transfer + scheduling) — the user-truth number.
            print(stats("prefill tok/s", pps))
            print(
                f"  TTFT          mean={s.mean(ttfts)*1000:6.0f}ms  "
                f"std={(s.stdev(ttfts)*1000 if len(ttfts) > 1 else 0):5.0f}ms  "
                f"min={min(ttfts)*1000:.0f}ms  max={max(ttfts)*1000:.0f}ms"
            )
            # Engine-internal rate from the vLLM stats log — windowed (tokens
            # per 10s stats window), so treat as approximate; the gap between
            # this and the client-observed rate is tokenization/transfer/
            # scheduling overhead, not prefill compute.
            if PP_MODE == "log":
                vals = scrape_prompt_throughput(CONTAINER, len(pps) * 2)
                vals = [v for v in vals if v > 0][-len(pps):]
                if vals:
                    print(stats("PP tok/s (engine log, windowed)", vals))

if ONLY in ("both", "narr"):
    run_set("narrative", PROMPT_NARR, MAX_NARR)
if ONLY in ("both", "code"):
    run_set("code", PROMPT_CODE, MAX_CODE)
if PP_MODE == "fallback":
    run_pp_fallback()
if os.environ.get("PREFILL_PROBE", "1") == "1":
    run_prefill_probe()
PYEOF

# GPU state
if command -v nvidia-smi >/dev/null 2>&1; then
  echo ""
  echo "=== GPU state ==="
  nvidia-smi --query-gpu=index,utilization.gpu,memory.used,memory.total,power.draw,temperature.gpu \
             --format=csv,noheader
fi

# MTP / spec-decode stats — only when running against a Docker container we own.
# Skipped silently in endpoint-first mode (CONTAINER=none).
if [[ "${CONTAINER:-}" != "none" ]] && command -v docker >/dev/null 2>&1 \
   && docker inspect "${CONTAINER}" >/dev/null 2>&1; then
  echo ""
  echo "=== Last 3 SpecDecoding metrics ==="
  docker logs "${CONTAINER}" 2>&1 | grep "SpecDecoding metrics" | tail -3 || true
fi