Ryan LudwigandCursor
f5d5dca09a
Dual composes inject PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,... for the PCIe (custom-all-reduce OFF) path. On NVLink / patched-PCIe-P2P rigs detect_nvlink.sh turns custom all-reduce ON, where expandable_segments backs allocations with a cuMemMap VA range that cudaIpcGetMemHandle rejects during graph-buffer registration -> custom_all_reduce.cuh:455 'invalid argument', worker dies at boot. The prior fix clobbered PYTORCH_CUDA_ALLOC_CONF to max_split_size_mb:512, discarding any other knob the user set (custom max_split_size_mb, garbage_collection_threshold, ...). Instead strip ONLY the expandable_segments token on the P2P path and keep the rest; fall back to max_split_size_mb:512 when nothing remains. The PCIe path keeps expandable_segments untouched. Strengthen test-detect-nvlink-alloc-conf.sh to assert the surgical contract (preserve neighbors, strip in any position, no dangling commas) and add the vllm#42609 row to docs/UPSTREAM.md (its "DP>1 AND TP>1" title under-scopes the bug; the same cuh:455 crash fires TP-only -- cf. vllm#42583 / vllm#45604). Validated on a 2x RTX 3090 NVLink rig (TP=2, NV4): before, boot crashes at custom_all_reduce.cuh:455 'invalid argument'; after, the dual stack boots clean (custom AR ON, expandable_segments stripped), /v1/models ready, serves a completion. Co-authored-by: Cursor <[email protected]>
88 lines
3.9 KiB
Bash
Executable File
88 lines
3.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# detect_nvlink.sh must DROP expandable_segments on the P2P / custom-all-reduce
|
|
# path (it crashes custom_all_reduce IPC graph-buffer registration — see
|
|
# docs/UPSTREAM.md → vllm#42609) while PRESERVING every other alloc-conf knob the
|
|
# compose / user set (max_split_size_mb, garbage_collection_threshold, ...). On
|
|
# the PCIe path (custom AR off) expandable_segments is kept untouched.
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
DETECT="${ROOT_DIR}/scripts/detect_nvlink.sh"
|
|
|
|
fail=0
|
|
|
|
# Resolve PYTORCH_CUDA_ALLOC_CONF as detect_nvlink.sh would leave it.
|
|
# $1 = NVLINK_MODE, $2 = pre-set PYTORCH_CUDA_ALLOC_CONF ("__UNSET__" to leave unset).
|
|
resolve() {
|
|
local mode="$1" pre="$2"
|
|
if [[ "$pre" == "__UNSET__" ]]; then
|
|
NVLINK_MODE="$mode" bash -c "unset PYTORCH_CUDA_ALLOC_CONF; source '$DETECT' >/dev/null; printf '%s' \"\${PYTORCH_CUDA_ALLOC_CONF:-}\""
|
|
else
|
|
PYTORCH_CUDA_ALLOC_CONF="$pre" NVLINK_MODE="$mode" bash -c "source '$DETECT' >/dev/null; printf '%s' \"\${PYTORCH_CUDA_ALLOC_CONF:-}\""
|
|
fi
|
|
}
|
|
|
|
# Exact-match assertion (catches dangling/duplicate commas, not just substrings).
|
|
assert_exact() {
|
|
local mode="$1" pre="$2" want="$3" got
|
|
got="$(resolve "$mode" "$pre")"
|
|
if [[ "$got" != "$want" ]]; then
|
|
echo "[detect-nvlink-alloc] FAIL mode=$mode pre='$pre' got='$got' want='$want'" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# Substring present + absent assertion (mode-parameterized robustness check).
|
|
assert_contains_absent() {
|
|
local mode="$1" pre="$2" present="$3" absent="$4" got
|
|
got="$(resolve "$mode" "$pre")"
|
|
if [[ "$got" != *"$present"* ]]; then
|
|
echo "[detect-nvlink-alloc] FAIL mode=$mode pre='$pre' got='$got' missing '$present'" >&2
|
|
fail=1
|
|
fi
|
|
if [[ "$got" == *"$absent"* ]]; then
|
|
echo "[detect-nvlink-alloc] FAIL mode=$mode pre='$pre' got='$got' must not contain '$absent'" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# --- P2P / custom-all-reduce paths (force_on = NVLink, pcie_p2p = patched driver) ---
|
|
|
|
# Compose default injection: strip expandable_segments, keep the rest.
|
|
assert_exact force_on "expandable_segments:True,max_split_size_mb:512" "max_split_size_mb:512"
|
|
|
|
# A user's non-default max_split_size_mb must SURVIVE (not silently reset to 512).
|
|
assert_exact pcie_p2p "expandable_segments:True,max_split_size_mb:256" "max_split_size_mb:256"
|
|
|
|
# expandable_segments mid-list: both neighbors preserved, no dangling/double commas.
|
|
assert_exact force_on \
|
|
"max_split_size_mb:512,expandable_segments:True,garbage_collection_threshold:0.9" \
|
|
"max_split_size_mb:512,garbage_collection_threshold:0.9"
|
|
|
|
# expandable_segments:False is also stripped (any value), neighbor preserved.
|
|
assert_exact force_on "expandable_segments:False,max_split_size_mb:512" "max_split_size_mb:512"
|
|
|
|
# Only expandable_segments set -> nothing left -> fall back to the NVLink default.
|
|
assert_exact force_on "expandable_segments:True" "max_split_size_mb:512"
|
|
|
|
# Unset entirely on a P2P rig -> the NVLink default, no expandable_segments.
|
|
assert_exact force_on "__UNSET__" "max_split_size_mb:512"
|
|
|
|
# Belt-and-suspenders: expandable_segments must never survive a P2P path.
|
|
assert_contains_absent force_on "expandable_segments:True,max_split_size_mb:256" "max_split_size_mb:256" "expandable_segments"
|
|
assert_contains_absent pcie_p2p "expandable_segments:True,max_split_size_mb:256" "max_split_size_mb:256" "expandable_segments"
|
|
|
|
# --- PCIe path (custom AR off) keeps expandable_segments untouched ---
|
|
|
|
# Compose default flows through unchanged.
|
|
assert_exact force_off "expandable_segments:True,max_split_size_mb:512" "expandable_segments:True,max_split_size_mb:512"
|
|
|
|
# Unset on PCIe -> the script's expandable_segments-on default.
|
|
assert_contains_absent force_off "__UNSET__" "expandable_segments" "max_split_size_mb:99999"
|
|
|
|
if [[ "$fail" -ne 0 ]]; then
|
|
echo "[detect-nvlink-alloc] FAILED" >&2
|
|
exit 1
|
|
fi
|
|
echo "[detect-nvlink-alloc] PASS: P2P strips ONLY expandable_segments (other knobs preserved); PCIe keeps it"
|