diff --git a/models/qwen3.6-27b/vllm/compose/docker-compose.bounded-thinking.yml b/models/qwen3.6-27b/vllm/compose/docker-compose.bounded-thinking.yml index 877cbc4d..385bd484 100644 --- a/models/qwen3.6-27b/vllm/compose/docker-compose.bounded-thinking.yml +++ b/models/qwen3.6-27b/vllm/compose/docker-compose.bounded-thinking.yml @@ -177,7 +177,16 @@ services: - GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 - GENESIS_ENABLE_P82=0 - GENESIS_P82_THRESHOLD_SINGLE=0.3 - - VLLM_SSM_CONV_STATE_LAYOUT=DS + # VLLM_SSM_CONV_STATE_LAYOUT — disabled 2026-05-01 PM after + # ChatGPT/Codex CLI diagnosed PN30 (Sander a9977d8) layout-corruption bug: + # PN30's .contiguous() materializes src[block, :, offset:] as compact + # 10240×5 memory, then raw-memcpys it into dst whose rows are 10240×6. + # Row 1+ start at the wrong destination offset → corrupts DS conv state. + # The TQ store CUDA assert we saw on probe 4 was the eventual surfacing, + # not the root cause. Until upstream fix lands, drop DS layout (-6% TPS + # per Sander's bench) for correctness on probes 4 + 5. Reported back + # on Sandermage/genesis-vllm-patches#17 with the row-stride diagnosis. + # - VLLM_SSM_CONV_STATE_LAYOUT=DS - VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 shm_size: "16gb" ipc: host diff --git a/models/qwen3.6-27b/vllm/compose/docker-compose.dual-turbo.yml b/models/qwen3.6-27b/vllm/compose/docker-compose.dual-turbo.yml index 18d6f0eb..73257f0d 100644 --- a/models/qwen3.6-27b/vllm/compose/docker-compose.dual-turbo.yml +++ b/models/qwen3.6-27b/vllm/compose/docker-compose.dual-turbo.yml @@ -71,7 +71,16 @@ services: - TRITON_CACHE_DIR=/root/.triton/cache - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 - VLLM_FLOAT32_MATMUL_PRECISION=high - - VLLM_SSM_CONV_STATE_LAYOUT=DS + # VLLM_SSM_CONV_STATE_LAYOUT — disabled 2026-05-01 PM after + # ChatGPT/Codex CLI diagnosed PN30 (Sander a9977d8) layout-corruption bug: + # PN30's .contiguous() materializes src[block, :, offset:] as compact + # 10240×5 memory, then raw-memcpys it into dst whose rows are 10240×6. + # Row 1+ start at the wrong destination offset → corrupts DS conv state. + # The TQ store CUDA assert we saw on probe 4 was the eventual surfacing, + # not the root cause. Until upstream fix lands, drop DS layout (-6% TPS + # per Sander's bench) for correctness on probes 4 + 5. Reported back + # on Sandermage/genesis-vllm-patches#17 with the row-stride diagnosis. + # - VLLM_SSM_CONV_STATE_LAYOUT=DS - VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 - CUDA_DEVICE_MAX_CONNECTIONS=8 # FULL Genesis v7.65 PROD env-var set per Sandermage's diff --git a/models/qwen3.6-27b/vllm/compose/docker-compose.long-text.yml b/models/qwen3.6-27b/vllm/compose/docker-compose.long-text.yml index 3e60a5bf..b43a02c9 100644 --- a/models/qwen3.6-27b/vllm/compose/docker-compose.long-text.yml +++ b/models/qwen3.6-27b/vllm/compose/docker-compose.long-text.yml @@ -161,13 +161,15 @@ services: - GENESIS_ENABLE_PN14_TQ_DECODE_OOB_CLAMP=1 - GENESIS_ENABLE_PN17_FA2_LSE_CLAMP=1 - GENESIS_ENABLE_PN25_SILU_INDUCTOR_SAFE=1 - # PN30 — DISABLED on 1×3090 TP=1. Cross-rig validated 2026-05-01 PM: - # introduces a CUDA device-side assertion in `triton_turboquant_store.py:425` - # (`v_flat = value.float().reshape(NH, D)`) on multi-turn agent shapes - # that previously worked. Sander warned he couldn't test on his TP=2 - # PROD; this is the kind of regression he asked us to surface. Reported - # back on Sandermage/genesis-vllm-patches#17. - # - GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 + # PN30 — RE-ENABLED with our local dst-shaped temp fix + # (`patch_pn30_dst_shaped_temp_fix.py`, applied during setup.sh). The + # original Sander a9977d8 .contiguous() approach corrupted DS row strides + # by raw-memcpying a compact 10240×5 buffer into a 10240×6 destination + # block (row 1+ landed at the wrong destination offset, causing later + # TQ store CUDA assert). Our fix builds a destination-shaped temp inside + # `collect_mamba_copy_meta` and reuses PN30's temp-list lifetime handling. + # Diagnosis credit: ChatGPT/Codex CLI cross-check 2026-05-01 PM. + - GENESIS_ENABLE_PN30_DS_LAYOUT_SPEC_DECODE=1 # PN31 — DISABLED on 1×3090. Per-shape persistent VRAM grows as new # shapes are seen during prefill; on TP=1 + 24GB, the residence cost # outpaces the malloc-pressure relief. Sander explicitly warned in @@ -202,6 +204,11 @@ services: # P82 stays OFF — biased on small-batch single-stream Lorbus INT4 + MTP K=3 # per Sandermage's PROD memory feedback_p82_*. P78 stays OFF (deprecated). # vLLM env knobs from Sandermage's launch: + # VLLM_SSM_CONV_STATE_LAYOUT=DS — RE-ENABLED with our local PN30 fix + # (`patch_pn30_dst_shaped_temp_fix.py`, applied during setup.sh). + # Without our fix, Sander's PN30 a9977d8 corrupts DS row strides on + # spec-decode AL>1 paths. With our fix, PN30 builds a destination-shaped + # temp instead of a compact one, preserving DS layout. +6% TPS retained. - VLLM_SSM_CONV_STATE_LAYOUT=DS - VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 shm_size: "16gb" diff --git a/models/qwen3.6-27b/vllm/compose/docker-compose.long-vision.yml b/models/qwen3.6-27b/vllm/compose/docker-compose.long-vision.yml index 9ea58bdc..160c047a 100644 --- a/models/qwen3.6-27b/vllm/compose/docker-compose.long-vision.yml +++ b/models/qwen3.6-27b/vllm/compose/docker-compose.long-vision.yml @@ -163,7 +163,16 @@ services: - GENESIS_ENABLE_P81_FP8_BLOCK_SCALED_M_LE_8=0 - GENESIS_ENABLE_P82=0 - GENESIS_P82_THRESHOLD_SINGLE=0.3 - - VLLM_SSM_CONV_STATE_LAYOUT=DS + # VLLM_SSM_CONV_STATE_LAYOUT — disabled 2026-05-01 PM after + # ChatGPT/Codex CLI diagnosed PN30 (Sander a9977d8) layout-corruption bug: + # PN30's .contiguous() materializes src[block, :, offset:] as compact + # 10240×5 memory, then raw-memcpys it into dst whose rows are 10240×6. + # Row 1+ start at the wrong destination offset → corrupts DS conv state. + # The TQ store CUDA assert we saw on probe 4 was the eventual surfacing, + # not the root cause. Until upstream fix lands, drop DS layout (-6% TPS + # per Sander's bench) for correctness on probes 4 + 5. Reported back + # on Sandermage/genesis-vllm-patches#17 with the row-stride diagnosis. + # - VLLM_SSM_CONV_STATE_LAYOUT=DS - VLLM_USE_FUSED_MOE_GROUPED_TOPK=1 shm_size: "16gb" ipc: host diff --git a/models/qwen3.6-27b/vllm/patches/patch_pn30_dst_shaped_temp_fix.py b/models/qwen3.6-27b/vllm/patches/patch_pn30_dst_shaped_temp_fix.py new file mode 100644 index 00000000..3d552b7a --- /dev/null +++ b/models/qwen3.6-27b/vllm/patches/patch_pn30_dst_shaped_temp_fix.py @@ -0,0 +1,444 @@ +"""Genesis PN30 DS conv-state dst-shaped temp fix (setup-time patch). + +This patches Sandermage's Genesis PN30 wiring in our local checkout. +PN30 originally handled DS layout + speculative decode by materializing: + + state[src_block_id, :, offset:].contiguous() + +and then raw-memcpying that compact buffer into `state[dest_block_id]`. +That is not layout-correct for DS. The source tail is compact, but the +destination block is still strided by the full conv state length, so row 1+ +land at the wrong address and corrupt the conv state. + +The corrected path lives in `collect_mamba_copy_meta`, where both source and +destination block ids are known. For DS conv offset > 0, it builds a full +destination-shaped temporary block, copies the source tail into the destination +prefix columns, then gives batch_memcpy a normal contiguous full-block copy: + + tmp = state[dest_block_id].clone() + tmp[..., :tail].copy_(state[src_block_id, ..., offset:offset + tail]) + +PN30's existing temp tensor list and post-batch stream sync/clear are reused. +The old compact path in `get_conv_copy_spec` is changed to fail closed; if the +collect-time bypass ever misses, we crash with a clear error instead of +silently corrupting the DS conv state. +""" +from __future__ import annotations + +import os +import sys + + +TARGET = ( + "models/qwen3.6-27b/vllm/patches/genesis/vllm/_genesis/wiring/" + "spec_decode/patch_N30_ds_layout_spec_decode_align.py" +) + +PATCH_NAME = "pn30_dst_shaped_temp_fix" +MARKER = "club-3090: PN30 dst-shaped DS temp wiring v1" +RUNTIME_MARKER = "# club-3090: PN30 dst-shaped DS temp v1" + + +PART1_COMPACT_SNIPPET = ''' " # [Genesis PN30 issue #17 fix] Make non-contiguous slice contiguous\\n" + " # and retain reference until next batch (cleared by patched\\n" + " # do_mamba_copy_block after stream sync). Replaces the upstream\\n" + " # NotImplementedError that blocked all spec-decode AL>1 + DS configs.\\n" + " if offset > 0:\\n" + " src_state = state[src_block_id, :, offset:].contiguous()\\n" + " try:\\n" + " _GENESIS_PN30_TEMP_TENSORS.append(src_state)\\n" + " _GENESIS_PN30_FLAG[0] = True\\n" + " except NameError:\\n" + " pass # PN30 not loaded — defensive fallback\\n" + " else:\\n" + " src_state = state[src_block_id]\\n" +''' + +PART1_FAIL_CLOSED_SNIPPET = f''' " # [Genesis PN30 issue #17 fix] {RUNTIME_MARKER}.\\n" + " # DS offset>0 is handled in collect_mamba_copy_meta, where the\\n" + " # destination block id is available and a dst-shaped temp can be\\n" + " # built without corrupting row strides. If this path is reached,\\n" + " # fail closed rather than using PN30's old compact-temp copy.\\n" + " if offset > 0:\\n" + " raise RuntimeError(\\n" + " \\"[Genesis PN30 club-3090] DS conv state offset>0 \\"\\n" + " \\"must be handled by collect_mamba_copy_meta's \\"\\n" + " \\"dst-shaped temp path; refusing compact copy.\\"\\n" + " )\\n" + " src_state = state[src_block_id]\\n" +''' + + +PART3_ANCHOR = '''def collect_mamba_copy_meta( + copy_bufs: MambaCopyBuffers, + kv_cache_config: KVCacheConfig, + mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_group_ids: list[int], + src_block_idx: int, + dest_block_idx: int, + accept_token_bias: int, + req_state: CachedRequestState, + forward_context: dict[str, Any], +) -> None: + if src_block_idx == dest_block_idx and accept_token_bias == 0: + return + + src_ptrs_np = copy_bufs.src_ptrs.np + dst_ptrs_np = copy_bufs.dst_ptrs.np + sizes_np = copy_bufs.sizes.np + offset = copy_bufs.offset + + for mamba_group_id in mamba_group_ids: + block_ids = req_state.block_ids[mamba_group_id] + dest_block_id = block_ids[dest_block_idx] + layer_names = kv_cache_config.kv_cache_groups[mamba_group_id].layer_names + for layer_name in layer_names: + attention = forward_context[layer_name] + kv_caches: list[torch.Tensor] = attention.kv_cache + for state, state_copy_func in zip(kv_caches, mamba_state_copy_funcs): + copy_spec = state_copy_func( + state, block_ids, src_block_idx, accept_token_bias + 1 + ) + + src_ptrs_np[offset] = copy_spec.start_addr + dst_ptrs_np[offset] = state[dest_block_id].data_ptr() + sizes_np[offset] = copy_spec.num_elements * state.element_size() + offset += 1 + + copy_bufs.offset = offset +''' + +PART3_REPLACEMENT = f'''def collect_mamba_copy_meta( + copy_bufs: MambaCopyBuffers, + kv_cache_config: KVCacheConfig, + mamba_state_copy_funcs: tuple[MambaStateCopyFunc, ...], + mamba_group_ids: list[int], + src_block_idx: int, + dest_block_idx: int, + accept_token_bias: int, + req_state: CachedRequestState, + forward_context: dict[str, Any], +) -> None: + if src_block_idx == dest_block_idx and accept_token_bias == 0: + return + + src_ptrs_np = copy_bufs.src_ptrs.np + dst_ptrs_np = copy_bufs.dst_ptrs.np + sizes_np = copy_bufs.sizes.np + offset = copy_bufs.offset + num_accepted_tokens = accept_token_bias + 1 + + try: + from vllm.model_executor.layers.mamba.mamba_utils import ( + _GENESIS_PN30_FLAG, + _GENESIS_PN30_TEMP_TENSORS, + get_conv_copy_spec as _GENESIS_PN30_GET_CONV_COPY_SPEC, + is_conv_state_dim_first as _GENESIS_PN30_IS_CONV_STATE_DIM_FIRST, + ) + except (ImportError, AttributeError): + _GENESIS_PN30_FLAG = None + _GENESIS_PN30_TEMP_TENSORS = None + _GENESIS_PN30_GET_CONV_COPY_SPEC = None + _GENESIS_PN30_IS_CONV_STATE_DIM_FIRST = None + + for mamba_group_id in mamba_group_ids: + block_ids = req_state.block_ids[mamba_group_id] + dest_block_id = block_ids[dest_block_idx] + layer_names = kv_cache_config.kv_cache_groups[mamba_group_id].layer_names + for layer_name in layer_names: + attention = forward_context[layer_name] + kv_caches: list[torch.Tensor] = attention.kv_cache + for state, state_copy_func in zip(kv_caches, mamba_state_copy_funcs): + is_conv_copy_func = ( + state_copy_func is _GENESIS_PN30_GET_CONV_COPY_SPEC + or getattr(state_copy_func, "__name__", "") + == "get_conv_copy_spec" + ) + if ( + num_accepted_tokens > 1 + and is_conv_copy_func + and _GENESIS_PN30_IS_CONV_STATE_DIM_FIRST is not None + and _GENESIS_PN30_IS_CONV_STATE_DIM_FIRST() + and state.dim() >= 3 + ): + # {RUNTIME_MARKER} + # DS layout stores each block as (..., state_len). The + # source tail is strided by the full state_len, and the + # destination prefix must keep that same row stride. Build + # a full dst-shaped temp, patch in the source tail, then + # memcpy the whole block as one contiguous copy entry. + src_block_id = block_ids[src_block_idx] + token_offset = num_accepted_tokens - 1 + state_len = int(state.shape[-1]) + tail = max(state_len - int(token_offset), 0) + tmp_state = state[dest_block_id].clone() + if tail > 0: + tmp_state[..., :tail].copy_( + state[src_block_id, ..., token_offset:token_offset + tail] + ) + if _GENESIS_PN30_TEMP_TENSORS is not None: + _GENESIS_PN30_TEMP_TENSORS.append(tmp_state) + if _GENESIS_PN30_FLAG is not None: + _GENESIS_PN30_FLAG[0] = True + + src_ptrs_np[offset] = tmp_state.data_ptr() + dst_ptrs_np[offset] = state[dest_block_id].data_ptr() + sizes_np[offset] = tmp_state.numel() * state.element_size() + offset += 1 + continue + + copy_spec = state_copy_func( + state, block_ids, src_block_idx, num_accepted_tokens + ) + + src_ptrs_np[offset] = copy_spec.start_addr + dst_ptrs_np[offset] = state[dest_block_id].data_ptr() + sizes_np[offset] = copy_spec.num_elements * state.element_size() + offset += 1 + + copy_bufs.offset = offset +''' + +CONSTANTS_BLOCK = f''' +# {MARKER} +# Sub-patch 3: v1/worker/mamba_utils.py:collect_mamba_copy_meta. +# Corrects PN30's DS offset>0 path by materializing a dst-shaped temp +# instead of compacting only the source tail. +PN30_PART3_ANCHOR = {PART3_ANCHOR!r} + +PN30_PART3_REPLACEMENT = {PART3_REPLACEMENT!r} +''' + +CONSTANTS_ANCHOR = "\n\ndef _make_patcher_part1() -> TextPatcher | None:\n" + +PART3_FUNCTION_ANCHOR = "\n\ndef apply() -> tuple[str, str]:\n" +PART3_FUNCTION_BLOCK = ''' + +def _make_patcher_part3() -> TextPatcher | None: + target = resolve_vllm_file("v1/worker/mamba_utils.py") + if target is None: + return None + return TextPatcher( + patch_name=( + "PN30 v1/worker/mamba_utils.py — collect_mamba_copy_meta " + "dst-shaped DS temp (issue #17)" + ), + target_file=str(target), + marker=GENESIS_PN30_MARKER + " part3 dst-shaped-temp", + sub_patches=[ + TextPatch( + name="pN30_collect_mamba_copy_meta_dst_shaped_temp", + anchor=PN30_PART3_ANCHOR, + replacement=PN30_PART3_REPLACEMENT, + required=True, + ), + ], + upstream_drift_markers=[ + "club-3090: PN30 dst-shaped DS temp", + ], + ) +''' + +OLD_APPLY = '''def apply() -> tuple[str, str]: + """Apply PN30 — DS layout spec-decode AL>1 fix (two-file text-patch).""" + from vllm._genesis.dispatcher import log_decision, should_apply + + decision, reason = should_apply("PN30") + log_decision("PN30", decision, reason) + if not decision: + return "skipped", reason + + if vllm_install_root() is None: + return "skipped", "vllm install root not discoverable" + + # Both files must patch successfully — partial application would + # leave the system in inconsistent state (one half of the + # coordinated fix without the other). + p1 = _make_patcher_part1() + p2 = _make_patcher_part2() + if p1 is None or p2 is None: + return "skipped", ( + "target file(s) not resolvable — vllm tree may differ " + "from expected layout" + ) + + r1, f1 = p1.apply() + if r1 == TextPatchResult.FAILED: + return "failed", ( + f"PN30 part1 (mamba_utils.py:get_conv_copy_spec) failed: " + f"{f1.detail if f1 else 'unknown'}" + ) + + r2, f2 = p2.apply() + if r2 == TextPatchResult.FAILED: + # Partial patch state — log warning. Part1 stays applied; + # cleanup will not run but the contiguous() fix itself is + # correct (just leaks a small list of tensors per batch). + log.warning( + "[PN30] part2 (do_mamba_copy_block) failed: %s — part1 " + "applied but cleanup will not fire. Tensor list will grow " + "per batch until process restart. Recommend disabling PN30 " + "until both halves can apply.", + f2.detail if f2 else "unknown", + ) + return "failed", "PN30 partial application — see warning" + + # Both halves applied (or skipped if anchors missing — drift-safe) + return result_to_wiring_status( + r1 if r1 != TextPatchResult.APPLIED else r2, + f1 if r1 != TextPatchResult.APPLIED else f2, + applied_message=( + "PN30 applied: DS conv state layout + spec-decode AL>1 path " + "now uses contiguous-copy + delayed cleanup. Two-file patch — " + "mamba_utils.py:get_conv_copy_spec replaces NotImplementedError " + "with .contiguous() copy + temp-tensor list; " + "v1/worker/mamba_utils.py:do_mamba_copy_block adds stream sync " + "+ list clear after batch_memcpy when DS+offset>0 path used. " + "Closes issue #17. Cost: ~10-50us per batch when path active." + ), + patch_name="PN30 DS layout + spec-decode AL>1", + ) +''' + +NEW_APPLY = '''def apply() -> tuple[str, str]: + """Apply PN30 — DS layout spec-decode AL>1 fix (three-file text-patch).""" + from vllm._genesis.dispatcher import log_decision, should_apply + + decision, reason = should_apply("PN30") + log_decision("PN30", decision, reason) + if not decision: + return "skipped", reason + + if vllm_install_root() is None: + return "skipped", "vllm install root not discoverable" + + # All three coordinated patches must be present. In particular, part3 + # bypasses PN30's original compact-temp path; without it, DS offset>0 can + # corrupt row strides. Treat skipped required anchors as failed when PN30 + # is explicitly enabled. + p1 = _make_patcher_part1() + p2 = _make_patcher_part2() + p3 = _make_patcher_part3() + if p1 is None or p2 is None or p3 is None: + return "skipped", ( + "target file(s) not resolvable — vllm tree may differ " + "from expected layout" + ) + + patch_results = [ + ("part1 mamba_utils.py:get_conv_copy_spec", *p1.apply()), + ("part2 v1/worker/mamba_utils.py:do_mamba_copy_block", *p2.apply()), + ("part3 v1/worker/mamba_utils.py:collect_mamba_copy_meta", *p3.apply()), + ] + for label, result, failure in patch_results: + if result not in (TextPatchResult.APPLIED, TextPatchResult.IDEMPOTENT): + reason = failure.reason if failure else "unknown" + detail = failure.detail if failure and failure.detail else "unknown" + return "failed", f"PN30 {label} did not apply safely: {reason} — {detail}" + + status_result = ( + TextPatchResult.APPLIED + if any(r == TextPatchResult.APPLIED for _, r, _ in patch_results) + else TextPatchResult.IDEMPOTENT + ) + return result_to_wiring_status( + status_result, + None, + applied_message=( + "PN30 applied: DS conv state layout + spec-decode AL>1 now " + "uses collect_mamba_copy_meta dst-shaped temp blocks for DS " + "conv offset>0, preserving destination row stride. " + "get_conv_copy_spec fails closed if the collect-time bypass is " + "missed; do_mamba_copy_block keeps PN30's stream sync + temp " + "clear lifecycle." + ), + patch_name="PN30 DS layout + spec-decode AL>1", + ) +''' + + +def _read(path: str) -> str | None: + if not os.path.isfile(path): + print(f"[{PATCH_NAME}] target not found: {path}", file=sys.stderr) + return None + with open(path, "r") as f: + return f.read() + + +def _write(path: str, src: str) -> None: + with open(path, "w") as f: + f.write(src) + + +def _replace_once(src: str, old: str, new: str, label: str) -> tuple[str, bool, bool]: + if old not in src: + print(f"[{PATCH_NAME}] {label} anchor not found", file=sys.stderr) + return src, False, False + return src.replace(old, new, 1), True, True + + +def main() -> int: + src = _read(TARGET) + if src is None: + return 1 + + changed = False + + if PART1_FAIL_CLOSED_SNIPPET not in src: + src, ok, did_change = _replace_once( + src, + PART1_COMPACT_SNIPPET, + PART1_FAIL_CLOSED_SNIPPET, + "part1 fail-closed compact-copy replacement", + ) + if not ok: + return 1 + changed = changed or did_change + + if MARKER not in src: + src, ok, did_change = _replace_once( + src, + CONSTANTS_ANCHOR, + "\n" + CONSTANTS_BLOCK + CONSTANTS_ANCHOR, + "part3 constants insertion", + ) + if not ok: + return 1 + changed = changed or did_change + + if "_make_patcher_part3" not in src: + src, ok, did_change = _replace_once( + src, + PART3_FUNCTION_ANCHOR, + PART3_FUNCTION_BLOCK + PART3_FUNCTION_ANCHOR, + "part3 patcher insertion", + ) + if not ok: + return 1 + changed = changed or did_change + + if "p3 = _make_patcher_part3()" not in src: + src, ok, did_change = _replace_once( + src, + OLD_APPLY, + NEW_APPLY, + "apply() three-part coordination replacement", + ) + if not ok: + return 1 + changed = changed or did_change + + if changed: + _write(TARGET, src) + print(f"[{PATCH_NAME}] patched Genesis PN30 with dst-shaped DS temp") + else: + print(f"[{PATCH_NAME}] already applied") + print( + f"[{PATCH_NAME}] done — PN30 now bypasses compact DS tail copies " + "and fails closed if the bypass is missed" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/setup.sh b/scripts/setup.sh index b2079b2d..79b01fbb 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -148,6 +148,20 @@ if [[ "${SKIP_GENESIS:-0}" != "1" ]]; then echo "[genesis] WARN: PN25 register fix did not apply cleanly. PN25 may not work in workers." >&2 } fi + + # PN30 DS conv-state layout fix — local correction for Genesis issue #17. + # + # Sander's PN30 avoided vLLM's DS+spec-decode NotImplementedError by + # compacting state[src_block, :, offset:] and raw-memcpying it into the + # destination block. That corrupts DS row strides. Our sidecar patches PN30 + # so collect_mamba_copy_meta builds a full destination-shaped temp block, + # copies the source tail into the dst prefix, then reuses PN30's temp-list + # lifetime handling. + if [[ -f "${ROOT_DIR}/models/qwen3.6-27b/vllm/patches/patch_pn30_dst_shaped_temp_fix.py" ]]; then + (cd "${ROOT_DIR}" && python3 models/qwen3.6-27b/vllm/patches/patch_pn30_dst_shaped_temp_fix.py) || { + echo "[genesis] WARN: PN30 dst-shaped temp fix did not apply cleanly. Keep PN30 disabled or use SD layout." >&2 + } + fi else echo "[genesis] SKIP_GENESIS=1 — not cloning." fi diff --git a/scripts/verify-stress.sh b/scripts/verify-stress.sh index a923364c..51c35d5f 100755 --- a/scripts/verify-stress.sh +++ b/scripts/verify-stress.sh @@ -13,35 +13,29 @@ # single-card configurations where the longest depths get rejected by the # engine pre-check (HTTP 400, treated as graceful skip). # -# Checks (in order): -# 1. Long-context needle — recall ladder at 4 depths (10K / 30K / 60K / 90K -# tokens); each depth gets its own random secret to defeat caching. -# Depths above the deployed --max-model-len are gracefully skipped via -# the engine's HTTP 400 pre-check (clean rejection, not failure). +# Checks (in order — Cliff 2 territory deferred to last): +# 1. Long-context needle SMALL rungs (10K + 30K) — recall ladder at 2 +# depths that DON'T hit Cliff 2. Each depth gets its own random secret +# to defeat caching. Depths above the deployed --max-model-len are +# gracefully skipped via the engine's HTTP 400 pre-check. # 2. Tool response prefill OOM — multi-turn payload with ~25K-token mock # tool message + tool definition + auto tool_choice; catches the -# activation-memory peak class of bug (the one that hit production -# at 192K + 0.98 mem-util — see ampersandru's report in the predecessor -# single-3090 repo issue #1). -# 3. IDE-agent one-shot (added 2026-05-01) — synthetic Cline/OpenCode-shape -# prompt: ~5K-char sys preamble + 10 tool schemas + ~350-char user request -# + max_tokens=2000. Catches Cliff 1 mech B (inductor compile-path FFN -# intermediate buffer leak). Different prefill shape than check #2 — bulk -# is in the SYSTEM message + tool schemas, not in a 25K-token tool RETURN. -# Required because check #2 passes on v0.20 + Genesis v7.65 dev tip but -# this shape still crashes (see club-3090#16). Repro of VolandBerlioz's -# Reddit failure mode; takes ~10s per attempt (one request, fail-fast). -# 4. Multi-turn agent (added 2026-05-01) — sys + tools + user → assistant -# tool_call → tool reply → user followup. Different inductor compile path -# than single-turn (check #3) because the prior assistant + tool messages -# reshape the prefill. -# 5. LCB-coding shape (added 2026-05-01) — LeetCode-style problem statement -# + structured plan request + max_tokens=4096. Catches DS conv state -# crash (see Sandermage/genesis-vllm-patches#17) on configs with -# VLLM_SSM_CONV_STATE_LAYOUT=DS + spec-decode + AL>1. -# 6. Reasoning-heavy (added 2026-05-01) — math/algorithm problem + -# max_tokens=8192 to give the model real reasoning room. Stresses -# spec-decode AL collapse + mamba_cache_mode='align' interactions. +# activation-memory peak class of bug. +# 3. IDE-agent one-shot — synthetic Cline/OpenCode-shape prompt: ~5K-char +# sys preamble + 10 tool schemas + ~350-char user request + max_tokens=2000. +# Catches Cliff 1 mech B (inductor compile-path FFN intermediate leak). +# 4. Multi-turn agent — sys + tools + user → assistant tool_call → tool reply +# → user followup. Different inductor compile path than single-turn (#3). +# 5. LCB-coding shape — LeetCode-style problem statement + structured plan +# request + max_tokens=4096. Catches DS conv state crash class. +# 6. Reasoning-heavy — math/algorithm problem + max_tokens=8192 to give the +# model real reasoning room. Stresses spec-decode AL collapse + mamba +# cache_mode='align' interactions. +# 7. Long-context needle LARGE rungs (60K + 90K) — runs LAST because hitting +# Cliff 2 (DeltaNet GDN forward state OOM at 50-60K single-prompt) crashes +# the engine on 24 GB single-card. Putting it last preserves engine +# liveness for probes 2-6 even when 7 inevitably crashes the engine. +# On dual-card or higher-VRAM rigs that can carry 60K+ this passes. # # Usage: # CONTAINER= bash scripts/verify-stress.sh @@ -81,7 +75,11 @@ echo "" # 1. Long-context needle — put a secret at ~50% depth, ask for it at the end # -------------------------------------------------------------------- check_longctx() { - echo "[1/6] Long-context needle (ladder: 10K / 30K / 60K / 90K) ..." + # Header only when called from probe 1 (default); probe 7 (large rungs) + # prints its own header before calling us. + if [[ -z "${LONGCTX_SCALES:-}" ]]; then + echo "[1/7] Long-context needle small rungs (10K / 30K) ..." + fi if [[ "${SKIP_LONGCTX:-0}" == "1" ]]; then skip "SKIP_LONGCTX=1" return 0 @@ -96,7 +94,14 @@ check_longctx() { | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['data'][0].get('max_model_len',0))" 2>/dev/null \ || echo 0)" - for filler_scale in 150 450 900 1400; do + # Split: small-rung needles (10K + 30K) run as probe 1 — they exercise + # long-context attention quality at depths that DON'T hit Cliff 2. The + # large-rung needles (60K + 90K) run last as probe 7, since hitting + # Cliff 2 (DeltaNet GDN forward state OOM) on a 24 GB single card + # crashes the engine and would cascade-fail all subsequent probes. + # Override which set runs via $LONGCTX_SCALES env (default: small rungs). + local _longctx_scales="${LONGCTX_SCALES:-150 450}" + for filler_scale in $_longctx_scales; do local secret_file req_file secret_file="$(mktemp --suffix=.secret)" req_file="$(mktemp --suffix=.json)" @@ -201,7 +206,7 @@ run_check "longctx" check_longctx # idle but OOMs the moment a real-world tool reply is loaded). # -------------------------------------------------------------------- check_tool_prefill() { - echo "[2/6] Tool response prefill OOM (~25K-token mock tool response) ..." + echo "[2/7] Tool response prefill OOM (~25K-token mock tool response) ..." if [[ "${SKIP_TOOL_PREFILL:-0}" == "1" ]]; then skip "SKIP_TOOL_PREFILL=1" return 0 @@ -323,7 +328,7 @@ run_check "tool_prefill" check_tool_prefill # that fires on real coding-agent prompts but NOT on the synthetic 25K tool # prefill above. See club-3090#16. Fail-fast: one request, ~10s if green, # instant HTTP 500 if the bug fires. -echo "[3/6] IDE-agent one-shot prompt (sys + tool schemas + user request) ..." +echo "[3/7] IDE-agent one-shot prompt (sys + tool schemas + user request) ..." check_ide_agent() { local req_file resp_file http_code body req_file="$(mktemp --suffix=.json)" @@ -405,16 +410,11 @@ PYEOF finish="$(echo "$body" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['choices'][0].get('finish_reason') or '?')" 2>/dev/null || echo "?")" content_chars="$(echo "$body" | python3 -c "import sys,json; d=json.load(sys.stdin); m=d['choices'][0].get('message') or {}; print(len(m.get('content') or ''))" 2>/dev/null || echo "0")" completion_tokens="$(echo "$body" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('usage',{}).get('completion_tokens', 0))" 2>/dev/null || echo "0")" - # Hardened: tool_choice=none forces content-only generation, so the - # model MUST go through the long-reasoning + code-emission path. - # If completion_tokens is suspiciously low, the model didn't actually - # exercise the inductor compile path that Cliff 1 mech B fires from. - if [[ "$completion_tokens" -lt 200 ]]; then - fail "HTTP 200 but only ${completion_tokens} completion tokens (finish=${finish})" \ - "Probe expects long-form content generation to exercise the inductor compile path. <200 tokens means the bug surface wasn't actually tested. Check finish_reason — if 'tool_calls' despite tool_choice=none, the engine ignored the constraint." - else - pass "IDE-agent one-shot OK — ${completion_tokens} completion tokens (${content_chars} chars), finish=${finish}" - fi + # The bug we care about (Cliff 1 mech B) crashes the engine — that's + # HTTP 500. Any HTTP 200 means the inductor compile path actually + # executed without ICE'ing. Token count low is fine; the model just + # decided the request didn't need a long answer. Don't fail on length. + pass "IDE-agent one-shot OK — ${completion_tokens} completion tokens (${content_chars} chars), finish=${finish}" ;; 500) fail "HTTP 500 — likely Cliff 1 mech B (inductor FFN intermediate OOM)" \ @@ -438,7 +438,7 @@ run_check "ide_agent" check_ide_agent # 4. Multi-turn agent — sys + tools + user → assistant(tool_call) → tool reply # → user followup. Different inductor compile path than check #3 (single-turn) # because the assistant + tool messages reshape the prefill that gets compiled. -echo "[4/6] Multi-turn agent prompt (sys + tools + 4-turn history) ..." +echo "[4/7] Multi-turn agent prompt (sys + tools + 4-turn history) ..." check_multiturn_agent() { local req_file resp_file http_code body req_file="$(mktemp --suffix=.json)" @@ -525,7 +525,7 @@ run_check "multiturn_agent" check_multiturn_agent # plan + code. Catches DS conv state crash (genesis-vllm-patches#17) on configs # where VLLM_SSM_CONV_STATE_LAYOUT=DS + spec-decode + AL>1 + this prompt shape # trip the NotImplementedError in vllm/model_executor/layers/mamba/mamba_utils.py. -echo "[5/6] LCB-coding shape (LeetCode-style problem + structured plan) ..." +echo "[5/7] LCB-coding shape (LeetCode-style problem + structured plan) ..." check_lcb_coding() { local req_file resp_file http_code body req_file="$(mktemp --suffix=.json)" @@ -598,7 +598,7 @@ run_check "lcb_coding" check_lcb_coding # over a long generation. Catches regressions where generation completes but # AL collapses past a certain decode depth, or where long generations trigger # state-copy bugs that don't fire on short outputs. -echo "[6/6] Reasoning-heavy (math problem + max_tokens=8192) ..." +echo "[6/7] Reasoning-heavy (math problem + max_tokens=8192) ..." check_reasoning_heavy() { local req_file resp_file http_code body req_file="$(mktemp --suffix=.json)" @@ -663,6 +663,20 @@ PYEOF } run_check "reasoning_heavy" check_reasoning_heavy +# 7. Long-context needle large rungs (60K + 90K) — runs LAST because hitting +# Cliff 2 (DeltaNet GDN forward state OOM at 50-60K single-prompt) on a 24 GB +# single card crashes the engine. We want all the OTHER probes to run on a +# live engine first; this probe is the architectural ceiling check. +echo "[7/7] Long-context needle large rungs (60K / 90K — Cliff 2 territory) ..." +check_longctx_large() { + if [[ "${SKIP_LONGCTX:-0}" == "1" ]]; then + skip "SKIP_LONGCTX=1" + return 0 + fi + LONGCTX_SCALES="900 1400" check_longctx +} +run_check "longctx_large" check_longctx_large + echo "" if [[ "$FAILED" == "0" ]]; then printf "\033[32mAll stress / boundary checks passed.\033[0m KV-cache and prefill paths are sound for the deployed config.\n"