From 812d6626025f880829d75bdb7e5ffc60d13f6f63 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Thu, 1 Oct 2026 21:15:44 -0400 Subject: [PATCH] recoverssm: allow pipeline parallelism for Qwen4Exp on the V2 runner Upstream RecoverSSM commits a step in the same step's postprocess, hence PP=1. Under pipeline parallelism a non-last rank learns a step's accepted counts only when the PP broadcast arrives pp_size steps later, while later builds overwrite the persistent commit inputs. The commit is now deferred with its own copy of those inputs, and the PP=1 check is lifted for Qwen4Exp on Model Runner V2. Config and CPU tests cover the deferral. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/config/vllm.py | 22 +++++- vllm/v1/attention/backends/gdn_recoverssm.py | 34 +++++++++- vllm/v1/attention/backends/ple_recoverssm.py | 23 ++++++- .../attention/backends/recoverssm_metadata.py | 41 ++++++++++- vllm/v1/worker/gpu/cudagraph_utils.py | 4 ++ vllm/v1/worker/gpu/model_runner.py | 11 +++ vllm/v1/worker/gpu/model_states/interface.py | 13 ++++ .../worker/gpu/model_states/mamba_hybrid.py | 14 ++++ vllm/v1/worker/gpu/model_states/recoverssm.py | 68 +++++++++++++++++++ vllm/v1/worker/gpu/pp_utils.py | 31 ++++++++- 10 files changed, 254 insertions(+), 7 deletions(-) diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index ff7319dc15..afd6361bb9 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -3565,9 +3565,25 @@ class VllmConfig: "RecoverSSM with align mode requires VLLM_USE_V2_MODEL_RUNNER=1" ) if self.parallel_config.pipeline_parallel_size > 1: - raise ValueError( - "RecoverSSM currently requires pipeline_parallel_size=1" - ) + # Non-last PP ranks get the accepted counts pp_size steps after + # their verify. Model Runner V2 defers the commit with a + # snapshot of its inputs (RecoverSSMState.detach_step) and + # masks freed rows; only the Qwen4Exp GDN/PLE metadata + # implement that snapshot (Kimi-K3 KDA does not). + if ( + self.model_config is None + or self.model_config.architecture + not in _QWEN4EXP_RECOVERSSM_ARCHS + ): + raise ValueError( + "RecoverSSM with pipeline_parallel_size > 1 is supported " + "only for Qwen4Exp GDN/PLE" + ) + if not self.use_v2_model_runner: + raise ValueError( + "RecoverSSM with pipeline_parallel_size > 1 requires " + "VLLM_USE_V2_MODEL_RUNNER=1" + ) if self.mamba_config.backend != MambaBackendEnum.TRITON: raise ValueError("RecoverSSM requires --mamba-backend triton") elif use_mamba_replayssm_spec: diff --git a/vllm/v1/attention/backends/gdn_recoverssm.py b/vllm/v1/attention/backends/gdn_recoverssm.py index 8b6349a772..e1c12ac6a3 100644 --- a/vllm/v1/attention/backends/gdn_recoverssm.py +++ b/vllm/v1/attention/backends/gdn_recoverssm.py @@ -8,7 +8,7 @@ column, num_accepted_tokens is always 1 (the checkpoint and the compacted conv w already hold the accepted state), and the metadata carries what the post-sampling commit needs. Mirrors the Kimi-K3 KDA builder's RecoverSSM handling.""" -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace from typing import Any import torch @@ -23,6 +23,7 @@ from vllm.v1.attention.backends.gdn_attn import ( from vllm.v1.attention.backends.recoverssm_metadata import ( RecoverSSMMetadata, RecoverSSMPostprocessMetadata, + clone_commit_tensors, recoverssm_request_indices, recoverssm_spec_rows, ) @@ -74,6 +75,32 @@ class GDNRecoverSSMMetadata(GDNAttentionMetadata, RecoverSSMMetadata): block_size=c.block_size, ) + def snapshot_for_deferred_commit(self) -> "GDNRecoverSSMMetadata": + c = self.recoverssm_commit + if c is None: + return self + return replace( + self, + recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes), + ) + + +def _check_context_binding(context: Any, checkpoints: list[torch.Tensor]) -> None: + """A cached commit context must point at the layers' live KV cache. + + The context stores raw base addresses of every layer's conv, state and + replay tensors. One that outlived its cache (the profiling KV cache is + torn down and re-allocated before serving) would keep that cache alive + and, worse, commit into freed memory. Builders are re-created with the + real cache, so a mismatch is a lifecycle bug: fail loudly.""" + bound = [t.data_ptr() for t in context.checkpoints] + live = [t.data_ptr() for t in checkpoints] + if bound != live: + raise RuntimeError( + "GDN RecoverSSM commit context is bound to a different KV cache than " + "its layers (stale context after a KV cache re-allocation)" + ) + class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder): # Uniform spec-decode batches replay as FULL graphs: the base builder stages the @@ -91,6 +118,11 @@ class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder): self._capturing = False def _get_context(self): + if self._context is not None: + fc = self.vllm_config.compilation_config.static_forward_context + _check_context_binding( + self._context, [fc[name].kv_cache[1] for name in self.layer_names] + ) if self._context is None: from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import ( GDNRecoverSSMCommitContext, diff --git a/vllm/v1/attention/backends/ple_recoverssm.py b/vllm/v1/attention/backends/ple_recoverssm.py index 9b43a6fd92..9e44ad8595 100644 --- a/vllm/v1/attention/backends/ple_recoverssm.py +++ b/vllm/v1/attention/backends/ple_recoverssm.py @@ -9,7 +9,7 @@ every active decode row takes the spec path, the conv reads at offset 0, and aft sampling the accepted window is compacted to the front (the shared RecoverSSM compaction kernel; in align mode it also writes the block-boundary window).""" -from dataclasses import dataclass, field, fields +from dataclasses import dataclass, field, fields, replace from typing import Any import torch @@ -25,6 +25,7 @@ from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.attention.backends.recoverssm_metadata import ( RecoverSSMMetadata, RecoverSSMPostprocessMetadata, + clone_commit_tensors, recoverssm_request_indices, recoverssm_spec_rows, ) @@ -228,6 +229,15 @@ class PleRecoverSSMMetadata(PleShortConvAttentionMetadata, RecoverSSMMetadata): block_size=c.block_size, ) + def snapshot_for_deferred_commit(self) -> "PleRecoverSSMMetadata": + c = self.recoverssm_commit + if c is None: + return self + return replace( + self, + recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes), + ) + class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder): # Same as the GDN RecoverSSM builder: uniform spec-decode batches replay as FULL @@ -245,6 +255,17 @@ class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder): self._capturing = False def _get_ctx(self): + if self._ctx is not None: + fc = self.vllm_config.compilation_config.static_forward_context + live = [fc[name].kv_cache[0].data_ptr() for name in self.layer_names] + if [s.data_ptr() for s in self._ctx.conv_states] != live: + # Same lifecycle guard as the GDN builder: never commit into a + # KV cache that was torn down (profiling) and re-allocated. + raise RuntimeError( + "PLE RecoverSSM commit context is bound to a different KV " + "cache than its layers (stale context after a KV cache " + "re-allocation)" + ) if self._ctx is None: fc = self.vllm_config.compilation_config.static_forward_context convs = [fc[name].kv_cache[0] for name in self.layer_names] diff --git a/vllm/v1/attention/backends/recoverssm_metadata.py b/vllm/v1/attention/backends/recoverssm_metadata.py index b1339e9523..135b3eba4c 100644 --- a/vllm/v1/attention/backends/recoverssm_metadata.py +++ b/vllm/v1/attention/backends/recoverssm_metadata.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import abc -from dataclasses import dataclass +from dataclasses import dataclass, replace import torch @@ -25,6 +25,25 @@ class RecoverSSMMetadata(abc.ABC): ) -> RecoverSSMPostprocessMetadata | None: raise NotImplementedError + def has_recoverssm_commit(self) -> bool: + """Whether this step's metadata carries a commit (any spec row).""" + return getattr(self, "recoverssm_commit", None) is not None + + def snapshot_for_deferred_commit(self) -> "RecoverSSMMetadata": + """A copy whose commit inputs no longer alias per-step buffers. + + Under pipeline parallelism a non-last rank learns the accepted counts + pp_size steps after its verify (the PP sampled-token broadcast), and + the steps in between rebuild the builders' persistent, graph-staged + buffers (spec state indices, query offsets, the batch-order block + table). The deferred commit must read this step's values, so they are + cloned here, on the current stream, before the next build overwrites + them. Implementations that do not support the PP deferral raise.""" + raise NotImplementedError( + f"{type(self).__name__} does not support the pipeline-parallel " + "RecoverSSM commit deferral" + ) + def recoverssm_spec_rows( is_prefilling_cpu: torch.Tensor, @@ -62,3 +81,23 @@ def recoverssm_request_indices( if num_spec_decodes == 0 or int(rows[-1]) == num_spec_decodes - 1: return None return rows.to(torch.int32).to(device, non_blocking=True) + + +def clone_commit_tensors(commit, num_spec_decodes: int): + """Clone the commit inputs a deferred (PP) commit reads, trimmed to the real + spec rows. Works for every RecoverSSM commit dataclass with the fields + state_indices, query_start_loc, request_indices, block_table and + num_computed_tokens.""" + n = num_spec_decodes + + def c(t: torch.Tensor | None) -> torch.Tensor | None: + return None if t is None else t.clone() + + return replace( + commit, + state_indices=commit.state_indices[:n].clone(), + query_start_loc=commit.query_start_loc[: n + 1].clone(), + request_indices=c(commit.request_indices), + block_table=c(commit.block_table), + num_computed_tokens=c(commit.num_computed_tokens), + ) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 0ea45ccf0e..cac4328c7f 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -1138,6 +1138,10 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None: # profiling KV cache. if (recoverssm := getattr(runner.model_state, "recoverssm", None)) is not None: recoverssm.reset() + # Under PP the non-last ranks keep detached RecoverSSM steps on the pending + # PP entries; they reference the profiling KV cache just the same. + if (pp_handler := getattr(runner, "pp_handler", None)) is not None: + pp_handler.drop_deferred_states() # Invalidate the align-mode Mamba group metadata cached from the # profiling KVCacheConfig: the real (e.g. PP-projected) config may # place Mamba layers into a different group layout, so it must be diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 9e0c5b77c7..140651316e 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1239,6 +1239,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.req_states.draft_tokens ) if outputs is not None: + # Restore the step's deferred model state (RecoverSSM commits) + # so this postprocess commits the step that sampled them. + self.model_state.restore_deferred_state( + outputs.pop("deferred_state", None) + ) self.postprocess_sampled(**outputs) def add_requests(self, scheduler_output: SchedulerOutput) -> None: @@ -2359,6 +2364,12 @@ class GPUModelRunner(LoRAModelRunnerMixin): ): raise RuntimeError("PLE test fault: exc_async_output") all_decode_next = self.pp_handler.receive(input_batch) + # The accepted counts of this step arrive pp_size steps later; keep + # the state their postprocess needs with this step's pending entry + # (before the immediate prefill-chunk postprocess below). + self.pp_handler.attach_deferred_state( + self.model_state.detach_deferred_state() + ) # Optimistically update num_computed_tokens for entire batch here. # Will be adjusted for rejections if necessary in update_requests. self.postprocess_num_computed_tokens(input_batch) diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index a011898e07..639808c2ba 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -163,6 +163,19 @@ class ModelState(ABC): ) -> None: return None + def detach_deferred_state(self) -> Any: + """Non-last PP rank, after this step's forward: hand over any state the + step's postprocess needs once its sampled tokens arrive (pp_size steps + later). The PP handler keeps it with the step's pending entry. None by + default (nothing deferred).""" + return None + + def restore_deferred_state(self, state: Any) -> None: + """Counterpart of `detach_deferred_state`, called right before the + deferred postprocess of the step that detached `state`.""" + if state is not None: + raise RuntimeError(f"{type(self).__name__} cannot restore {state!r}") + @abstractmethod def prepare_inputs_embeds( self, diff --git a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py index d0b25b22a9..2d72c2a84a 100644 --- a/vllm/v1/worker/gpu/model_states/mamba_hybrid.py +++ b/vllm/v1/worker/gpu/model_states/mamba_hybrid.py @@ -358,6 +358,20 @@ class MambaHybridModelState(DefaultModelState): ) return attn_metadata + def detach_deferred_state(self) -> Any: + # Non-last PP rank: the RecoverSSM commit of this step has to wait for + # the accepted counts of the PP broadcast; snapshot its inputs now. + if self.recoverssm is None: + return None + return self.recoverssm.detach_step() + + def restore_deferred_state(self, state: Any) -> None: + if state is None: + return + if self.recoverssm is None: + raise RuntimeError("deferred RecoverSSM state without RecoverSSM") + self.recoverssm.restore_step(state) + def postprocess_state( self, idx_mapping: torch.Tensor, diff --git a/vllm/v1/worker/gpu/model_states/recoverssm.py b/vllm/v1/worker/gpu/model_states/recoverssm.py index 3cfbecea26..2aafa25972 100644 --- a/vllm/v1/worker/gpu/model_states/recoverssm.py +++ b/vllm/v1/worker/gpu/model_states/recoverssm.py @@ -1,25 +1,77 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass from typing import Any import torch +from vllm.logger import init_logger from vllm.triton_utils import tl, triton from vllm.v1.attention.backends.recoverssm_metadata import RecoverSSMMetadata from vllm.v1.worker.utils import AttentionGroup +logger = init_logger(__name__) + + +@dataclass(frozen=True) +class DeferredRecoverSSMStep: + """One verify step's RecoverSSM commits, detached on a non-last PP rank. + + The rank verified the window at step T but receives the accepted counts + only when the PP broadcast of step T is consumed (pp_size steps later). + The metadata is a snapshot (`snapshot_for_deferred_commit`), so the steps + in between cannot overwrite its commit inputs. It rides on the PP + handler's pending entry for step T and is restored right before that + entry's postprocess.""" + + metadata: tuple[RecoverSSMMetadata, ...] + class RecoverSSMState: """Coordinates RecoverSSM metadata between attention and postprocessing.""" def __init__(self) -> None: self._step: tuple[RecoverSSMMetadata, ...] | None = None + # True while `_step` is a restored PP-deferred step: its commit must + # skip rows whose request was freed since the verify (their state + # blocks may already belong to another request). + self._step_is_deferred = False def reset(self) -> None: """Drop the recorded step. Its metadata holds the commit contexts, which reference the KV cache tensors they were built for.""" self._step = None + self._step_is_deferred = False + + def detach_step(self) -> DeferredRecoverSSMStep | None: + """Take this step's commits for a deferred (non-last PP rank) commit. + + Returns None when the step has nothing to commit (no spec rows).""" + step = self._step + self._step = None + self._step_is_deferred = False + if step is None: + return None + live = tuple(m for m in step if m.has_recoverssm_commit()) + if not live: + return None + return DeferredRecoverSSMStep( + tuple(m.snapshot_for_deferred_commit() for m in live) + ) + + def restore_step(self, deferred: DeferredRecoverSSMStep | None) -> None: + """Make a detached step the one the next `commit_step` commits.""" + if self._step is not None: + # Every non-last-rank step detaches its record in sample_tokens, so + # only a capture-time record (for_capture=False under PIECEWISE) + # can be here; it has no sampled tokens and is never committed. + logger.warning_once( + "RecoverSSM: dropping a recorded step that was never committed " + "(expected only after a PIECEWISE capture)" + ) + self._step = None if deferred is None else deferred.metadata + self._step_is_deferred = deferred is not None def record_step( self, @@ -49,10 +101,26 @@ class RecoverSSMState: num_accepted_tokens: torch.Tensor, ) -> None: step = self._step + deferred = self._step_is_deferred self._step = None + self._step_is_deferred = False if isinstance(num_sampled, int) or step is None: + if deferred and step is not None: + raise RuntimeError( + "RecoverSSM: a deferred PP commit was restored without " + "sampled counts" + ) return + if deferred: + # Under PP the consume filter marks freed/non-sampling rows with -1 + # in idx_mapping. A zero count makes the commit plan skip the row + # (commit_len 0): no state, boundary or conv write into blocks the + # request no longer owns. + num_sampled = torch.where( + idx_mapping >= 0, num_sampled, torch.zeros_like(num_sampled) + ) + for metadata in step: postprocess_meta = metadata.commit_recoverssm_state(num_sampled) if postprocess_meta is None: diff --git a/vllm/v1/worker/gpu/pp_utils.py b/vllm/v1/worker/gpu/pp_utils.py index 818d80759a..16fe0ba62d 100644 --- a/vllm/v1/worker/gpu/pp_utils.py +++ b/vllm/v1/worker/gpu/pp_utils.py @@ -3,7 +3,7 @@ """Pipeline Parallelism utils for V2 Model Runner.""" from collections import deque -from dataclasses import dataclass +from dataclasses import dataclass, replace import numpy as np import torch @@ -33,6 +33,9 @@ class PendingRecv: # detect requests aborted since then. gen_at_receive_np: np.ndarray # [num_reqs] draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps] + # Model-state data the step's postprocess needs once the sampled tokens + # arrive (RecoverSSM: the snapshot of the step's deferred commits). + deferred_state: object | None = None def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None: @@ -167,8 +170,34 @@ class PPHandler: num_sampled=slot.num_sampled, num_rejected=slot.num_rejected, idx_mapping=idx_mapping, + deferred_state=slot.deferred_state, ) + def attach_deferred_state(self, state: object | None) -> None: + """Attach model-state data to the entry `receive` just reserved for this + step. Called right after `receive` in the same step.""" + assert not self.is_last_rank + if state is None: + return + slot = self.queue[-1] + if slot is None: + # receive() skipped the broadcast: no row of this step samples, so + # no row can have a speculative window to commit. + raise RuntimeError( + "deferred model state for a step without sampled outputs" + ) + self.queue[-1] = replace(slot, deferred_state=state) + + def drop_deferred_states(self) -> int: + """Release every pending entry's deferred state (profiling teardown: the + state references the KV cache it was built for). Returns the count.""" + dropped = 0 + for i, slot in enumerate(self.queue): + if slot is not None and slot.deferred_state is not None: + self.queue[i] = replace(slot, deferred_state=None) + dropped += 1 + return dropped + def broadcast_drafts( self, draft_tokens: torch.Tensor, input_batch: InputBatch ) -> None: