recoverssm: allow pipeline parallelism for Qwen4Exp on the V2 runner

Upstream RecoverSSM commits a step in the same step's postprocess, hence PP=1. Under pipeline parallelism a
non-last rank learns a step's accepted counts only when the PP broadcast arrives pp_size steps later, while later
builds overwrite the persistent commit inputs. The commit is now deferred with its own copy of those inputs, and
the PP=1 check is lifted for Qwen4Exp on Model Runner V2. Config and CPU tests cover the deferral.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:08:56 +07:00
co-authored by Hinata
parent 8918d40b59
commit 812d662602
10 changed files with 254 additions and 7 deletions
+19 -3
View File
@@ -3565,9 +3565,25 @@ class VllmConfig:
"RecoverSSM with align mode requires VLLM_USE_V2_MODEL_RUNNER=1"
)
if self.parallel_config.pipeline_parallel_size > 1:
raise ValueError(
"RecoverSSM currently requires pipeline_parallel_size=1"
)
# Non-last PP ranks get the accepted counts pp_size steps after
# their verify. Model Runner V2 defers the commit with a
# snapshot of its inputs (RecoverSSMState.detach_step) and
# masks freed rows; only the Qwen4Exp GDN/PLE metadata
# implement that snapshot (Kimi-K3 KDA does not).
if (
self.model_config is None
or self.model_config.architecture
not in _QWEN4EXP_RECOVERSSM_ARCHS
):
raise ValueError(
"RecoverSSM with pipeline_parallel_size > 1 is supported "
"only for Qwen4Exp GDN/PLE"
)
if not self.use_v2_model_runner:
raise ValueError(
"RecoverSSM with pipeline_parallel_size > 1 requires "
"VLLM_USE_V2_MODEL_RUNNER=1"
)
if self.mamba_config.backend != MambaBackendEnum.TRITON:
raise ValueError("RecoverSSM requires --mamba-backend triton")
elif use_mamba_replayssm_spec:
+33 -1
View File
@@ -8,7 +8,7 @@ column, num_accepted_tokens is always 1 (the checkpoint and the compacted conv w
already hold the accepted state), and the metadata carries what the post-sampling commit
needs. Mirrors the Kimi-K3 KDA builder's RecoverSSM handling."""
from dataclasses import dataclass, field, fields
from dataclasses import dataclass, field, fields, replace
from typing import Any
import torch
@@ -23,6 +23,7 @@ from vllm.v1.attention.backends.gdn_attn import (
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMMetadata,
RecoverSSMPostprocessMetadata,
clone_commit_tensors,
recoverssm_request_indices,
recoverssm_spec_rows,
)
@@ -74,6 +75,32 @@ class GDNRecoverSSMMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
block_size=c.block_size,
)
def snapshot_for_deferred_commit(self) -> "GDNRecoverSSMMetadata":
c = self.recoverssm_commit
if c is None:
return self
return replace(
self,
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
)
def _check_context_binding(context: Any, checkpoints: list[torch.Tensor]) -> None:
"""A cached commit context must point at the layers' live KV cache.
The context stores raw base addresses of every layer's conv, state and
replay tensors. One that outlived its cache (the profiling KV cache is
torn down and re-allocated before serving) would keep that cache alive
and, worse, commit into freed memory. Builders are re-created with the
real cache, so a mismatch is a lifecycle bug: fail loudly."""
bound = [t.data_ptr() for t in context.checkpoints]
live = [t.data_ptr() for t in checkpoints]
if bound != live:
raise RuntimeError(
"GDN RecoverSSM commit context is bound to a different KV cache than "
"its layers (stale context after a KV cache re-allocation)"
)
class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
# Uniform spec-decode batches replay as FULL graphs: the base builder stages the
@@ -91,6 +118,11 @@ class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
self._capturing = False
def _get_context(self):
if self._context is not None:
fc = self.vllm_config.compilation_config.static_forward_context
_check_context_binding(
self._context, [fc[name].kv_cache[1] for name in self.layer_names]
)
if self._context is None:
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
GDNRecoverSSMCommitContext,
+22 -1
View File
@@ -9,7 +9,7 @@ every active decode row takes the spec path, the conv reads at offset 0, and aft
sampling the accepted window is compacted to the front (the shared RecoverSSM compaction
kernel; in align mode it also writes the block-boundary window)."""
from dataclasses import dataclass, field, fields
from dataclasses import dataclass, field, fields, replace
from typing import Any
import torch
@@ -25,6 +25,7 @@ from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMMetadata,
RecoverSSMPostprocessMetadata,
clone_commit_tensors,
recoverssm_request_indices,
recoverssm_spec_rows,
)
@@ -228,6 +229,15 @@ class PleRecoverSSMMetadata(PleShortConvAttentionMetadata, RecoverSSMMetadata):
block_size=c.block_size,
)
def snapshot_for_deferred_commit(self) -> "PleRecoverSSMMetadata":
c = self.recoverssm_commit
if c is None:
return self
return replace(
self,
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
)
class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
# Same as the GDN RecoverSSM builder: uniform spec-decode batches replay as FULL
@@ -245,6 +255,17 @@ class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
self._capturing = False
def _get_ctx(self):
if self._ctx is not None:
fc = self.vllm_config.compilation_config.static_forward_context
live = [fc[name].kv_cache[0].data_ptr() for name in self.layer_names]
if [s.data_ptr() for s in self._ctx.conv_states] != live:
# Same lifecycle guard as the GDN builder: never commit into a
# KV cache that was torn down (profiling) and re-allocated.
raise RuntimeError(
"PLE RecoverSSM commit context is bound to a different KV "
"cache than its layers (stale context after a KV cache "
"re-allocation)"
)
if self._ctx is None:
fc = self.vllm_config.compilation_config.static_forward_context
convs = [fc[name].kv_cache[0] for name in self.layer_names]
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import abc
from dataclasses import dataclass
from dataclasses import dataclass, replace
import torch
@@ -25,6 +25,25 @@ class RecoverSSMMetadata(abc.ABC):
) -> RecoverSSMPostprocessMetadata | None:
raise NotImplementedError
def has_recoverssm_commit(self) -> bool:
"""Whether this step's metadata carries a commit (any spec row)."""
return getattr(self, "recoverssm_commit", None) is not None
def snapshot_for_deferred_commit(self) -> "RecoverSSMMetadata":
"""A copy whose commit inputs no longer alias per-step buffers.
Under pipeline parallelism a non-last rank learns the accepted counts
pp_size steps after its verify (the PP sampled-token broadcast), and
the steps in between rebuild the builders' persistent, graph-staged
buffers (spec state indices, query offsets, the batch-order block
table). The deferred commit must read this step's values, so they are
cloned here, on the current stream, before the next build overwrites
them. Implementations that do not support the PP deferral raise."""
raise NotImplementedError(
f"{type(self).__name__} does not support the pipeline-parallel "
"RecoverSSM commit deferral"
)
def recoverssm_spec_rows(
is_prefilling_cpu: torch.Tensor,
@@ -62,3 +81,23 @@ def recoverssm_request_indices(
if num_spec_decodes == 0 or int(rows[-1]) == num_spec_decodes - 1:
return None
return rows.to(torch.int32).to(device, non_blocking=True)
def clone_commit_tensors(commit, num_spec_decodes: int):
"""Clone the commit inputs a deferred (PP) commit reads, trimmed to the real
spec rows. Works for every RecoverSSM commit dataclass with the fields
state_indices, query_start_loc, request_indices, block_table and
num_computed_tokens."""
n = num_spec_decodes
def c(t: torch.Tensor | None) -> torch.Tensor | None:
return None if t is None else t.clone()
return replace(
commit,
state_indices=commit.state_indices[:n].clone(),
query_start_loc=commit.query_start_loc[: n + 1].clone(),
request_indices=c(commit.request_indices),
block_table=c(commit.block_table),
num_computed_tokens=c(commit.num_computed_tokens),
)
+4
View File
@@ -1138,6 +1138,10 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
# profiling KV cache.
if (recoverssm := getattr(runner.model_state, "recoverssm", None)) is not None:
recoverssm.reset()
# Under PP the non-last ranks keep detached RecoverSSM steps on the pending
# PP entries; they reference the profiling KV cache just the same.
if (pp_handler := getattr(runner, "pp_handler", None)) is not None:
pp_handler.drop_deferred_states()
# Invalidate the align-mode Mamba group metadata cached from the
# profiling KVCacheConfig: the real (e.g. PP-projected) config may
# place Mamba layers into a different group layout, so it must be
+11
View File
@@ -1239,6 +1239,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.req_states.draft_tokens
)
if outputs is not None:
# Restore the step's deferred model state (RecoverSSM commits)
# so this postprocess commits the step that sampled them.
self.model_state.restore_deferred_state(
outputs.pop("deferred_state", None)
)
self.postprocess_sampled(**outputs)
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
@@ -2359,6 +2364,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
):
raise RuntimeError("PLE test fault: exc_async_output")
all_decode_next = self.pp_handler.receive(input_batch)
# The accepted counts of this step arrive pp_size steps later; keep
# the state their postprocess needs with this step's pending entry
# (before the immediate prefill-chunk postprocess below).
self.pp_handler.attach_deferred_state(
self.model_state.detach_deferred_state()
)
# Optimistically update num_computed_tokens for entire batch here.
# Will be adjusted for rejections if necessary in update_requests.
self.postprocess_num_computed_tokens(input_batch)
@@ -163,6 +163,19 @@ class ModelState(ABC):
) -> None:
return None
def detach_deferred_state(self) -> Any:
"""Non-last PP rank, after this step's forward: hand over any state the
step's postprocess needs once its sampled tokens arrive (pp_size steps
later). The PP handler keeps it with the step's pending entry. None by
default (nothing deferred)."""
return None
def restore_deferred_state(self, state: Any) -> None:
"""Counterpart of `detach_deferred_state`, called right before the
deferred postprocess of the step that detached `state`."""
if state is not None:
raise RuntimeError(f"{type(self).__name__} cannot restore {state!r}")
@abstractmethod
def prepare_inputs_embeds(
self,
@@ -358,6 +358,20 @@ class MambaHybridModelState(DefaultModelState):
)
return attn_metadata
def detach_deferred_state(self) -> Any:
# Non-last PP rank: the RecoverSSM commit of this step has to wait for
# the accepted counts of the PP broadcast; snapshot its inputs now.
if self.recoverssm is None:
return None
return self.recoverssm.detach_step()
def restore_deferred_state(self, state: Any) -> None:
if state is None:
return
if self.recoverssm is None:
raise RuntimeError("deferred RecoverSSM state without RecoverSSM")
self.recoverssm.restore_step(state)
def postprocess_state(
self,
idx_mapping: torch.Tensor,
@@ -1,25 +1,77 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
import torch
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.recoverssm_metadata import RecoverSSMMetadata
from vllm.v1.worker.utils import AttentionGroup
logger = init_logger(__name__)
@dataclass(frozen=True)
class DeferredRecoverSSMStep:
"""One verify step's RecoverSSM commits, detached on a non-last PP rank.
The rank verified the window at step T but receives the accepted counts
only when the PP broadcast of step T is consumed (pp_size steps later).
The metadata is a snapshot (`snapshot_for_deferred_commit`), so the steps
in between cannot overwrite its commit inputs. It rides on the PP
handler's pending entry for step T and is restored right before that
entry's postprocess."""
metadata: tuple[RecoverSSMMetadata, ...]
class RecoverSSMState:
"""Coordinates RecoverSSM metadata between attention and postprocessing."""
def __init__(self) -> None:
self._step: tuple[RecoverSSMMetadata, ...] | None = None
# True while `_step` is a restored PP-deferred step: its commit must
# skip rows whose request was freed since the verify (their state
# blocks may already belong to another request).
self._step_is_deferred = False
def reset(self) -> None:
"""Drop the recorded step. Its metadata holds the commit contexts, which
reference the KV cache tensors they were built for."""
self._step = None
self._step_is_deferred = False
def detach_step(self) -> DeferredRecoverSSMStep | None:
"""Take this step's commits for a deferred (non-last PP rank) commit.
Returns None when the step has nothing to commit (no spec rows)."""
step = self._step
self._step = None
self._step_is_deferred = False
if step is None:
return None
live = tuple(m for m in step if m.has_recoverssm_commit())
if not live:
return None
return DeferredRecoverSSMStep(
tuple(m.snapshot_for_deferred_commit() for m in live)
)
def restore_step(self, deferred: DeferredRecoverSSMStep | None) -> None:
"""Make a detached step the one the next `commit_step` commits."""
if self._step is not None:
# Every non-last-rank step detaches its record in sample_tokens, so
# only a capture-time record (for_capture=False under PIECEWISE)
# can be here; it has no sampled tokens and is never committed.
logger.warning_once(
"RecoverSSM: dropping a recorded step that was never committed "
"(expected only after a PIECEWISE capture)"
)
self._step = None if deferred is None else deferred.metadata
self._step_is_deferred = deferred is not None
def record_step(
self,
@@ -49,10 +101,26 @@ class RecoverSSMState:
num_accepted_tokens: torch.Tensor,
) -> None:
step = self._step
deferred = self._step_is_deferred
self._step = None
self._step_is_deferred = False
if isinstance(num_sampled, int) or step is None:
if deferred and step is not None:
raise RuntimeError(
"RecoverSSM: a deferred PP commit was restored without "
"sampled counts"
)
return
if deferred:
# Under PP the consume filter marks freed/non-sampling rows with -1
# in idx_mapping. A zero count makes the commit plan skip the row
# (commit_len 0): no state, boundary or conv write into blocks the
# request no longer owns.
num_sampled = torch.where(
idx_mapping >= 0, num_sampled, torch.zeros_like(num_sampled)
)
for metadata in step:
postprocess_meta = metadata.commit_recoverssm_state(num_sampled)
if postprocess_meta is None:
+30 -1
View File
@@ -3,7 +3,7 @@
"""Pipeline Parallelism utils for V2 Model Runner."""
from collections import deque
from dataclasses import dataclass
from dataclasses import dataclass, replace
import numpy as np
import torch
@@ -33,6 +33,9 @@ class PendingRecv:
# detect requests aborted since then.
gen_at_receive_np: np.ndarray # [num_reqs]
draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps]
# Model-state data the step's postprocess needs once the sampled tokens
# arrive (RecoverSSM: the snapshot of the step's deferred commits).
deferred_state: object | None = None
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
@@ -167,8 +170,34 @@ class PPHandler:
num_sampled=slot.num_sampled,
num_rejected=slot.num_rejected,
idx_mapping=idx_mapping,
deferred_state=slot.deferred_state,
)
def attach_deferred_state(self, state: object | None) -> None:
"""Attach model-state data to the entry `receive` just reserved for this
step. Called right after `receive` in the same step."""
assert not self.is_last_rank
if state is None:
return
slot = self.queue[-1]
if slot is None:
# receive() skipped the broadcast: no row of this step samples, so
# no row can have a speculative window to commit.
raise RuntimeError(
"deferred model state for a step without sampled outputs"
)
self.queue[-1] = replace(slot, deferred_state=state)
def drop_deferred_states(self) -> int:
"""Release every pending entry's deferred state (profiling teardown: the
state references the KV cache it was built for). Returns the count."""
dropped = 0
for i, slot in enumerate(self.queue):
if slot is not None and slot.deferred_state is not None:
self.queue[i] = replace(slot, deferred_state=None)
dropped += 1
return dropped
def broadcast_drafts(
self, draft_tokens: torch.Tensor, input_batch: InputBatch
) -> None: