recoverssm: allow pipeline parallelism for Qwen4Exp on the V2 runner
Upstream RecoverSSM commits a step in the same step's postprocess, hence PP=1. Under pipeline parallelism a non-last rank learns a step's accepted counts only when the PP broadcast arrives pp_size steps later, while later builds overwrite the persistent commit inputs. The commit is now deferred with its own copy of those inputs, and the PP=1 check is lifted for Qwen4Exp on Model Runner V2. Config and CPU tests cover the deferral. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+19
-3
@@ -3565,9 +3565,25 @@ class VllmConfig:
|
||||
"RecoverSSM with align mode requires VLLM_USE_V2_MODEL_RUNNER=1"
|
||||
)
|
||||
if self.parallel_config.pipeline_parallel_size > 1:
|
||||
raise ValueError(
|
||||
"RecoverSSM currently requires pipeline_parallel_size=1"
|
||||
)
|
||||
# Non-last PP ranks get the accepted counts pp_size steps after
|
||||
# their verify. Model Runner V2 defers the commit with a
|
||||
# snapshot of its inputs (RecoverSSMState.detach_step) and
|
||||
# masks freed rows; only the Qwen4Exp GDN/PLE metadata
|
||||
# implement that snapshot (Kimi-K3 KDA does not).
|
||||
if (
|
||||
self.model_config is None
|
||||
or self.model_config.architecture
|
||||
not in _QWEN4EXP_RECOVERSSM_ARCHS
|
||||
):
|
||||
raise ValueError(
|
||||
"RecoverSSM with pipeline_parallel_size > 1 is supported "
|
||||
"only for Qwen4Exp GDN/PLE"
|
||||
)
|
||||
if not self.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"RecoverSSM with pipeline_parallel_size > 1 requires "
|
||||
"VLLM_USE_V2_MODEL_RUNNER=1"
|
||||
)
|
||||
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError("RecoverSSM requires --mamba-backend triton")
|
||||
elif use_mamba_replayssm_spec:
|
||||
|
||||
@@ -8,7 +8,7 @@ column, num_accepted_tokens is always 1 (the checkpoint and the compacted conv w
|
||||
already hold the accepted state), and the metadata carries what the post-sampling commit
|
||||
needs. Mirrors the Kimi-K3 KDA builder's RecoverSSM handling."""
|
||||
|
||||
from dataclasses import dataclass, field, fields
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.attention.backends.gdn_attn import (
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
clone_commit_tensors,
|
||||
recoverssm_request_indices,
|
||||
recoverssm_spec_rows,
|
||||
)
|
||||
@@ -74,6 +75,32 @@ class GDNRecoverSSMMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
|
||||
block_size=c.block_size,
|
||||
)
|
||||
|
||||
def snapshot_for_deferred_commit(self) -> "GDNRecoverSSMMetadata":
|
||||
c = self.recoverssm_commit
|
||||
if c is None:
|
||||
return self
|
||||
return replace(
|
||||
self,
|
||||
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
|
||||
)
|
||||
|
||||
|
||||
def _check_context_binding(context: Any, checkpoints: list[torch.Tensor]) -> None:
|
||||
"""A cached commit context must point at the layers' live KV cache.
|
||||
|
||||
The context stores raw base addresses of every layer's conv, state and
|
||||
replay tensors. One that outlived its cache (the profiling KV cache is
|
||||
torn down and re-allocated before serving) would keep that cache alive
|
||||
and, worse, commit into freed memory. Builders are re-created with the
|
||||
real cache, so a mismatch is a lifecycle bug: fail loudly."""
|
||||
bound = [t.data_ptr() for t in context.checkpoints]
|
||||
live = [t.data_ptr() for t in checkpoints]
|
||||
if bound != live:
|
||||
raise RuntimeError(
|
||||
"GDN RecoverSSM commit context is bound to a different KV cache than "
|
||||
"its layers (stale context after a KV cache re-allocation)"
|
||||
)
|
||||
|
||||
|
||||
class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
# Uniform spec-decode batches replay as FULL graphs: the base builder stages the
|
||||
@@ -91,6 +118,11 @@ class GDNRecoverSSMMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
self._capturing = False
|
||||
|
||||
def _get_context(self):
|
||||
if self._context is not None:
|
||||
fc = self.vllm_config.compilation_config.static_forward_context
|
||||
_check_context_binding(
|
||||
self._context, [fc[name].kv_cache[1] for name in self.layer_names]
|
||||
)
|
||||
if self._context is None:
|
||||
from vllm.model_executor.layers.mamba.gdn.recoverssm_gdn import (
|
||||
GDNRecoverSSMCommitContext,
|
||||
|
||||
@@ -9,7 +9,7 @@ every active decode row takes the spec path, the conv reads at offset 0, and aft
|
||||
sampling the accepted window is compacted to the front (the shared RecoverSSM compaction
|
||||
kernel; in align mode it also writes the block-boundary window)."""
|
||||
|
||||
from dataclasses import dataclass, field, fields
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -25,6 +25,7 @@ from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
clone_commit_tensors,
|
||||
recoverssm_request_indices,
|
||||
recoverssm_spec_rows,
|
||||
)
|
||||
@@ -228,6 +229,15 @@ class PleRecoverSSMMetadata(PleShortConvAttentionMetadata, RecoverSSMMetadata):
|
||||
block_size=c.block_size,
|
||||
)
|
||||
|
||||
def snapshot_for_deferred_commit(self) -> "PleRecoverSSMMetadata":
|
||||
c = self.recoverssm_commit
|
||||
if c is None:
|
||||
return self
|
||||
return replace(
|
||||
self,
|
||||
recoverssm_commit=clone_commit_tensors(c, self.num_spec_decodes),
|
||||
)
|
||||
|
||||
|
||||
class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||
# Same as the GDN RecoverSSM builder: uniform spec-decode batches replay as FULL
|
||||
@@ -245,6 +255,17 @@ class PleRecoverSSMMetadataBuilder(PleShortConvAttentionMetadataBuilder):
|
||||
self._capturing = False
|
||||
|
||||
def _get_ctx(self):
|
||||
if self._ctx is not None:
|
||||
fc = self.vllm_config.compilation_config.static_forward_context
|
||||
live = [fc[name].kv_cache[0].data_ptr() for name in self.layer_names]
|
||||
if [s.data_ptr() for s in self._ctx.conv_states] != live:
|
||||
# Same lifecycle guard as the GDN builder: never commit into a
|
||||
# KV cache that was torn down (profiling) and re-allocated.
|
||||
raise RuntimeError(
|
||||
"PLE RecoverSSM commit context is bound to a different KV "
|
||||
"cache than its layers (stale context after a KV cache "
|
||||
"re-allocation)"
|
||||
)
|
||||
if self._ctx is None:
|
||||
fc = self.vllm_config.compilation_config.static_forward_context
|
||||
convs = [fc[name].kv_cache[0] for name in self.layer_names]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import abc
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import torch
|
||||
|
||||
@@ -25,6 +25,25 @@ class RecoverSSMMetadata(abc.ABC):
|
||||
) -> RecoverSSMPostprocessMetadata | None:
|
||||
raise NotImplementedError
|
||||
|
||||
def has_recoverssm_commit(self) -> bool:
|
||||
"""Whether this step's metadata carries a commit (any spec row)."""
|
||||
return getattr(self, "recoverssm_commit", None) is not None
|
||||
|
||||
def snapshot_for_deferred_commit(self) -> "RecoverSSMMetadata":
|
||||
"""A copy whose commit inputs no longer alias per-step buffers.
|
||||
|
||||
Under pipeline parallelism a non-last rank learns the accepted counts
|
||||
pp_size steps after its verify (the PP sampled-token broadcast), and
|
||||
the steps in between rebuild the builders' persistent, graph-staged
|
||||
buffers (spec state indices, query offsets, the batch-order block
|
||||
table). The deferred commit must read this step's values, so they are
|
||||
cloned here, on the current stream, before the next build overwrites
|
||||
them. Implementations that do not support the PP deferral raise."""
|
||||
raise NotImplementedError(
|
||||
f"{type(self).__name__} does not support the pipeline-parallel "
|
||||
"RecoverSSM commit deferral"
|
||||
)
|
||||
|
||||
|
||||
def recoverssm_spec_rows(
|
||||
is_prefilling_cpu: torch.Tensor,
|
||||
@@ -62,3 +81,23 @@ def recoverssm_request_indices(
|
||||
if num_spec_decodes == 0 or int(rows[-1]) == num_spec_decodes - 1:
|
||||
return None
|
||||
return rows.to(torch.int32).to(device, non_blocking=True)
|
||||
|
||||
|
||||
def clone_commit_tensors(commit, num_spec_decodes: int):
|
||||
"""Clone the commit inputs a deferred (PP) commit reads, trimmed to the real
|
||||
spec rows. Works for every RecoverSSM commit dataclass with the fields
|
||||
state_indices, query_start_loc, request_indices, block_table and
|
||||
num_computed_tokens."""
|
||||
n = num_spec_decodes
|
||||
|
||||
def c(t: torch.Tensor | None) -> torch.Tensor | None:
|
||||
return None if t is None else t.clone()
|
||||
|
||||
return replace(
|
||||
commit,
|
||||
state_indices=commit.state_indices[:n].clone(),
|
||||
query_start_loc=commit.query_start_loc[: n + 1].clone(),
|
||||
request_indices=c(commit.request_indices),
|
||||
block_table=c(commit.block_table),
|
||||
num_computed_tokens=c(commit.num_computed_tokens),
|
||||
)
|
||||
|
||||
@@ -1138,6 +1138,10 @@ def _teardown_profiling_state(runner: "GPUModelRunner") -> None:
|
||||
# profiling KV cache.
|
||||
if (recoverssm := getattr(runner.model_state, "recoverssm", None)) is not None:
|
||||
recoverssm.reset()
|
||||
# Under PP the non-last ranks keep detached RecoverSSM steps on the pending
|
||||
# PP entries; they reference the profiling KV cache just the same.
|
||||
if (pp_handler := getattr(runner, "pp_handler", None)) is not None:
|
||||
pp_handler.drop_deferred_states()
|
||||
# Invalidate the align-mode Mamba group metadata cached from the
|
||||
# profiling KVCacheConfig: the real (e.g. PP-projected) config may
|
||||
# place Mamba layers into a different group layout, so it must be
|
||||
|
||||
@@ -1239,6 +1239,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.req_states.draft_tokens
|
||||
)
|
||||
if outputs is not None:
|
||||
# Restore the step's deferred model state (RecoverSSM commits)
|
||||
# so this postprocess commits the step that sampled them.
|
||||
self.model_state.restore_deferred_state(
|
||||
outputs.pop("deferred_state", None)
|
||||
)
|
||||
self.postprocess_sampled(**outputs)
|
||||
|
||||
def add_requests(self, scheduler_output: SchedulerOutput) -> None:
|
||||
@@ -2359,6 +2364,12 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_async_output")
|
||||
all_decode_next = self.pp_handler.receive(input_batch)
|
||||
# The accepted counts of this step arrive pp_size steps later; keep
|
||||
# the state their postprocess needs with this step's pending entry
|
||||
# (before the immediate prefill-chunk postprocess below).
|
||||
self.pp_handler.attach_deferred_state(
|
||||
self.model_state.detach_deferred_state()
|
||||
)
|
||||
# Optimistically update num_computed_tokens for entire batch here.
|
||||
# Will be adjusted for rejections if necessary in update_requests.
|
||||
self.postprocess_num_computed_tokens(input_batch)
|
||||
|
||||
@@ -163,6 +163,19 @@ class ModelState(ABC):
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def detach_deferred_state(self) -> Any:
|
||||
"""Non-last PP rank, after this step's forward: hand over any state the
|
||||
step's postprocess needs once its sampled tokens arrive (pp_size steps
|
||||
later). The PP handler keeps it with the step's pending entry. None by
|
||||
default (nothing deferred)."""
|
||||
return None
|
||||
|
||||
def restore_deferred_state(self, state: Any) -> None:
|
||||
"""Counterpart of `detach_deferred_state`, called right before the
|
||||
deferred postprocess of the step that detached `state`."""
|
||||
if state is not None:
|
||||
raise RuntimeError(f"{type(self).__name__} cannot restore {state!r}")
|
||||
|
||||
@abstractmethod
|
||||
def prepare_inputs_embeds(
|
||||
self,
|
||||
|
||||
@@ -358,6 +358,20 @@ class MambaHybridModelState(DefaultModelState):
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def detach_deferred_state(self) -> Any:
|
||||
# Non-last PP rank: the RecoverSSM commit of this step has to wait for
|
||||
# the accepted counts of the PP broadcast; snapshot its inputs now.
|
||||
if self.recoverssm is None:
|
||||
return None
|
||||
return self.recoverssm.detach_step()
|
||||
|
||||
def restore_deferred_state(self, state: Any) -> None:
|
||||
if state is None:
|
||||
return
|
||||
if self.recoverssm is None:
|
||||
raise RuntimeError("deferred RecoverSSM state without RecoverSSM")
|
||||
self.recoverssm.restore_step(state)
|
||||
|
||||
def postprocess_state(
|
||||
self,
|
||||
idx_mapping: torch.Tensor,
|
||||
|
||||
@@ -1,25 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import RecoverSSMMetadata
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DeferredRecoverSSMStep:
|
||||
"""One verify step's RecoverSSM commits, detached on a non-last PP rank.
|
||||
|
||||
The rank verified the window at step T but receives the accepted counts
|
||||
only when the PP broadcast of step T is consumed (pp_size steps later).
|
||||
The metadata is a snapshot (`snapshot_for_deferred_commit`), so the steps
|
||||
in between cannot overwrite its commit inputs. It rides on the PP
|
||||
handler's pending entry for step T and is restored right before that
|
||||
entry's postprocess."""
|
||||
|
||||
metadata: tuple[RecoverSSMMetadata, ...]
|
||||
|
||||
|
||||
class RecoverSSMState:
|
||||
"""Coordinates RecoverSSM metadata between attention and postprocessing."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._step: tuple[RecoverSSMMetadata, ...] | None = None
|
||||
# True while `_step` is a restored PP-deferred step: its commit must
|
||||
# skip rows whose request was freed since the verify (their state
|
||||
# blocks may already belong to another request).
|
||||
self._step_is_deferred = False
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop the recorded step. Its metadata holds the commit contexts, which
|
||||
reference the KV cache tensors they were built for."""
|
||||
self._step = None
|
||||
self._step_is_deferred = False
|
||||
|
||||
def detach_step(self) -> DeferredRecoverSSMStep | None:
|
||||
"""Take this step's commits for a deferred (non-last PP rank) commit.
|
||||
|
||||
Returns None when the step has nothing to commit (no spec rows)."""
|
||||
step = self._step
|
||||
self._step = None
|
||||
self._step_is_deferred = False
|
||||
if step is None:
|
||||
return None
|
||||
live = tuple(m for m in step if m.has_recoverssm_commit())
|
||||
if not live:
|
||||
return None
|
||||
return DeferredRecoverSSMStep(
|
||||
tuple(m.snapshot_for_deferred_commit() for m in live)
|
||||
)
|
||||
|
||||
def restore_step(self, deferred: DeferredRecoverSSMStep | None) -> None:
|
||||
"""Make a detached step the one the next `commit_step` commits."""
|
||||
if self._step is not None:
|
||||
# Every non-last-rank step detaches its record in sample_tokens, so
|
||||
# only a capture-time record (for_capture=False under PIECEWISE)
|
||||
# can be here; it has no sampled tokens and is never committed.
|
||||
logger.warning_once(
|
||||
"RecoverSSM: dropping a recorded step that was never committed "
|
||||
"(expected only after a PIECEWISE capture)"
|
||||
)
|
||||
self._step = None if deferred is None else deferred.metadata
|
||||
self._step_is_deferred = deferred is not None
|
||||
|
||||
def record_step(
|
||||
self,
|
||||
@@ -49,10 +101,26 @@ class RecoverSSMState:
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
step = self._step
|
||||
deferred = self._step_is_deferred
|
||||
self._step = None
|
||||
self._step_is_deferred = False
|
||||
if isinstance(num_sampled, int) or step is None:
|
||||
if deferred and step is not None:
|
||||
raise RuntimeError(
|
||||
"RecoverSSM: a deferred PP commit was restored without "
|
||||
"sampled counts"
|
||||
)
|
||||
return
|
||||
|
||||
if deferred:
|
||||
# Under PP the consume filter marks freed/non-sampling rows with -1
|
||||
# in idx_mapping. A zero count makes the commit plan skip the row
|
||||
# (commit_len 0): no state, boundary or conv write into blocks the
|
||||
# request no longer owns.
|
||||
num_sampled = torch.where(
|
||||
idx_mapping >= 0, num_sampled, torch.zeros_like(num_sampled)
|
||||
)
|
||||
|
||||
for metadata in step:
|
||||
postprocess_meta = metadata.commit_recoverssm_state(num_sampled)
|
||||
if postprocess_meta is None:
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Pipeline Parallelism utils for V2 Model Runner."""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -33,6 +33,9 @@ class PendingRecv:
|
||||
# detect requests aborted since then.
|
||||
gen_at_receive_np: np.ndarray # [num_reqs]
|
||||
draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps]
|
||||
# Model-state data the step's postprocess needs once the sampled tokens
|
||||
# arrive (RecoverSSM: the snapshot of the step's deferred commits).
|
||||
deferred_state: object | None = None
|
||||
|
||||
|
||||
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
@@ -167,8 +170,34 @@ class PPHandler:
|
||||
num_sampled=slot.num_sampled,
|
||||
num_rejected=slot.num_rejected,
|
||||
idx_mapping=idx_mapping,
|
||||
deferred_state=slot.deferred_state,
|
||||
)
|
||||
|
||||
def attach_deferred_state(self, state: object | None) -> None:
|
||||
"""Attach model-state data to the entry `receive` just reserved for this
|
||||
step. Called right after `receive` in the same step."""
|
||||
assert not self.is_last_rank
|
||||
if state is None:
|
||||
return
|
||||
slot = self.queue[-1]
|
||||
if slot is None:
|
||||
# receive() skipped the broadcast: no row of this step samples, so
|
||||
# no row can have a speculative window to commit.
|
||||
raise RuntimeError(
|
||||
"deferred model state for a step without sampled outputs"
|
||||
)
|
||||
self.queue[-1] = replace(slot, deferred_state=state)
|
||||
|
||||
def drop_deferred_states(self) -> int:
|
||||
"""Release every pending entry's deferred state (profiling teardown: the
|
||||
state references the KV cache it was built for). Returns the count."""
|
||||
dropped = 0
|
||||
for i, slot in enumerate(self.queue):
|
||||
if slot is not None and slot.deferred_state is not None:
|
||||
self.queue[i] = replace(slot, deferred_state=None)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_tokens: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user