[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-30 21:41:17 +00:00
committed by GitHub
co-authored by Andrii Skliar
parent 7e583e615c
commit deb148e730
18 changed files with 1029 additions and 113 deletions
+2 -2
View File
@@ -74,11 +74,11 @@ class MambaBase(AttentionLayerBase):
mamba_type=self.mamba_type,
tp_replicated=self.is_kv_cache_tp_replicated,
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
# RecoverSSM verifies the whole window off one checkpoint, so it
# ReplaySSM verifies the whole window off one checkpoint, so it
# never writes the baseline's per-draft-token state slots.
num_speculative_blocks=(
0
if vllm_config.cache_config.use_kda_recoverssm
if vllm_config.cache_config.use_replayssm
else vllm_config.num_speculative_tokens
),
)
@@ -43,6 +43,7 @@ from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
commit_replayssm_ring_trackers,
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
@@ -69,6 +70,13 @@ from vllm.v1.kv_cache_interface import KVCacheGroupSpec
logger = init_logger(__name__)
def _view_mtp_decode_tensor(
tensor: torch.Tensor, decode_batch: int, spec_query_len: int
) -> torch.Tensor:
return tensor.view(decode_batch, spec_query_len, *tensor.shape[1:])
# Added by the IBM Team, 2024
@@ -536,6 +544,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_query_len = torch.empty(0, dtype=torch.int32)
self._commits_replayssm_trackers = True
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
@@ -724,7 +734,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
ring_start = prev_num_accepted = None
use_spec_decode = self.num_spec > 0
ring_start = prev_num_accepted = prev_query_len = None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -746,6 +757,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
ring_start = self._replayssm_ring_start
prev_num_accepted = self._replayssm_prev_num_accepted
prev_query_len = self._replayssm_prev_query_len
else:
x_cache = dt_cache = B_cache = None
has_initial_states_p = attn_metadata.has_initial_states_p
@@ -757,6 +769,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
last_chunk_indices_p = attn_metadata.last_chunk_indices_p
state_indices_tensor_p = attn_metadata.state_indices_tensor_p
state_indices_tensor_d = attn_metadata.state_indices_tensor_d
replayssm_state_indices_d = attn_metadata.replayssm_state_indices_d
num_accepted_tokens = attn_metadata.num_accepted_tokens
query_start_loc_d = attn_metadata.query_start_loc_d
num_decodes = attn_metadata.num_decodes
@@ -996,6 +1009,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_indices_tensor_p,
)
@@ -1003,7 +1017,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
if has_decode:
assert state_indices_tensor_d is not None
if is_mamba_cache_all:
if self.num_spec > 0:
if use_spec_decode:
assert block_idx_last_scheduled_token_prev_step_d is not None
input_indices = (
block_idx_last_scheduled_token_prev_step_d.unsqueeze(1)
@@ -1031,6 +1045,34 @@ class MambaMixer2(MambaBase, PluggableLayer):
state_indices_tensor_d_input = state_indices_tensor_d
state_indices_tensor_d_output = state_indices_tensor_d
if (
self.use_replayssm
and self.mamba_config.backend == MambaBackendEnum.FLASHINFER
and use_spec_decode
and self._commits_replayssm_trackers
):
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert replayssm_state_indices_d is not None
assert num_accepted_tokens is not None
assert query_start_loc_d is not None
assert x_cache is not None
assert self.replayssm_buffer_len is not None
# The previous MTP query's accepted prefix is known only now.
# Commit it once, before this cache group's layers evaluate the
# current speculative query with the shared ring positions.
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
replayssm_state_indices_d,
num_accepted_tokens,
query_start_loc_d,
logical_window=self.replayssm_buffer_len,
ring_buffer_len=x_cache.size(2),
)
# 2. Convolution sequence transformation
hidden_states_B_C_d = causal_conv1d_update(
hidden_states_B_C_d,
@@ -1043,7 +1085,13 @@ class MambaMixer2(MambaBase, PluggableLayer):
initial_state_idx=block_idx_last_computed_token_d,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc_d,
max_query_len=state_indices_tensor_d.size(-1),
# ReplaySSM keeps one physical state block while a speculative
# decode call still processes the full target + draft window.
max_query_len=(
1 + self.num_spec
if self.use_replayssm and use_spec_decode
else state_indices_tensor_d.size(-1)
),
)
hidden_states_d, B_d, C_d = self.split_hidden_states_B_C_fn(
@@ -1079,7 +1127,35 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
assert ring_start is not None
assert prev_num_accepted is not None
assert prev_query_len is not None
assert attn_metadata.replayssm_scratch is not None
fi_cu_seqlens = query_start_loc_d
fi_max_seqlen = None
if use_spec_decode:
spec_query_len = 1 + self.num_spec
fi_max_seqlen = spec_query_len
assert replayssm_state_indices_d is not None
decode_batch = replayssm_state_indices_d.size(0)
if num_decode_tokens == decode_batch * spec_query_len:
hidden_states_d = _view_mtp_decode_tensor(
hidden_states_d, decode_batch, spec_query_len
)
dt_d = _view_mtp_decode_tensor(
dt_d, decode_batch, spec_query_len
)
B_d = _view_mtp_decode_tensor(
B_d, decode_batch, spec_query_len
)
C_d = _view_mtp_decode_tensor(
C_d, decode_batch, spec_query_len
)
preallocated_ssm_out_d = _view_mtp_decode_tensor(
preallocated_ssm_out_d,
decode_batch,
spec_query_len,
)
fi_cu_seqlens = None
fi_max_seqlen = None
selective_state_update_replayssm_flashinfer(
ssm_state,
hidden_states_d,
@@ -1093,19 +1169,25 @@ class MambaMixer2(MambaBase, PluggableLayer):
dt_cache,
ring_start,
prev_num_accepted,
prev_query_len,
logical_window=self.replayssm_buffer_len,
D=D_d,
dt_bias=dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
state_batch_indices=replayssm_state_indices_d,
scratch=attn_metadata.replayssm_scratch,
update_trackers=self._updates_replayssm_trackers,
update_trackers=(
self._updates_replayssm_trackers and not use_spec_decode
),
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
stochastic_rounding_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
cu_seqlens=fi_cu_seqlens,
max_seqlen=fi_max_seqlen,
enable_pdl=False,
)
else:
selective_state_update_replayssm_output_only(
@@ -1187,6 +1269,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
tp_world_size=tp_world_size,
logical_window=self.replayssm_buffer_len,
backend=self.mamba_config.backend,
num_speculative_tokens=self.num_spec,
)
return base_shape
@@ -1204,9 +1287,10 @@ def share_replayssm_ring_trackers(
Layers backed by one KV-cache group use the same physical block indices and
can therefore share cursors. Different KV-cache groups may assign different
block indices to the same request and must keep separate cursor tensors.
The final local layer in each group advances its cursors after every layer
in that group has consumed the previous values.
block indices to the same request and must keep separate cursor tensors. For
speculative decode, the first local layer commits the preceding acceptance;
for standard decode, the final local layer advances after all layers consume
the previous values.
"""
replayssm_mixers: dict[str, MambaMixer2] = {}
for layer_name in ordered_layer_names:
@@ -1232,6 +1316,7 @@ def share_replayssm_ring_trackers(
groups_by_namespace.setdefault(namespace, []).append(layer_name)
for group_layer_names in groups_by_namespace.values():
first_layer_name = group_layer_names[0]
last_layer_name = group_layer_names[-1]
first_mixer = replayssm_mixers[group_layer_names[0]]
@@ -1246,11 +1331,17 @@ def share_replayssm_ring_trackers(
ring_start = torch.zeros(num_blocks, dtype=torch.int32, device=device)
prev_num_accepted = torch.zeros_like(ring_start)
prev_query_len = torch.zeros_like(ring_start)
for layer_name in group_layer_names:
mixer = replayssm_mixers[layer_name]
mixer._replayssm_ring_start = ring_start
mixer._replayssm_prev_num_accepted = prev_num_accepted
mixer._updates_replayssm_trackers = layer_name == last_layer_name
mixer._replayssm_prev_query_len = prev_query_len
mixer._commits_replayssm_trackers = False
mixer._updates_replayssm_trackers = False
replayssm_mixers[first_layer_name]._commits_replayssm_trackers = True
replayssm_mixers[last_layer_name]._updates_replayssm_trackers = True
def mamba_mixer2(
@@ -227,6 +227,7 @@ class MambaStateShapeCalculator:
tp_world_size: int,
logical_window: int,
backend: MambaBackendEnum,
num_speculative_tokens: int = 0,
) -> tuple[
tuple[int, int],
tuple[int, int, int],
@@ -241,8 +242,8 @@ class MambaStateShapeCalculator:
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and appended token together.
ring_buffer_len += 1
# FlashInfer keeps the live window and current verify window together.
ring_buffer_len += 1 + num_speculative_tokens
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
@@ -31,6 +31,7 @@ logger = init_logger(__name__)
def _update_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices_stride,
n_slots,
@@ -52,6 +53,9 @@ def _update_replayssm_ring_trackers_kernel(
if RESET:
tl.store(ring_start + slots, 0, mask=valid)
tl.store(prev_num_accepted + slots, 0, mask=valid)
# MTP alone reads prev_query_len. Standard decode deliberately leaves
# it unchanged; reset clears the pending MTP window after prefill.
tl.store(prev_query_len + slots, 0, mask=valid)
else:
prev = tl.load(prev_num_accepted + slots, mask=valid, other=0)
start = tl.load(ring_start + slots, mask=valid, other=0)
@@ -66,9 +70,70 @@ def _update_replayssm_ring_trackers_kernel(
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
@triton.jit(
do_not_specialize=["n_slots", "state_batch_indices_stride"],
do_not_specialize_on_alignment=["state_batch_indices"],
)
def _commit_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices_stride,
n_slots,
num_states,
logical_window: tl.constexpr,
ring_buffer_len: tl.constexpr,
pad_slot_id: tl.constexpr,
BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_slots
slots = tl.load(
state_batch_indices + offsets * state_batch_indices_stride,
mask=mask,
other=pad_slot_id,
)
valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states)
accepted_since_checkpoint = tl.load(prev_num_accepted + slots, mask=valid, other=0)
current_ring_start = tl.load(ring_start + slots, mask=valid, other=0)
previous_speculative_query_len = tl.load(
prev_query_len + slots, mask=valid, other=0
)
accepted_from_previous_query = tl.load(
num_accepted_tokens + offsets, mask=mask, other=0
)
must_checkpoint = (previous_speculative_query_len > 0) & (
accepted_since_checkpoint + previous_speculative_query_len > logical_window
)
next_start = tl.where(
must_checkpoint,
(current_ring_start + accepted_since_checkpoint) % ring_buffer_len,
current_ring_start,
)
next_prev = tl.where(
previous_speculative_query_len == 0,
0,
tl.where(
must_checkpoint,
accepted_from_previous_query,
accepted_since_checkpoint + accepted_from_previous_query,
),
)
current_speculative_query_len = tl.load(
query_start_loc + offsets + 1, mask=mask, other=0
) - tl.load(query_start_loc + offsets, mask=mask, other=0)
tl.store(ring_start + slots, next_start, mask=valid)
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
tl.store(prev_query_len + slots, current_speculative_query_len, mask=valid)
def update_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
logical_window: int | None = None,
ring_buffer_len: int | None = None,
@@ -90,10 +155,15 @@ def update_replayssm_ring_trackers(
_update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
state_batch_indices.stride(0),
n_slots,
min(ring_start.numel(), prev_num_accepted.numel()),
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
@@ -105,6 +175,7 @@ def update_replayssm_ring_trackers(
def reset_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
@@ -112,11 +183,62 @@ def reset_replayssm_ring_trackers(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
pad_slot_id=pad_slot_id,
)
def commit_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
prev_query_len: torch.Tensor,
state_batch_indices: torch.Tensor,
num_accepted_tokens: torch.Tensor,
query_start_loc: torch.Tensor,
logical_window: int,
ring_buffer_len: int,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Commit the preceding speculative window and record the current one.
MTP evaluates a target token and its draft tokens together, but the number
accepted from that query is available only on the next forward pass. This
function then advances the per-request ring by the accepted prefix and
records the current query length for the following pass. A zero
``prev_query_len`` means that reset/prefill left no prior MTP query to
commit; slot validity is handled independently by the kernel mask.
Standard single-token decode needs no delayed commit: its ReplaySSM kernel
advances the shared ring trackers directly after every token.
"""
if state_batch_indices.dim() > 1:
state_batch_indices = state_batch_indices[:, 0]
n_slots = state_batch_indices.numel()
if n_slots == 0:
return
block = 128
_commit_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
num_accepted_tokens,
query_start_loc,
state_batch_indices.stride(0),
n_slots,
min(
ring_start.numel(),
prev_num_accepted.numel(),
prev_query_len.numel(),
),
logical_window,
ring_buffer_len,
pad_slot_id,
BLOCK=block,
)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
@@ -385,6 +507,7 @@ def selective_state_update_replayssm_flashinfer(
dt_cache: torch.Tensor,
ring_start: torch.Tensor,
prev_num_accepted_tokens: torch.Tensor,
prev_query_len: torch.Tensor,
logical_window: int,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
@@ -395,6 +518,9 @@ def selective_state_update_replayssm_flashinfer(
update_trackers: bool = True,
enable_stochastic_rounding: bool = False,
stochastic_rounding_philox_rounds: int = 0,
cu_seqlens: torch.Tensor | None = None,
max_seqlen: int | None = None,
enable_pdl: bool = False,
) -> torch.Tensor:
"""Run FlashInfer checkpointing SSU and optionally advance shared trackers."""
if _flashinfer_replayssm_kernel is None:
@@ -404,11 +530,12 @@ def selective_state_update_replayssm_flashinfer(
)
if x.dim() == 3:
x = x.unsqueeze(1)
dt = dt.unsqueeze(1)
B = B.unsqueeze(1)
C = C.unsqueeze(1)
out = out.unsqueeze(1)
dim = 0 if cu_seqlens is not None else 1
x = x.unsqueeze(dim)
dt = dt.unsqueeze(dim)
B = B.unsqueeze(dim)
C = C.unsqueeze(dim)
out = out.unsqueeze(dim)
indices = state_batch_indices
if indices is not None and indices.dim() > 1:
@@ -443,6 +570,9 @@ def selective_state_update_replayssm_flashinfer(
pad_slot_id=null_block_id,
rand_seed=rand_seed,
philox_rounds=stochastic_rounding_philox_rounds or 10,
cu_seqlens=cu_seqlens,
max_seqlen=max_seqlen,
enable_pdl=enable_pdl,
cb_scaled=cb_scaled,
cumAdt_vec=cumAdt_vec,
cb_old=cb_old,
@@ -451,6 +581,7 @@ def selective_state_update_replayssm_flashinfer(
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted_tokens,
prev_query_len,
indices,
logical_window=logical_window,
ring_buffer_len=x_cache.size(2),
+1
View File
@@ -815,6 +815,7 @@ class NemotronHForCausalLM(
tp_world_size=parallel_config.tensor_parallel_size,
logical_window=cache_config.replayssm_buffer_len,
backend=vllm_config.mamba_config.backend,
num_speculative_tokens=vllm_config.num_speculative_tokens,
)
return base_shape
+30 -7
View File
@@ -21,6 +21,7 @@ logger = init_logger(__name__)
def _replayssm_autotune_kwargs(
runner: "GPUModelRunner",
max_token_prefill_kwargs: dict[str, Any],
) -> tuple[int, dict[str, Any]] | None:
config = runner.vllm_config
if not (
@@ -46,11 +47,16 @@ def _replayssm_autotune_kwargs(
runner.max_num_tokens // query_len,
runner.kv_cache_config.num_blocks - 1,
)
if max_num_reqs <= 0:
logger.warning_once(
"Skipping FlashInfer ReplaySSM autotuning because no non-padding "
"state slot is available."
)
return None
decode_kwargs = {
**max_token_prefill_kwargs,
"num_tokens": max_num_reqs * query_len,
"skip_eplb": True,
"is_profile": True,
"randomize_inputs": True,
"uniform_decode": True,
}
if config.use_v2_model_runner:
@@ -75,18 +81,22 @@ def _temporary_replayssm_autotune_state(
)
reset_tensors: dict[int, torch.Tensor] = {}
tracker_specs: dict[int, tuple[torch.Tensor, torch.Tensor, int, int]] = {}
tracker_specs: dict[
int, tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int]
] = {}
for module in runner.get_model().modules():
if not isinstance(module, MambaMixer2) or not module.use_replayssm:
continue
assert module.replayssm_buffer_len is not None
ring_start = module._replayssm_ring_start
prev_num_accepted = module._replayssm_prev_num_accepted
prev_query_len = module._replayssm_prev_query_len
tracker_specs.setdefault(
ring_start.data_ptr(),
(
ring_start,
prev_num_accepted,
prev_query_len,
module.replayssm_buffer_len,
module.kv_cache[2].size(2),
),
@@ -95,6 +105,7 @@ def _temporary_replayssm_autotune_state(
*module.kv_cache,
ring_start,
prev_num_accepted,
prev_query_len,
)
for tensor in tensors:
if tensor.numel():
@@ -121,20 +132,26 @@ def _temporary_replayssm_autotune_state(
for (
ring_start,
prev_num_accepted,
prev_query_len,
logical_window,
ring_buffer_len,
) in tracker_specs.values():
# Compile reset (prefill) and advance (decode) before inference.
# The final reset leaves the decode tuning run in a clean state.
reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots)
reset_replayssm_ring_trackers(
ring_start, prev_num_accepted, prev_query_len, state_slots
)
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_slots,
logical_window,
ring_buffer_len,
)
reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots)
reset_replayssm_ring_trackers(
ring_start, prev_num_accepted, prev_query_len, state_slots
)
try:
yield
@@ -151,7 +168,13 @@ def _temporary_replayssm_autotune_state(
def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None:
autotune = _replayssm_autotune_kwargs(runner)
max_token_prefill_kwargs = {
"num_tokens": runner.scheduler_config.max_num_batched_tokens,
"skip_eplb": True,
"is_profile": True,
"randomize_inputs": True,
}
autotune = _replayssm_autotune_kwargs(runner, max_token_prefill_kwargs)
if autotune is None:
return
max_num_reqs, decode_kwargs = autotune