[Mamba] Add FlashInfer ReplaySSM backend (#52506)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-01 10:26:43 -04:00
committed by GitHub
co-authored by Andrii Skliar
parent 25efcfa788
commit adebc41b7e
19 changed files with 1029 additions and 105 deletions
+148 -37
View File
@@ -2,10 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
import torch
from torch import nn
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
from vllm.config.mamba import MambaBackendEnum
from vllm.distributed import (
divide,
get_tensor_model_parallel_rank,
@@ -16,6 +19,7 @@ from vllm.distributed import (
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp, PluggableLayer
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
MergedColumnParallelLinear,
@@ -38,7 +42,11 @@ from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_outpu
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import (
LoaderFunction,
@@ -58,6 +66,7 @@ from vllm.utils.torch_utils import (
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
logger = init_logger(__name__)
@@ -515,10 +524,12 @@ class MambaMixer2(MambaBase, PluggableLayer):
raise ValueError(
"--use-replayssm requires tensor-parallel heads to divide evenly"
)
# The tuple is (conv_state, ssm_state); with the cached (ReplaySSM) decode
# kernel enabled it is (conv_state, ssm_state, x_cache, dt_cache, B_cache).
# ReplaySSM appends x/dt/B rings to (conv_state, ssm_state).
_n_state = 5 if self.use_replayssm else 2
self.kv_cache = tuple(torch.tensor([]) for _ in range(_n_state))
self._replayssm_ring_start = torch.empty(0, dtype=torch.int32)
self._replayssm_prev_num_accepted = torch.empty(0, dtype=torch.int32)
self._updates_replayssm_trackers = True
self.num_spec = vllm_config.num_speculative_tokens
if self.num_spec > 0:
@@ -706,6 +717,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
assert self.cache_config is not None
mamba_block_size = self.cache_config.mamba_block_size
is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all"
ring_start = prev_num_accepted = None
attn_metadata: AttentionMetadata | None = None
if attn_metadata_raw is not None:
@@ -723,7 +735,10 @@ class MambaMixer2(MambaBase, PluggableLayer):
)
ssm_state = self.kv_cache[1]
if self.use_replayssm:
x_cache, dt_cache, B_cache = self.kv_cache[2:]
x_cache, dt_cache, B_cache = self.kv_cache[2:5]
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
ring_start = self._replayssm_ring_start
prev_num_accepted = self._replayssm_prev_num_accepted
else:
x_cache = dt_cache = B_cache = None
has_initial_states_p = attn_metadata.has_initial_states_p
@@ -982,6 +997,13 @@ class MambaMixer2(MambaBase, PluggableLayer):
# tensor
assert state_indices_tensor_p is not None
ssm_state[state_indices_tensor_p] = varlen_states
if ring_start is not None and self._updates_replayssm_trackers:
assert prev_num_accepted is not None
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_indices_tensor_p,
)
# Process decode requests
if has_decode:
@@ -1060,35 +1082,64 @@ class MambaMixer2(MambaBase, PluggableLayer):
)
if self.use_replayssm:
assert self.replayssm_buffer_len is not None
selective_state_update_replayssm_output_only(
ssm_state,
hidden_states_d,
dt_d,
A_d,
B_d,
C_d,
D_d,
dt_bias,
dt_softplus=True,
x_cache=x_cache,
dt_cache=dt_cache,
B_cache=B_cache,
bc_pre=attn_metadata.bc_pre_scratch,
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
out=preallocated_ssm_out_d,
# Stochastic Rounding for the vanilla decode path is read
# from mamba_config inside ssu_dispatch; the replay kernel
# isn't a dispatch backend, so pass it here.
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
cache_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
)
if self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
assert ring_start is not None
assert prev_num_accepted is not None
assert attn_metadata.replayssm_scratch is not None
selective_state_update_replayssm_flashinfer(
ssm_state,
hidden_states_d,
dt_d,
A_d,
B_d,
C_d,
preallocated_ssm_out_d,
x_cache,
B_cache,
dt_cache,
ring_start,
prev_num_accepted,
logical_window=self.replayssm_buffer_len,
D=D_d,
dt_bias=dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
scratch=attn_metadata.replayssm_scratch,
update_trackers=self._updates_replayssm_trackers,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
stochastic_rounding_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
)
else:
selective_state_update_replayssm_output_only(
ssm_state,
hidden_states_d,
dt_d,
A_d,
B_d,
C_d,
D_d,
dt_bias,
dt_softplus=True,
x_cache=x_cache,
dt_cache=dt_cache,
B_cache=B_cache,
bc_pre=attn_metadata.bc_pre_scratch,
write_pos=attn_metadata.write_pos_d,
is_flush=attn_metadata.is_flush_d,
max_cache_len=self.replayssm_buffer_len,
state_batch_indices=state_indices_tensor_d_input,
out=preallocated_ssm_out_d,
enable_stochastic_rounding=(
self.mamba_config.enable_stochastic_rounding
),
cache_philox_rounds=(
self.mamba_config.stochastic_rounding_philox_rounds
),
)
else:
selective_state_update(
ssm_state,
@@ -1137,10 +1188,11 @@ class MambaMixer2(MambaBase, PluggableLayer):
if self.use_replayssm:
assert self.replayssm_buffer_len is not None
return MambaStateShapeCalculator.append_replayssm_ring(
base_shape,
self.n_groups,
tp_world_size,
self.replayssm_buffer_len,
base_shapes=base_shape,
n_groups=self.n_groups,
tp_world_size=tp_world_size,
logical_window=self.replayssm_buffer_len,
backend=self.mamba_config.backend,
)
return base_shape
@@ -1149,6 +1201,65 @@ class MambaMixer2(MambaBase, PluggableLayer):
return MambaAttentionBackendEnum.MAMBA2
def share_replayssm_ring_trackers(
ordered_layer_names: list[str],
forward_context: dict[str, Attention],
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
) -> None:
"""Share ring cursors within each cache-slot index namespace.
Layers backed by one KV-cache group use the same physical block indices and
can therefore share cursors. Different KV-cache groups may assign different
block indices to the same request and must keep separate cursor tensors.
The final local layer in each group advances its cursors after every layer
in that group has consumed the previous values.
"""
replayssm_mixers: dict[str, MambaMixer2] = {}
for layer_name in ordered_layer_names:
layer = forward_context[layer_name]
if (
isinstance(layer, MambaMixer2)
and layer.use_replayssm
and layer.mamba_config.backend == MambaBackendEnum.FLASHINFER
):
replayssm_mixers[layer_name] = layer
layer_to_group: dict[str, int] = {}
if kv_cache_groups:
for group_idx, group in enumerate(kv_cache_groups):
for layer_name in group.layer_names:
layer_to_group[layer_name] = group_idx
groups_by_namespace: dict[int | str, list[str]] = {}
for layer_name in ordered_layer_names:
if layer_name not in replayssm_mixers:
continue
namespace = layer_to_group.get(layer_name, layer_name)
groups_by_namespace.setdefault(namespace, []).append(layer_name)
for group_layer_names in groups_by_namespace.values():
last_layer_name = group_layer_names[-1]
first_mixer = replayssm_mixers[group_layer_names[0]]
first_state = first_mixer.kv_cache[1]
num_blocks, device = first_state.shape[0], first_state.device
for layer_name in group_layer_names:
state = replayssm_mixers[layer_name].kv_cache[1]
if (state.shape[0], state.device) != (num_blocks, device):
raise ValueError(
"ReplaySSM layers in one cache group must share cache capacity"
)
ring_start = torch.zeros(num_blocks, dtype=torch.int32, device=device)
prev_num_accepted = torch.zeros_like(ring_start)
for layer_name in group_layer_names:
mixer = replayssm_mixers[layer_name]
mixer._replayssm_ring_start = ring_start
mixer._replayssm_prev_num_accepted = prev_num_accepted
mixer._updates_replayssm_trackers = layer_name == last_layer_name
def mamba_mixer2(
projected_states: torch.Tensor,
output: torch.Tensor,
@@ -10,6 +10,7 @@ import torch
import vllm.envs as envs
from vllm.config.cache import MambaDType
from vllm.config.mamba import MambaBackendEnum
from vllm.config.model import ModelDType
from vllm.distributed import divide
from vllm.logger import init_logger
@@ -214,20 +215,25 @@ class MambaStateShapeCalculator:
base_shapes: tuple[tuple[int, ...], ...],
n_groups: int,
tp_world_size: int,
replayssm_buffer_len: int,
logical_window: int,
backend: MambaBackendEnum,
) -> tuple[tuple[int, ...], ...]:
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
``n_groups``.
"""Append the physical ReplaySSM ring shapes.
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
B_cache uses the un-extended ``n_groups``.
"""
ring_buffer_len = logical_window
if backend == MambaBackendEnum.FLASHINFER:
# FlashInfer keeps the live window and appended token together.
ring_buffer_len += 1
local_nheads, head_dim, state_size = base_shapes[1]
local_ngroups = divide(n_groups, tp_world_size)
return (
*base_shapes,
(local_nheads, replayssm_buffer_len, head_dim),
(local_nheads, replayssm_buffer_len),
(local_ngroups, replayssm_buffer_len, state_size),
(local_nheads, ring_buffer_len, head_dim),
(local_nheads, ring_buffer_len),
(local_ngroups, ring_buffer_len, state_size),
)
@classmethod
@@ -10,11 +10,14 @@ the backend defaults to 'cpu'.
"""
from abc import ABC, abstractmethod
from collections.abc import Callable
from functools import cache
import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
@@ -22,6 +25,99 @@ from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
logger = init_logger(__name__)
@triton.jit(
do_not_specialize=["n_slots", "state_batch_indices_stride"],
do_not_specialize_on_alignment=["state_batch_indices"],
)
def _update_replayssm_ring_trackers_kernel(
ring_start,
prev_num_accepted,
state_batch_indices,
state_batch_indices_stride,
n_slots,
num_states,
logical_window: tl.constexpr,
ring_buffer_len: tl.constexpr,
pad_slot_id: tl.constexpr,
RESET: tl.constexpr,
BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < n_slots
slots = tl.load(
state_batch_indices + offsets * state_batch_indices_stride,
mask=mask,
other=pad_slot_id,
)
valid = mask & (slots != pad_slot_id) & (slots >= 0) & (slots < num_states)
if RESET:
tl.store(ring_start + slots, 0, mask=valid)
tl.store(prev_num_accepted + slots, 0, mask=valid)
else:
prev = tl.load(prev_num_accepted + slots, mask=valid, other=0)
start = tl.load(ring_start + slots, mask=valid, other=0)
must_checkpoint = prev + 1 > logical_window
next_start = tl.where(
must_checkpoint,
(start + prev) % ring_buffer_len,
start,
)
next_prev = tl.where(must_checkpoint, 1, prev + 1)
tl.store(ring_start + slots, next_start, mask=valid)
tl.store(prev_num_accepted + slots, next_prev, mask=valid)
def update_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
state_batch_indices: torch.Tensor,
logical_window: int | None = None,
ring_buffer_len: int | None = None,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Reset selected trackers, or advance them when a window is provided."""
if state_batch_indices.dim() > 1:
state_batch_indices = state_batch_indices[:, 0]
n_slots = state_batch_indices.numel()
if n_slots == 0:
return
reset = logical_window is None
if reset:
logical_window = 0
ring_buffer_len = 1
else:
assert ring_buffer_len is not None
block = 128
_update_replayssm_ring_trackers_kernel[(triton.cdiv(n_slots, block),)](
ring_start,
prev_num_accepted,
state_batch_indices,
state_batch_indices.stride(0),
n_slots,
min(ring_start.numel(), prev_num_accepted.numel()),
logical_window,
ring_buffer_len,
pad_slot_id,
RESET=reset,
BLOCK=block,
)
def reset_replayssm_ring_trackers(
ring_start: torch.Tensor,
prev_num_accepted: torch.Tensor,
state_batch_indices: torch.Tensor,
pad_slot_id: int = NULL_BLOCK_ID,
) -> None:
"""Reset selected ReplaySSM ring trackers."""
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_batch_indices,
pad_slot_id=pad_slot_id,
)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
@@ -262,15 +358,115 @@ _BACKEND_REGISTRY: dict[MambaBackendEnum, type[MambaSSUBackend]] = {
_mamba_ssu_backend: MambaSSUBackend | None = None
_flashinfer_replayssm_kernel: Callable[..., torch.Tensor] | None = None
@cache
def flashinfer_replayssm_autotune_supported() -> bool:
"""Return True when FlashInfer exposes ReplaySSM autotuning."""
try:
from flashinfer.mamba.checkpointing_ssu import ( # noqa: F401
CheckpointingSSURunner,
)
except ImportError:
return False
return True
def selective_state_update_replayssm_flashinfer(
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
out: torch.Tensor,
x_cache: torch.Tensor,
B_cache: torch.Tensor,
dt_cache: torch.Tensor,
ring_start: torch.Tensor,
prev_num_accepted_tokens: torch.Tensor,
logical_window: int,
D: torch.Tensor | None = None,
dt_bias: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
scratch: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
update_trackers: bool = True,
enable_stochastic_rounding: bool = False,
stochastic_rounding_philox_rounds: int = 0,
) -> torch.Tensor:
"""Run FlashInfer checkpointing SSU and optionally advance shared trackers."""
if _flashinfer_replayssm_kernel is None:
raise RuntimeError(
"FlashInfer ReplaySSM has not been initialized. "
"Call initialize_mamba_ssu_backend() with use_replayssm=True."
)
if x.dim() == 3:
x = x.unsqueeze(1)
dt = dt.unsqueeze(1)
B = B.unsqueeze(1)
C = C.unsqueeze(1)
out = out.unsqueeze(1)
indices = state_batch_indices
if indices is not None and indices.dim() > 1:
indices = indices[:, 0]
cb_scaled = cumAdt_vec = cb_old = None
if scratch is not None:
cb_scaled, cumAdt_vec, cb_old = scratch
rand_seed = (
torch.randint(0, 2**32, (1,), device=state.device, dtype=torch.int64)
if enable_stochastic_rounding
else None
)
result = _flashinfer_replayssm_kernel(
state,
x_cache,
B_cache,
dt_cache,
ring_start,
prev_num_accepted_tokens,
x,
dt,
A,
B,
C,
out,
D=D,
dt_bias=dt_bias,
dt_softplus=dt_softplus,
state_batch_indices=indices,
pad_slot_id=null_block_id,
rand_seed=rand_seed,
philox_rounds=stochastic_rounding_philox_rounds or 10,
cb_scaled=cb_scaled,
cumAdt_vec=cumAdt_vec,
cb_old=cb_old,
)
if update_trackers and indices is not None:
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted_tokens,
indices,
logical_window=logical_window,
ring_buffer_len=x_cache.size(2),
pad_slot_id=null_block_id,
)
return result
def initialize_mamba_ssu_backend(
mamba_config: MambaConfig,
kv_cache_config: KVCacheConfig,
*,
use_replayssm: bool = False,
) -> None:
"""Initialize the global Mamba SSU backend.
No-op if `kv_cache_config` contains no specs that call
selective_state_update.
"""
"""Initialize the Mamba SSU backend and optional FlashInfer ReplaySSM."""
if not any(
isinstance(g.kv_cache_spec, MambaSpec)
and g.kv_cache_spec.mamba_type
@@ -279,13 +475,9 @@ def initialize_mamba_ssu_backend(
):
return
global _mamba_ssu_backend
global _flashinfer_replayssm_kernel, _mamba_ssu_backend
backend = mamba_config.backend
# On CPU-only platforms (PowerPC, x86 without CUDA) Triton JIT is
# unstable or unavailable. Silently fall back to the CPU
# backend unless the user explicitly chose something other than "triton".
if backend == MambaBackendEnum.TRITON:
from vllm.platforms import current_platform
@@ -301,13 +493,28 @@ def initialize_mamba_ssu_backend(
f"Unknown Mamba SSU backend: {backend}. "
f"Valid options: {list(_BACKEND_REGISTRY.keys())}"
)
if use_replayssm and backend not in (
MambaBackendEnum.TRITON,
MambaBackendEnum.FLASHINFER,
):
raise ValueError(f"ReplaySSM does not support mamba backend {backend.value!r}")
backend_cls = _BACKEND_REGISTRY[backend]
if isinstance(_mamba_ssu_backend, backend_cls):
return
if not isinstance(_mamba_ssu_backend, backend_cls):
_mamba_ssu_backend = backend_cls(mamba_config)
logger.info("Using %s Mamba SSU backend.", _mamba_ssu_backend.name)
_mamba_ssu_backend = backend_cls(mamba_config)
logger.info("Using %s Mamba SSU backend.", _mamba_ssu_backend.name)
_flashinfer_replayssm_kernel = None
if use_replayssm and backend == MambaBackendEnum.FLASHINFER:
try:
from flashinfer.mamba.checkpointing_ssu import checkpointing_ssu
except ImportError as e:
raise ImportError(
"FlashInfer ReplaySSM requires a compatible flashinfer-python package"
) from e
_flashinfer_replayssm_kernel = checkpointing_ssu
if use_replayssm:
logger.info("Using %s ReplaySSM backend.", backend.value)
def get_mamba_ssu_backend() -> MambaSSUBackend:
+7 -5
View File
@@ -747,7 +747,8 @@ class NemotronHForCausalLM(
)
if cache_config.use_replayssm:
return MambaStateDtypeCalculator.append_replayssm_ring(
base_dtype, vllm_config.model_config.dtype
base_dtype,
vllm_config.model_config.dtype,
)
return base_dtype
@@ -784,10 +785,11 @@ class NemotronHForCausalLM(
)
if cache_config.use_replayssm:
return MambaStateShapeCalculator.append_replayssm_ring(
base_shape,
hf_config.n_groups,
parallel_config.tensor_parallel_size,
cache_config.replayssm_buffer_len,
base_shapes=base_shape,
n_groups=hf_config.n_groups,
tp_world_size=parallel_config.tensor_parallel_size,
logical_window=cache_config.replayssm_buffer_len,
backend=vllm_config.mamba_config.backend,
)
return base_shape
@@ -38,6 +38,9 @@ from vllm.model_executor.warmup.kimi_k3_triton_warmup import (
kimi_k3_triton_warmup,
)
from vllm.model_executor.warmup.qwen_triton_warmup import qwen_triton_warmup
from vllm.model_executor.warmup.replayssm_warmup import (
replayssm_autotune_warmup,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from vllm.utils.flashinfer import has_flashinfer
@@ -335,6 +338,7 @@ def flashinfer_autotune(runner: "GPUModelRunner") -> None:
fi_utils.autotune(tune_mode=True, **autotune_kwargs),
):
_run_flashinfer_autotune_dummy_runs(runner)
replayssm_autotune_warmup(runner)
finally:
set_autotune_process_group(None)
@@ -0,0 +1,159 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any
import torch
from vllm.config.mamba import MambaBackendEnum
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
flashinfer_replayssm_autotune_supported,
)
if TYPE_CHECKING:
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
logger = init_logger(__name__)
def _replayssm_autotune_kwargs(
runner: "GPUModelRunner",
) -> tuple[int, dict[str, Any]] | None:
config = runner.vllm_config
if not (
config.cache_config.use_replayssm
and config.mamba_config.backend == MambaBackendEnum.FLASHINFER
):
return None
if not flashinfer_replayssm_autotune_supported():
logger.info_once(
"Skipping FlashInfer ReplaySSM autotuning because "
"flashinfer.mamba.checkpointing_ssu.CheckpointingSSURunner "
"is unavailable."
)
return None
v2_runner: Any = runner
query_len = (
v2_runner.decode_query_len
if config.use_v2_model_runner
else runner.uniform_decode_query_len
)
max_num_reqs = min(
runner.scheduler_config.max_num_seqs,
runner.max_num_tokens // query_len,
runner.kv_cache_config.num_blocks - 1,
)
decode_kwargs = {
"num_tokens": max_num_reqs * query_len,
"skip_eplb": True,
"is_profile": True,
"randomize_inputs": True,
"uniform_decode": True,
}
if config.use_v2_model_runner:
decode_kwargs["valid_dummy_state_slots"] = True
else:
decode_kwargs.update(
allow_microbatching=False,
force_attention=True,
profile_seq_lens=query_len + 1,
)
return max_num_reqs, decode_kwargs
@contextmanager
def _temporary_replayssm_autotune_state(
runner: "GPUModelRunner", max_num_reqs: int
) -> Iterator[None]:
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
reset_replayssm_ring_trackers,
update_replayssm_ring_trackers,
)
reset_tensors: dict[int, torch.Tensor] = {}
tracker_specs: dict[int, tuple[torch.Tensor, torch.Tensor, int, int]] = {}
for module in runner.get_model().modules():
if not isinstance(module, MambaMixer2) or not module.use_replayssm:
continue
assert module.replayssm_buffer_len is not None
ring_start = module._replayssm_ring_start
prev_num_accepted = module._replayssm_prev_num_accepted
tracker_specs.setdefault(
ring_start.data_ptr(),
(
ring_start,
prev_num_accepted,
module.replayssm_buffer_len,
module.kv_cache[2].size(2),
),
)
tensors = (
*module.kv_cache,
ring_start,
prev_num_accepted,
)
for tensor in tensors:
if tensor.numel():
reset_tensors.setdefault(tensor.data_ptr(), tensor)
v2_runner: Any = runner
block_tables = saved_block_ids = None
if not runner.vllm_config.use_v2_model_runner:
block_tables = runner.input_batch.block_table.block_tables
saved_block_ids = tuple(
block_table.block_table.np[:max_num_reqs, 0].copy()
for block_table in block_tables
)
dummy_block_ids = range(1, max_num_reqs + 1)
for block_table in block_tables:
block_table.block_table.np[:max_num_reqs, 0] = dummy_block_ids
runner.input_batch.block_table.commit_block_table(max_num_reqs)
first_tracker = next(iter(tracker_specs.values()), None)
if first_tracker is not None and first_tracker[0].is_cuda:
state_slots = torch.arange(
1, max_num_reqs + 1, dtype=torch.int32, device=first_tracker[0].device
)
for (
ring_start,
prev_num_accepted,
logical_window,
ring_buffer_len,
) in tracker_specs.values():
# Compile reset (prefill) and advance (decode) before inference.
# The final reset leaves the decode tuning run in a clean state.
reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots)
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
state_slots,
logical_window,
ring_buffer_len,
)
reset_replayssm_ring_trackers(ring_start, prev_num_accepted, state_slots)
try:
yield
finally:
if runner.vllm_config.use_v2_model_runner:
v2_runner.block_tables.get_dummy_block_tables(max_num_reqs)
else:
assert block_tables is not None and saved_block_ids is not None
for block_table, block_ids in zip(block_tables, saved_block_ids):
block_table.block_table.np[:max_num_reqs, 0] = block_ids
runner.input_batch.block_table.commit_block_table(max_num_reqs)
for tensor in reset_tensors.values():
tensor[1 : max_num_reqs + 1].zero_()
def replayssm_autotune_warmup(runner: "GPUModelRunner") -> None:
autotune = _replayssm_autotune_kwargs(runner)
if autotune is None:
return
max_num_reqs, decode_kwargs = autotune
with _temporary_replayssm_autotune_state(runner, max_num_reqs):
runner._dummy_run(**decode_kwargs)