[Mamba] Flashinfer selective_state_update (#36162)

Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
roikoren755
2026-04-14 15:10:58 -04:00
committed by GitHub
parent 30679319e8
commit ecf5ff7ce3
15 changed files with 516 additions and 71 deletions
@@ -30,10 +30,8 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
selective_scan_fn,
selective_state_update,
)
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import (
@@ -431,14 +429,12 @@ class MambaMixer(MambaBase, PluggableLayer):
B_d,
C_d,
self.D,
gate_d.transpose(0, 1),
time_proj_bias,
z=gate_d.transpose(0, 1),
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
out=scan_outputs_d,
enable_stochastic_rounding=self.cache_config.enable_mamba_cache_stochastic_rounding,
cache_philox_rounds=self.cache_config.mamba_cache_philox_rounds,
)
scan_outputs_d = scan_outputs_d.transpose(0, 1)
@@ -31,10 +31,10 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import (
LoaderFunction,
@@ -890,8 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
B_d,
C_d,
D_d,
z=None,
dt_bias=dt_bias,
dt_bias,
dt_softplus=True,
state_batch_indices=state_indices_tensor_d_input,
dst_state_batch_indices=state_indices_tensor_d_output,
@@ -899,8 +898,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=query_start_loc_d,
is_blackwell=self.is_blackwell,
enable_stochastic_rounding=self.cache_config.enable_mamba_cache_stochastic_rounding,
cache_philox_rounds=self.cache_config.mamba_cache_philox_rounds,
)
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
@@ -323,9 +323,9 @@ def selective_state_update(
A,
B,
C,
D=None,
D,
dt_bias,
z=None,
dt_bias=None,
dt_softplus=False,
state_batch_indices=None,
dst_state_batch_indices=None,
@@ -374,11 +374,11 @@ def selective_state_update(
B = B.unsqueeze(1)
if C.dim() == 2:
C = C.unsqueeze(1)
if D is not None and D.dim() == 1:
if D.dim() == 1:
D = D.unsqueeze(0)
if z is not None and z.dim() == 2:
z = z.unsqueeze(1)
if dt_bias is not None and dt_bias.dim() == 1:
if dt_bias.dim() == 1:
dt_bias = dt_bias.unsqueeze(0)
if out.dim() == 2:
out = out.unsqueeze(1)
@@ -410,12 +410,10 @@ def selective_state_update(
assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
assert B.shape == (batch, ngroups, dstate)
assert C.shape == B.shape
if D is not None:
assert D.shape == (nheads, dim)
assert D.shape == (nheads, dim)
if z is not None:
assert z.shape == x.shape
if dt_bias is not None:
assert dt_bias.shape == (nheads, dim)
assert dt_bias.shape == (nheads, dim)
if state_batch_indices is not None:
assert state_batch_indices.shape[0] >= N
assert state_batch_indices.shape[1] >= max_seqlen
@@ -506,7 +504,8 @@ def selective_state_update(
dt.stride(0),
dt.stride(1),
dt.stride(2),
*(dt_bias.stride(0), dt_bias.stride(1)) if dt_bias is not None else 0,
dt_bias.stride(0),
dt_bias.stride(1),
A.stride(0),
A.stride(1),
A.stride(2),
@@ -516,7 +515,8 @@ def selective_state_update(
C.stride(0),
C.stride(1),
C.stride(2),
*(D.stride(0), D.stride(1)) if D is not None else 0,
D.stride(0),
D.stride(1),
z_strides[0],
z_strides[1],
z_strides[2],
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Dispatch module for Mamba selective state update (SSU) backends.
Provides a unified `selective_state_update` function that dispatches to
either the Triton or FlashInfer backend based on the configured
`MambaBackendEnum`. Follows SGLang's dispatch pattern adapted for vLLM.
"""
from abc import ABC, abstractmethod
import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig
from vllm.logger import init_logger
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
logger = init_logger(__name__)
class MambaSSUBackend(ABC):
"""Abstract base class for Mamba SSU backends."""
def __init__(self, mamba_config: MambaConfig):
self._mamba_config = mamba_config
@property
@abstractmethod
def name(self) -> str: ...
@abstractmethod
def __call__(
self,
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D: torch.Tensor,
dt_bias: torch.Tensor,
z: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
dst_state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
out: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
cu_seqlens: torch.Tensor | None = None,
is_blackwell: bool = False,
) -> None: ...
class TritonSSUBackend(MambaSSUBackend):
"""Triton-based SSU backend (vLLM's default)."""
def __init__(self, mamba_config: MambaConfig):
super().__init__(mamba_config)
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
selective_state_update as _triton_selective_state_update,
)
self._kernel = _triton_selective_state_update
@property
def name(self) -> str:
return "triton"
def __call__(
self,
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D: torch.Tensor,
dt_bias: torch.Tensor,
z: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
dst_state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
out: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
cu_seqlens: torch.Tensor | None = None,
is_blackwell: bool = False,
) -> None:
self._kernel(
state,
x,
dt,
A,
B,
C,
D=D,
z=z,
dt_bias=dt_bias,
dt_softplus=dt_softplus,
state_batch_indices=state_batch_indices,
dst_state_batch_indices=dst_state_batch_indices,
null_block_id=null_block_id,
out=out,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=cu_seqlens,
is_blackwell=is_blackwell,
enable_stochastic_rounding=self._mamba_config.enable_stochastic_rounding,
cache_philox_rounds=self._mamba_config.stochastic_rounding_philox_rounds,
)
class FlashInferSSUBackend(MambaSSUBackend):
"""FlashInfer-based SSU backend."""
def __init__(self, mamba_config: MambaConfig):
super().__init__(mamba_config)
try:
from flashinfer.mamba import selective_state_update as _fi_ssu
except ImportError as e:
raise ImportError(
"FlashInfer is required for the flashinfer Mamba SSU backend. "
"Please install flashinfer (>= 0.6.4): "
"pip install flashinfer-python"
) from e
self._kernel = _fi_ssu
@property
def name(self) -> str:
return "flashinfer"
def __call__(
self,
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D: torch.Tensor,
dt_bias: torch.Tensor,
z: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
dst_state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
out: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
cu_seqlens: torch.Tensor | None = None,
is_blackwell: bool = False,
) -> None:
rand_seed = (
torch.randint(0, 2**32, (1,), device=state.device)
if self._mamba_config.enable_stochastic_rounding
else None
)
self._kernel(
state,
x,
dt,
A,
B,
C,
D=D,
z=z,
dt_bias=dt_bias,
dt_softplus=dt_softplus,
state_batch_indices=state_batch_indices,
dst_state_batch_indices=dst_state_batch_indices,
cu_seqlens=cu_seqlens,
num_accepted_tokens=num_accepted_tokens,
cache_steps=state_batch_indices.size(-1)
if cu_seqlens is not None and state_batch_indices is not None
else 0,
pad_slot_id=null_block_id,
out=out,
rand_seed=rand_seed,
philox_rounds=self._mamba_config.stochastic_rounding_philox_rounds or 10,
)
_BACKEND_REGISTRY: dict[MambaBackendEnum, type[MambaSSUBackend]] = {
MambaBackendEnum.TRITON: TritonSSUBackend,
MambaBackendEnum.FLASHINFER: FlashInferSSUBackend,
}
_mamba_ssu_backend: MambaSSUBackend | None = None
def initialize_mamba_ssu_backend(mamba_config: MambaConfig) -> None:
"""Initialize the global Mamba SSU backend.
Args:
mamba_config: Mamba configuration.
"""
global _mamba_ssu_backend
backend = mamba_config.backend
if backend not in _BACKEND_REGISTRY:
raise ValueError(
f"Unknown Mamba SSU backend: {backend}. "
f"Valid options: {list(_BACKEND_REGISTRY.keys())}"
)
_mamba_ssu_backend = _BACKEND_REGISTRY[backend](mamba_config)
logger.info("Using %s Mamba SSU backend.", _mamba_ssu_backend.name)
def get_mamba_ssu_backend() -> MambaSSUBackend:
"""Get the current Mamba SSU backend. Raises if not initialized."""
if _mamba_ssu_backend is None:
raise RuntimeError(
"Mamba SSU backend has not been initialized. "
"Call initialize_mamba_ssu_backend() first."
)
return _mamba_ssu_backend
def selective_state_update(
state: torch.Tensor,
x: torch.Tensor,
dt: torch.Tensor,
A: torch.Tensor,
B: torch.Tensor,
C: torch.Tensor,
D: torch.Tensor,
dt_bias: torch.Tensor,
z: torch.Tensor | None = None,
dt_softplus: bool = False,
state_batch_indices: torch.Tensor | None = None,
dst_state_batch_indices: torch.Tensor | None = None,
null_block_id: int = NULL_BLOCK_ID,
out: torch.Tensor | None = None,
num_accepted_tokens: torch.Tensor | None = None,
cu_seqlens: torch.Tensor | None = None,
is_blackwell: bool = False,
) -> None:
"""Unified dispatch for Mamba selective state update.
Delegates to the initialized backend (Triton or FlashInfer).
"""
get_mamba_ssu_backend()(
state,
x,
dt,
A,
B,
C,
D,
dt_bias,
z=z,
dt_softplus=dt_softplus,
state_batch_indices=state_batch_indices,
dst_state_batch_indices=dst_state_batch_indices,
null_block_id=null_block_id,
out=out,
num_accepted_tokens=num_accepted_tokens,
cu_seqlens=cu_seqlens,
is_blackwell=is_blackwell,
)