[Mamba] Flashinfer selective_state_update (#36162)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
@@ -30,10 +30,8 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
|
||||
selective_scan_fn,
|
||||
selective_state_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_scan_fn
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -431,14 +429,12 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
B_d,
|
||||
C_d,
|
||||
self.D,
|
||||
gate_d.transpose(0, 1),
|
||||
time_proj_bias,
|
||||
z=gate_d.transpose(0, 1),
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
dst_state_batch_indices=state_indices_tensor_d_output,
|
||||
out=scan_outputs_d,
|
||||
enable_stochastic_rounding=self.cache_config.enable_mamba_cache_stochastic_rounding,
|
||||
cache_philox_rounds=self.cache_config.mamba_cache_philox_rounds,
|
||||
)
|
||||
scan_outputs_d = scan_outputs_d.transpose(0, 1)
|
||||
|
||||
|
||||
@@ -31,10 +31,10 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
LoaderFunction,
|
||||
@@ -890,8 +890,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
B_d,
|
||||
C_d,
|
||||
D_d,
|
||||
z=None,
|
||||
dt_bias=dt_bias,
|
||||
dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d_input,
|
||||
dst_state_batch_indices=state_indices_tensor_d_output,
|
||||
@@ -899,8 +898,6 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cu_seqlens=query_start_loc_d,
|
||||
is_blackwell=self.is_blackwell,
|
||||
enable_stochastic_rounding=self.cache_config.enable_mamba_cache_stochastic_rounding,
|
||||
cache_philox_rounds=self.cache_config.mamba_cache_philox_rounds,
|
||||
)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
|
||||
|
||||
@@ -323,9 +323,9 @@ def selective_state_update(
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=None,
|
||||
D,
|
||||
dt_bias,
|
||||
z=None,
|
||||
dt_bias=None,
|
||||
dt_softplus=False,
|
||||
state_batch_indices=None,
|
||||
dst_state_batch_indices=None,
|
||||
@@ -374,11 +374,11 @@ def selective_state_update(
|
||||
B = B.unsqueeze(1)
|
||||
if C.dim() == 2:
|
||||
C = C.unsqueeze(1)
|
||||
if D is not None and D.dim() == 1:
|
||||
if D.dim() == 1:
|
||||
D = D.unsqueeze(0)
|
||||
if z is not None and z.dim() == 2:
|
||||
z = z.unsqueeze(1)
|
||||
if dt_bias is not None and dt_bias.dim() == 1:
|
||||
if dt_bias.dim() == 1:
|
||||
dt_bias = dt_bias.unsqueeze(0)
|
||||
if out.dim() == 2:
|
||||
out = out.unsqueeze(1)
|
||||
@@ -410,12 +410,10 @@ def selective_state_update(
|
||||
assert nheads % ngroups == 0, "nheads must be divisible by ngroups"
|
||||
assert B.shape == (batch, ngroups, dstate)
|
||||
assert C.shape == B.shape
|
||||
if D is not None:
|
||||
assert D.shape == (nheads, dim)
|
||||
assert D.shape == (nheads, dim)
|
||||
if z is not None:
|
||||
assert z.shape == x.shape
|
||||
if dt_bias is not None:
|
||||
assert dt_bias.shape == (nheads, dim)
|
||||
assert dt_bias.shape == (nheads, dim)
|
||||
if state_batch_indices is not None:
|
||||
assert state_batch_indices.shape[0] >= N
|
||||
assert state_batch_indices.shape[1] >= max_seqlen
|
||||
@@ -506,7 +504,8 @@ def selective_state_update(
|
||||
dt.stride(0),
|
||||
dt.stride(1),
|
||||
dt.stride(2),
|
||||
*(dt_bias.stride(0), dt_bias.stride(1)) if dt_bias is not None else 0,
|
||||
dt_bias.stride(0),
|
||||
dt_bias.stride(1),
|
||||
A.stride(0),
|
||||
A.stride(1),
|
||||
A.stride(2),
|
||||
@@ -516,7 +515,8 @@ def selective_state_update(
|
||||
C.stride(0),
|
||||
C.stride(1),
|
||||
C.stride(2),
|
||||
*(D.stride(0), D.stride(1)) if D is not None else 0,
|
||||
D.stride(0),
|
||||
D.stride(1),
|
||||
z_strides[0],
|
||||
z_strides[1],
|
||||
z_strides[2],
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Dispatch module for Mamba selective state update (SSU) backends.
|
||||
|
||||
Provides a unified `selective_state_update` function that dispatches to
|
||||
either the Triton or FlashInfer backend based on the configured
|
||||
`MambaBackendEnum`. Follows SGLang's dispatch pattern adapted for vLLM.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MambaSSUBackend(ABC):
|
||||
"""Abstract base class for Mamba SSU backends."""
|
||||
|
||||
def __init__(self, mamba_config: MambaConfig):
|
||||
self._mamba_config = mamba_config
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def name(self) -> str: ...
|
||||
|
||||
@abstractmethod
|
||||
def __call__(
|
||||
self,
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
state_batch_indices: torch.Tensor | None = None,
|
||||
dst_state_batch_indices: torch.Tensor | None = None,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
out: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
is_blackwell: bool = False,
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class TritonSSUBackend(MambaSSUBackend):
|
||||
"""Triton-based SSU backend (vLLM's default)."""
|
||||
|
||||
def __init__(self, mamba_config: MambaConfig):
|
||||
super().__init__(mamba_config)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
|
||||
selective_state_update as _triton_selective_state_update,
|
||||
)
|
||||
|
||||
self._kernel = _triton_selective_state_update
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "triton"
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
state_batch_indices: torch.Tensor | None = None,
|
||||
dst_state_batch_indices: torch.Tensor | None = None,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
out: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
is_blackwell: bool = False,
|
||||
) -> None:
|
||||
self._kernel(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=dt_softplus,
|
||||
state_batch_indices=state_batch_indices,
|
||||
dst_state_batch_indices=dst_state_batch_indices,
|
||||
null_block_id=null_block_id,
|
||||
out=out,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cu_seqlens=cu_seqlens,
|
||||
is_blackwell=is_blackwell,
|
||||
enable_stochastic_rounding=self._mamba_config.enable_stochastic_rounding,
|
||||
cache_philox_rounds=self._mamba_config.stochastic_rounding_philox_rounds,
|
||||
)
|
||||
|
||||
|
||||
class FlashInferSSUBackend(MambaSSUBackend):
|
||||
"""FlashInfer-based SSU backend."""
|
||||
|
||||
def __init__(self, mamba_config: MambaConfig):
|
||||
super().__init__(mamba_config)
|
||||
try:
|
||||
from flashinfer.mamba import selective_state_update as _fi_ssu
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"FlashInfer is required for the flashinfer Mamba SSU backend. "
|
||||
"Please install flashinfer (>= 0.6.4): "
|
||||
"pip install flashinfer-python"
|
||||
) from e
|
||||
self._kernel = _fi_ssu
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "flashinfer"
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
state_batch_indices: torch.Tensor | None = None,
|
||||
dst_state_batch_indices: torch.Tensor | None = None,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
out: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
is_blackwell: bool = False,
|
||||
) -> None:
|
||||
rand_seed = (
|
||||
torch.randint(0, 2**32, (1,), device=state.device)
|
||||
if self._mamba_config.enable_stochastic_rounding
|
||||
else None
|
||||
)
|
||||
|
||||
self._kernel(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D=D,
|
||||
z=z,
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=dt_softplus,
|
||||
state_batch_indices=state_batch_indices,
|
||||
dst_state_batch_indices=dst_state_batch_indices,
|
||||
cu_seqlens=cu_seqlens,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cache_steps=state_batch_indices.size(-1)
|
||||
if cu_seqlens is not None and state_batch_indices is not None
|
||||
else 0,
|
||||
pad_slot_id=null_block_id,
|
||||
out=out,
|
||||
rand_seed=rand_seed,
|
||||
philox_rounds=self._mamba_config.stochastic_rounding_philox_rounds or 10,
|
||||
)
|
||||
|
||||
|
||||
_BACKEND_REGISTRY: dict[MambaBackendEnum, type[MambaSSUBackend]] = {
|
||||
MambaBackendEnum.TRITON: TritonSSUBackend,
|
||||
MambaBackendEnum.FLASHINFER: FlashInferSSUBackend,
|
||||
}
|
||||
|
||||
_mamba_ssu_backend: MambaSSUBackend | None = None
|
||||
|
||||
|
||||
def initialize_mamba_ssu_backend(mamba_config: MambaConfig) -> None:
|
||||
"""Initialize the global Mamba SSU backend.
|
||||
|
||||
Args:
|
||||
mamba_config: Mamba configuration.
|
||||
"""
|
||||
global _mamba_ssu_backend
|
||||
|
||||
backend = mamba_config.backend
|
||||
if backend not in _BACKEND_REGISTRY:
|
||||
raise ValueError(
|
||||
f"Unknown Mamba SSU backend: {backend}. "
|
||||
f"Valid options: {list(_BACKEND_REGISTRY.keys())}"
|
||||
)
|
||||
|
||||
_mamba_ssu_backend = _BACKEND_REGISTRY[backend](mamba_config)
|
||||
logger.info("Using %s Mamba SSU backend.", _mamba_ssu_backend.name)
|
||||
|
||||
|
||||
def get_mamba_ssu_backend() -> MambaSSUBackend:
|
||||
"""Get the current Mamba SSU backend. Raises if not initialized."""
|
||||
if _mamba_ssu_backend is None:
|
||||
raise RuntimeError(
|
||||
"Mamba SSU backend has not been initialized. "
|
||||
"Call initialize_mamba_ssu_backend() first."
|
||||
)
|
||||
return _mamba_ssu_backend
|
||||
|
||||
|
||||
def selective_state_update(
|
||||
state: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
dt: torch.Tensor,
|
||||
A: torch.Tensor,
|
||||
B: torch.Tensor,
|
||||
C: torch.Tensor,
|
||||
D: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
z: torch.Tensor | None = None,
|
||||
dt_softplus: bool = False,
|
||||
state_batch_indices: torch.Tensor | None = None,
|
||||
dst_state_batch_indices: torch.Tensor | None = None,
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
out: torch.Tensor | None = None,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
is_blackwell: bool = False,
|
||||
) -> None:
|
||||
"""Unified dispatch for Mamba selective state update.
|
||||
|
||||
Delegates to the initialized backend (Triton or FlashInfer).
|
||||
"""
|
||||
get_mamba_ssu_backend()(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
D,
|
||||
dt_bias,
|
||||
z=z,
|
||||
dt_softplus=dt_softplus,
|
||||
state_batch_indices=state_batch_indices,
|
||||
dst_state_batch_indices=dst_state_batch_indices,
|
||||
null_block_id=null_block_id,
|
||||
out=out,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
cu_seqlens=cu_seqlens,
|
||||
is_blackwell=is_blackwell,
|
||||
)
|
||||
@@ -38,10 +38,10 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import selective_state_update
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import selective_state_update
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -447,13 +447,11 @@ class Plamo2MambaMixer(MambaBase, PluggableLayer):
|
||||
B,
|
||||
C,
|
||||
D,
|
||||
dt_bias,
|
||||
z=gate_d.reshape(num_decodes, -1, self.head_dim),
|
||||
dt_bias=dt_bias,
|
||||
dt_softplus=True,
|
||||
state_batch_indices=state_indices_tensor_d,
|
||||
out=preallocated_ssm_out_d.view(num_decodes, -1, self.head_dim),
|
||||
enable_stochastic_rounding=self.cache_config.enable_mamba_cache_stochastic_rounding,
|
||||
cache_philox_rounds=self.cache_config.mamba_cache_philox_rounds,
|
||||
)
|
||||
|
||||
# 4. Final linear projection
|
||||
|
||||
Reference in New Issue
Block a user