[Misc] Replace mamba_type string literals with MambaAttentionBackendEnum (#41188)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
wangxiyuan
2026-05-11 03:59:36 +00:00
committed by GitHub
parent 7f95e66a11
commit 5536fc0c01
18 changed files with 64 additions and 58 deletions
+1 -1
View File
@@ -142,7 +142,7 @@ We use "mamba-like" to refer to layers that possess a state that is updated in-p
For implementing new custom mamba-like layers, one should inherit from `MambaBase` and implement the methods `get_state_dtype`, `get_state_shape` to calculate the data types and state shapes at runtime, as well as `mamba_type` and `get_attn_backend`.
It is also necessary to implement the "attention meta-data" class which handles the meta-data that is common across all layers.
Please see [`LinearAttentionMetadata`](../../../vllm/v1/attention/backends/linear_attn.py) or [`ShortConvAttentionMetadata`](../../../vllm/v1/attention/backends/short_conv_attn.py) for examples of this.
It is also worth noting that we should update `MAMBA_TYPE_TO_BACKEND_MAP` and `MambaAttentionBackendEnum` in [`registry.py`](../../../vllm/v1/attention/backends/registry.py) when adding a new mamba backend.
It is also worth noting that we should update `MambaAttentionBackendEnum` in [`registry.py`](../../../vllm/v1/attention/backends/registry.py) when adding a new mamba backend.
Finally, if one wants to support torch compile and CUDA graphs, it necessary to wrap the call to the mamba-like layer inside a custom op and register it.
Please see the calls to `direct_register_custom_op` in [vllm/model_executor/models/minimax_text_01.py](../../../vllm/model_executor/models/minimax_text_01.py) or [vllm/model_executor/layers/mamba/short_conv.py](../../../vllm/model_executor/layers/mamba/short_conv.py) for examples of this.
The new custom op should then be added to the list `_attention_ops` in [vllm/config/compilation.py](../../../vllm/config/compilation.py) to ensure that piecewise CUDA graphs works as intended.
+10 -2
View File
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
selective_state_update,
)
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
@@ -27,7 +28,9 @@ except ImportError:
HAS_FLASHINFER = False
def _kv_cache_config_with_ssu(mamba_type: str = "mamba2") -> KVCacheConfig:
def _kv_cache_config_with_ssu(
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2,
) -> KVCacheConfig:
spec = MambaSpec(
block_size=16,
shapes=((16, 64),),
@@ -77,7 +80,12 @@ def test_uninitialized_backend_raises():
@pytest.mark.parametrize(
"mamba_type", ["linear_attention", "gdn_attention", "short_conv"]
"mamba_type",
[
MambaAttentionBackendEnum.LINEAR,
MambaAttentionBackendEnum.GDN_ATTN,
MambaAttentionBackendEnum.SHORT_CONV,
],
)
def test_init_is_noop_for_non_ssu_mamba_type(mamba_type):
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
@@ -13,6 +13,7 @@ from vllm.model_executor.models.minimax_text_01 import MiniMaxText01LinearAttent
from vllm.v1.attention.backends.linear_attn import LinearAttentionBackend
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionBackend
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionBackend
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
@@ -32,7 +33,7 @@ from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
use_rms_norm=True,
),
Mamba1AttentionBackend,
"mamba1",
MambaAttentionBackendEnum.MAMBA1,
),
(
MambaMixer2,
@@ -48,7 +49,7 @@ from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
head_dim=32,
),
Mamba2AttentionBackend,
"mamba2",
MambaAttentionBackendEnum.MAMBA2,
),
(
MiniMaxText01LinearAttention,
@@ -64,7 +65,7 @@ from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
linear_layer_idx=0,
),
LinearAttentionBackend,
"linear_attention",
MambaAttentionBackendEnum.LINEAR,
),
(
ShortConv,
@@ -74,7 +75,7 @@ from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionBackend
layer_idx=0,
),
ShortConvAttentionBackend,
"short_conv",
MambaAttentionBackendEnum.SHORT_CONV,
),
],
)
@@ -97,10 +98,14 @@ def test_mamba_layers_get_attn_backend(
@pytest.mark.parametrize(
"layer_class,expected_backend,expected_mamba_type",
[
(MambaMixer, Mamba1AttentionBackend, "mamba1"),
(MambaMixer2, Mamba2AttentionBackend, "mamba2"),
(MiniMaxText01LinearAttention, LinearAttentionBackend, "linear_attention"),
(ShortConv, ShortConvAttentionBackend, "short_conv"),
(MambaMixer, Mamba1AttentionBackend, MambaAttentionBackendEnum.MAMBA1),
(MambaMixer2, Mamba2AttentionBackend, MambaAttentionBackendEnum.MAMBA2),
(
MiniMaxText01LinearAttention,
LinearAttentionBackend,
MambaAttentionBackendEnum.LINEAR,
),
(ShortConv, ShortConvAttentionBackend, MambaAttentionBackendEnum.SHORT_CONV),
],
)
def test_mamba_layers_have_unified_interface(
@@ -13,6 +13,7 @@ from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.mamba_utils import is_conv_state_dim_first
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
@@ -103,7 +104,7 @@ def derive_mamba_conv_split(
MambaConvSplitInfo with per-rank x_local, b_local, conv_rows,
conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes).
"""
if mamba_spec.mamba_type != "mamba2":
if mamba_spec.mamba_type != MambaAttentionBackendEnum.MAMBA2:
raise NotImplementedError(
f"3-read conv transfer only supports Mamba2 models, "
f"got mamba_type={mamba_spec.mamba_type!r}. "
+3 -2
View File
@@ -17,6 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from .fla.ops.kda import (
FusedRMSNormGated,
@@ -84,8 +85,8 @@ direct_register_custom_op(
class KimiDeltaAttention(nn.Module, MambaBase):
@property
def mamba_type(self) -> str:
return "gdn_attention"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.GDN_ATTN
def get_state_dtype(
self,
+2 -1
View File
@@ -8,6 +8,7 @@ import torch
from vllm.config import VllmConfig
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.selector import get_mamba_attn_backend
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
@@ -33,7 +34,7 @@ class MambaBase(AttentionLayerBase):
@property
@abstractmethod
def mamba_type(self) -> str:
def mamba_type(self) -> MambaAttentionBackendEnum:
pass
@abstractmethod
@@ -64,6 +64,7 @@ from vllm.utils.torch_utils import (
direct_register_custom_op,
)
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Optional ROCm AITER Triton kernels for the GDN decode fast-path.
# Availability is checked centrally via rocm_aiter_ops; the actual function
@@ -237,8 +238,8 @@ class ChunkGatedDeltaRule(CustomOp):
@PluggableLayer.register("gated_delta_net_attention")
class GatedDeltaNetAttention(PluggableLayer, MambaBase):
@property
def mamba_type(self) -> str:
return "gdn_attention"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.GDN_ATTN
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
@CustomOp.register("minimax_text01_rmsnorm_tp")
@@ -246,8 +247,8 @@ class MiniMaxText01LinearKernel:
class MiniMaxText01LinearAttention(nn.Module, MambaBase):
@property
def mamba_type(self) -> str:
return "linear_attention"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.LINEAR
def get_state_dtype(self) -> tuple[torch.dtype]:
assert self.model_config is not None
@@ -42,6 +42,7 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
@@ -476,8 +477,8 @@ class MambaMixer(MambaBase, PluggableLayer):
)
@property
def mamba_type(self) -> str:
return "mamba1"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.MAMBA1
def _time_proj_bias(self) -> torch.Tensor | None:
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
@@ -52,6 +52,7 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Added by the IBM Team, 2024
@@ -935,8 +936,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
)
@property
def mamba_type(self) -> str:
return "mamba2"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.MAMBA2
def mamba_mixer2(
@@ -14,6 +14,7 @@ import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig
from vllm.logger import init_logger
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
@@ -200,7 +201,8 @@ def initialize_mamba_ssu_backend(
"""
if not any(
isinstance(g.kv_cache_spec, MambaSpec)
and g.kv_cache_spec.mamba_type in ("mamba1", "mamba2")
and g.kv_cache_spec.mamba_type
in (MambaAttentionBackendEnum.MAMBA1, MambaAttentionBackendEnum.MAMBA2)
for g in kv_cache_config.kv_cache_groups
):
return
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
)
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
@@ -223,8 +224,8 @@ class ShortConv(MambaBase, CustomOp):
)
@property
def mamba_type(self) -> str:
return "short_conv"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.SHORT_CONV
def short_conv(
@@ -64,6 +64,7 @@ from vllm.model_executor.models.bailing_moe import BailingMLP
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from .interfaces import HasInnerState, IsHybrid, SupportsPP
from .utils import (
@@ -444,8 +445,8 @@ class BailingMoELinearAttention(PluggableLayer, MambaBase):
# --8<-- [end:bailing_moe_linear_attention]
@property
def mamba_type(self) -> str:
return "linear_attention"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.LINEAR
def get_state_shape(self) -> tuple[tuple[int, ...], ...]:
"""Return state shape for linear attention cache.
+3 -2
View File
@@ -92,6 +92,7 @@ from vllm.triton_utils.allocation import set_triton_allocator
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
from .utils import (
@@ -136,8 +137,8 @@ class OlmoHybridGatedDeltaNet(nn.Module, MambaBase):
"""
@property
def mamba_type(self) -> str:
return "gdn_attention"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.GDN_ATTN
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
+3 -2
View File
@@ -72,6 +72,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.torch_utils import direct_register_custom_op
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
# Only used for type hinting.
if TYPE_CHECKING:
@@ -478,8 +479,8 @@ class Plamo2MambaMixer(MambaBase, PluggableLayer):
)
@property
def mamba_type(self) -> str:
return "mamba2"
def mamba_type(self) -> MambaAttentionBackendEnum:
return MambaAttentionBackendEnum.MAMBA2
def plamo2_mamba_mixer(
-10
View File
@@ -193,16 +193,6 @@ class MambaAttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
_MAMBA_ATTN_OVERRIDES.pop(self, None)
MAMBA_TYPE_TO_BACKEND_MAP = {
"mamba1": MambaAttentionBackendEnum.MAMBA1.name,
"mamba2": MambaAttentionBackendEnum.MAMBA2.name,
"short_conv": MambaAttentionBackendEnum.SHORT_CONV.name,
"linear_attention": MambaAttentionBackendEnum.LINEAR.name,
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN.name,
"custom": MambaAttentionBackendEnum.CUSTOM.name,
}
_ATTN_OVERRIDES: dict[AttentionBackendEnum, str] = {}
_MAMBA_ATTN_OVERRIDES: dict[MambaAttentionBackendEnum, str] = {}
+4 -15
View File
@@ -12,7 +12,6 @@ from vllm.logger import init_logger
from vllm.utils.import_utils import resolve_obj_by_qualname
from vllm.v1.attention.backend import AttentionBackend, AttentionType
from vllm.v1.attention.backends.registry import (
MAMBA_TYPE_TO_BACKEND_MAP,
MambaAttentionBackendEnum,
)
@@ -138,7 +137,7 @@ def _cached_get_attn_backend(
def get_mamba_attn_backend(
mamba_type: str,
mamba_type: MambaAttentionBackendEnum,
) -> type[AttentionBackend]:
"""Select which mamba attention backend to use and lazily import it."""
return _cached_get_mamba_attn_backend(mamba_type)
@@ -146,21 +145,11 @@ def get_mamba_attn_backend(
@cache
def _cached_get_mamba_attn_backend(
mamba_type: str,
mamba_type: MambaAttentionBackendEnum,
) -> type[AttentionBackend]:
assert mamba_type and isinstance(mamba_type, str)
assert mamba_type and isinstance(mamba_type, MambaAttentionBackendEnum)
selected_backend = None
try:
backend_name = MAMBA_TYPE_TO_BACKEND_MAP[mamba_type]
selected_backend = MambaAttentionBackendEnum[backend_name]
except KeyError as e:
raise ValueError(
f"Invalid mamba attention backend type: '{mamba_type}'. Valid "
f"types are: {list(MAMBA_TYPE_TO_BACKEND_MAP.keys())}"
) from e
mamba_attn_backend = selected_backend.get_class()
mamba_attn_backend = mamba_type.get_class()
if envs.VLLM_BATCH_INVARIANT and not mamba_attn_backend.supports_batch_invariance():
raise RuntimeError(
"VLLM batch_invariant mode is not supported for "
+2 -1
View File
@@ -16,6 +16,7 @@ from typing_extensions import Self
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import get_dtype_size, nvfp4_kv_cache_full_dim
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -532,7 +533,7 @@ class MambaSpec(KVCacheSpec):
shapes: tuple[tuple[int, ...], ...]
dtypes: tuple[torch.dtype]
page_size_padded: int | None = None
mamba_type: str = "mamba2"
mamba_type: MambaAttentionBackendEnum = MambaAttentionBackendEnum.MAMBA2
mamba_cache_mode: str = "none"
num_speculative_blocks: int = 0