[Misc] Replace mamba_type string literals with MambaAttentionBackendEnum (#41188)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
from .fla.ops.kda import (
|
||||
FusedRMSNormGated,
|
||||
@@ -84,8 +85,8 @@ direct_register_custom_op(
|
||||
|
||||
class KimiDeltaAttention(nn.Module, MambaBase):
|
||||
@property
|
||||
def mamba_type(self) -> str:
|
||||
return "gdn_attention"
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.GDN_ATTN
|
||||
|
||||
def get_state_dtype(
|
||||
self,
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.selector import get_mamba_attn_backend
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec, MambaSpec
|
||||
|
||||
@@ -33,7 +34,7 @@ class MambaBase(AttentionLayerBase):
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def mamba_type(self) -> str:
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -64,6 +64,7 @@ from vllm.utils.torch_utils import (
|
||||
direct_register_custom_op,
|
||||
)
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
# Optional ROCm AITER Triton kernels for the GDN decode fast-path.
|
||||
# Availability is checked centrally via rocm_aiter_ops; the actual function
|
||||
@@ -237,8 +238,8 @@ class ChunkGatedDeltaRule(CustomOp):
|
||||
@PluggableLayer.register("gated_delta_net_attention")
|
||||
class GatedDeltaNetAttention(PluggableLayer, MambaBase):
|
||||
@property
|
||||
def mamba_type(self) -> str:
|
||||
return "gdn_attention"
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.GDN_ATTN
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, torch.dtype]:
|
||||
return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.linear_attn import LinearAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
@CustomOp.register("minimax_text01_rmsnorm_tp")
|
||||
@@ -246,8 +247,8 @@ class MiniMaxText01LinearKernel:
|
||||
|
||||
class MiniMaxText01LinearAttention(nn.Module, MambaBase):
|
||||
@property
|
||||
def mamba_type(self) -> str:
|
||||
return "linear_attention"
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.LINEAR
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype]:
|
||||
assert self.model_config is not None
|
||||
|
||||
@@ -42,6 +42,7 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
|
||||
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
|
||||
@@ -476,8 +477,8 @@ class MambaMixer(MambaBase, PluggableLayer):
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> str:
|
||||
return "mamba1"
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.MAMBA1
|
||||
|
||||
def _time_proj_bias(self) -> torch.Tensor | None:
|
||||
if hasattr(self.dt_proj, "bias") and self.dt_proj.bias is not None:
|
||||
|
||||
@@ -52,6 +52,7 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
|
||||
# Added by the IBM Team, 2024
|
||||
|
||||
@@ -935,8 +936,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> str:
|
||||
return "mamba2"
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.MAMBA2
|
||||
|
||||
|
||||
def mamba_mixer2(
|
||||
|
||||
@@ -14,6 +14,7 @@ import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
|
||||
|
||||
@@ -200,7 +201,8 @@ def initialize_mamba_ssu_backend(
|
||||
"""
|
||||
if not any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
and g.kv_cache_spec.mamba_type in ("mamba1", "mamba2")
|
||||
and g.kv_cache_spec.mamba_type
|
||||
in (MambaAttentionBackendEnum.MAMBA1, MambaAttentionBackendEnum.MAMBA2)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
):
|
||||
return
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
)
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.short_conv_attn import ShortConvAttentionMetadata
|
||||
|
||||
|
||||
@@ -223,8 +224,8 @@ class ShortConv(MambaBase, CustomOp):
|
||||
)
|
||||
|
||||
@property
|
||||
def mamba_type(self) -> str:
|
||||
return "short_conv"
|
||||
def mamba_type(self) -> MambaAttentionBackendEnum:
|
||||
return MambaAttentionBackendEnum.SHORT_CONV
|
||||
|
||||
|
||||
def short_conv(
|
||||
|
||||
Reference in New Issue
Block a user