[Mamba] Flashinfer selective_state_update (#36162)

Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
roikoren755
2026-04-14 15:10:58 -04:00
committed by GitHub
parent 30679319e8
commit ecf5ff7ce3
15 changed files with 516 additions and 71 deletions
+3
View File
@@ -16,6 +16,7 @@ from vllm.config.kv_events import KVEventsConfig
from vllm.config.kv_transfer import KVTransferConfig
from vllm.config.load import LoadConfig
from vllm.config.lora import LoRAConfig
from vllm.config.mamba import MambaConfig
from vllm.config.model import (
ModelConfig,
iter_architecture_defaults,
@@ -83,6 +84,8 @@ __all__ = [
"LoadConfig",
# From vllm.config.lora
"LoRAConfig",
# From vllm.config.mamba
"MambaConfig",
# From vllm.config.model
"ModelConfig",
"iter_architecture_defaults",
-34
View File
@@ -123,14 +123,6 @@ class CacheConfig:
- "align": only cache the mamba state of the last token of each scheduler step and
when the token is at position i * block_size.
"""
enable_mamba_cache_stochastic_rounding: bool = False
"""Enable stochastic rounding when writing SSM state to fp16 cache.
Uses random bits to unbias the rounding error, which can improve
numerical stability for long sequences."""
mamba_cache_philox_rounds: int = 0
"""Number of Philox PRNG rounds for stochastic rounding random number
generation. 0 uses the Triton default. Higher values improve randomness
quality at the cost of compute."""
# Will be set after profiling.
num_gpu_blocks: int | None = field(default=None, init=False)
@@ -258,29 +250,3 @@ class CacheConfig:
str(cache_dtype),
)
return cache_dtype
def __post_init__(self):
if self.enable_mamba_cache_stochastic_rounding:
from vllm.platforms import current_platform
if not current_platform.is_cuda():
raise ValueError(
"Stochastic rounding for Mamba cache is only supported "
"on NVIDIA CUDA platforms. Please do not specify "
"`--enable-mamba-cache-stochastic-rounding`."
)
if not current_platform.is_device_capability_family(100):
raise ValueError(
"Stochastic rounding for Mamba cache requires compute "
"capability 10.0 (data center Blackwell). The `cvt.rs` PTX "
"instruction is not supported on your GPU. Please do not specify "
"`--enable-mamba-cache-stochastic-rounding`."
)
if self.mamba_ssm_cache_dtype != "float16":
raise ValueError(
"Stochastic rounding for Mamba cache requires "
"the SSM cache to be float16. Please set it explicitly, "
"by specifying `--mamba-ssm-cache-dtype float16`, or disable "
"stochastic rounding by not specifying "
"`--enable-mamba-cache-stochastic-rounding`."
)
+76
View File
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from enum import Enum, EnumMeta
from typing import Any
from pydantic import field_validator
from vllm.config.utils import config
class _MambaBackendEnumMeta(EnumMeta):
"""Metaclass for MambaBackendEnum to provide better error messages."""
def __getitem__(cls, name: str):
try:
return super().__getitem__(name)
except KeyError:
valid = ", ".join(cls.__members__.keys())
raise ValueError(
f"Unknown Mamba SSU backend: '{name}'. Valid options are: {valid}"
) from None
class MambaBackendEnum(Enum, metaclass=_MambaBackendEnumMeta):
"""Enumeration of supported Mamba SSU (selective state update) backends."""
TRITON = "triton"
FLASHINFER = "flashinfer"
@config
class MambaConfig:
"""Configuration for Mamba SSM backends."""
backend: MambaBackendEnum = MambaBackendEnum.TRITON
"""Mamba SSU backend to use."""
enable_stochastic_rounding: bool = False
"""Enable stochastic rounding when writing SSM state to fp16 cache.
Uses random bits to unbias the rounding error, which can improve
numerical stability for long sequences."""
stochastic_rounding_philox_rounds: int = 0
"""Number of Philox PRNG rounds for stochastic rounding random number
generation. 0 uses the Triton default. Higher values improve randomness
quality at the cost of compute."""
@field_validator("backend", mode="before")
@classmethod
def validate_backend_before(cls, value: Any) -> Any:
"""Enable parsing of the `backend` enum type from string."""
if isinstance(value, str):
return MambaBackendEnum[value.upper()]
return value
def __post_init__(self):
if self.enable_stochastic_rounding:
from vllm.platforms import current_platform
if not current_platform.is_cuda():
raise ValueError(
"Stochastic rounding for Mamba cache is only supported "
"on NVIDIA CUDA platforms. Please do not specify "
"`--enable-mamba-cache-stochastic-rounding`."
)
if (
self.backend == MambaBackendEnum.TRITON
and not current_platform.is_device_capability_family(100)
):
raise ValueError(
"Stochastic rounding for Mamba cache with triton backend requires "
"compute capability 10.0 (data center Blackwell). The `cvt.rs` "
"PTX instruction is not supported on your GPU. Please do not "
"specify `--enable-mamba-cache-stochastic-rounding`, "
"or set `--mamba-backend flashinfer`."
)
+15
View File
@@ -37,6 +37,7 @@ from .kv_events import KVEventsConfig
from .kv_transfer import KVTransferConfig
from .load import LoadConfig
from .lora import LoRAConfig
from .mamba import MambaConfig
from .model import ModelConfig
from .observability import ObservabilityConfig
from .offload import OffloadConfig
@@ -275,6 +276,8 @@ class VllmConfig:
"""Model weight offloading configuration."""
attention_config: AttentionConfig = Field(default_factory=AttentionConfig)
"""Attention configuration."""
mamba_config: MambaConfig = Field(default_factory=MambaConfig)
"""Mamba configuration."""
kernel_config: KernelConfig = Field(default_factory=KernelConfig)
"""Kernel configuration."""
lora_config: LoRAConfig | None = None
@@ -717,6 +720,18 @@ class VllmConfig:
if self.lora_config is not None:
self.lora_config.verify_with_model_config(self.model_config)
if (
self.mamba_config.enable_stochastic_rounding
and self.cache_config.mamba_ssm_cache_dtype != "float16"
):
raise ValueError(
"Stochastic rounding for Mamba cache requires "
"the SSM cache to be float16. Please set it explicitly, "
"by specifying `--mamba-ssm-cache-dtype float16`, or disable "
"stochastic rounding by not specifying "
"`--enable-mamba-cache-stochastic-rounding`."
)
if self.quant_config is None and self.model_config is not None:
self.quant_config = VllmConfig._get_quantization_config(
self.model_config, self.load_config