[Mamba] Flashinfer selective_state_update (#36162)

Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
roikoren755
2026-04-14 15:10:58 -04:00
committed by GitHub
parent 30679319e8
commit ecf5ff7ce3
15 changed files with 516 additions and 71 deletions
+42 -11
View File
@@ -45,6 +45,7 @@ from vllm.config import (
KVTransferConfig,
LoadConfig,
LoRAConfig,
MambaConfig,
ModelConfig,
MultiModalConfig,
ObservabilityConfig,
@@ -72,6 +73,7 @@ from vllm.config.cache import (
from vllm.config.device import Device
from vllm.config.kernel import IrOpPriorityConfig, MoEBackend
from vllm.config.lora import MaxLoRARanks
from vllm.config.mamba import MambaBackendEnum
from vllm.config.model import (
ConvertOption,
HfOverrides,
@@ -578,6 +580,7 @@ class EngineArgs:
pooler_config: PoolerConfig | None = ModelConfig.pooler_config
compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config")
kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config")
enable_flashinfer_autotune: bool = get_field(
KernelConfig, "enable_flashinfer_autotune"
@@ -610,10 +613,12 @@ class EngineArgs:
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
enable_mamba_cache_stochastic_rounding: bool = (
CacheConfig.enable_mamba_cache_stochastic_rounding
MambaConfig.enable_stochastic_rounding
)
mamba_cache_philox_rounds: int = CacheConfig.mamba_cache_philox_rounds
mamba_cache_philox_rounds: int = MambaConfig.stochastic_rounding_philox_rounds
additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")
@@ -655,6 +660,8 @@ class EngineArgs:
self.compilation_config = CompilationConfig(**self.compilation_config)
if isinstance(self.attention_config, dict):
self.attention_config = AttentionConfig(**self.attention_config)
if isinstance(self.mamba_config, dict):
self.mamba_config = MambaConfig(**self.mamba_config)
if isinstance(self.kernel_config, dict):
self.kernel_config = KernelConfig(**self.kernel_config)
if isinstance(self.eplb_config, dict):
@@ -825,6 +832,22 @@ class EngineArgs:
"--attention-backend", **attention_kwargs["backend"]
)
# Mamba arguments
mamba_kwargs = get_kwargs(MambaConfig)
mamba_group = parser.add_argument_group(
title="MambaConfig",
description=MambaConfig.__doc__,
)
mamba_group.add_argument("--mamba-backend", **mamba_kwargs["backend"])
mamba_group.add_argument(
"--enable-mamba-cache-stochastic-rounding",
**mamba_kwargs["enable_stochastic_rounding"],
)
mamba_group.add_argument(
"--mamba-cache-philox-rounds",
**mamba_kwargs["stochastic_rounding_philox_rounds"],
)
# Structured outputs arguments
structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig)
structured_outputs_group = parser.add_argument_group(
@@ -1050,13 +1073,6 @@ class EngineArgs:
cache_group.add_argument(
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
)
cache_group.add_argument(
"--enable-mamba-cache-stochastic-rounding",
**cache_kwargs["enable_mamba_cache_stochastic_rounding"],
)
cache_group.add_argument(
"--mamba-cache-philox-rounds", **cache_kwargs["mamba_cache_philox_rounds"]
)
cache_group.add_argument(
"--kv-offloading-size", **cache_kwargs["kv_offloading_size"]
)
@@ -1622,8 +1638,6 @@ class EngineArgs:
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
mamba_block_size=self.mamba_block_size,
mamba_cache_mode=self.mamba_cache_mode,
enable_mamba_cache_stochastic_rounding=self.enable_mamba_cache_stochastic_rounding,
mamba_cache_philox_rounds=self.mamba_cache_philox_rounds,
kv_offloading_size=self.kv_offloading_size,
kv_offloading_backend=self.kv_offloading_backend,
)
@@ -1934,6 +1948,22 @@ class EngineArgs:
self.attention_backend
)
# Mamba config overrides
mamba_config = copy.deepcopy(self.mamba_config)
# Convert string to enum if needed (CLI parsing returns a string)
if isinstance(self.mamba_backend, str):
mamba_config.backend = MambaBackendEnum[self.mamba_backend.upper()]
else:
mamba_config.backend = self.mamba_backend
if self.enable_mamba_cache_stochastic_rounding:
mamba_config.enable_stochastic_rounding = (
self.enable_mamba_cache_stochastic_rounding
)
if self.mamba_cache_philox_rounds:
mamba_config.stochastic_rounding_philox_rounds = (
self.mamba_cache_philox_rounds
)
# Kernel config overrides
kernel_config = copy.deepcopy(self.kernel_config)
if self.enable_flashinfer_autotune is not None:
@@ -2032,6 +2062,7 @@ class EngineArgs:
load_config=load_config,
offload_config=offload_config,
attention_config=attention_config,
mamba_config=mamba_config,
kernel_config=kernel_config,
lora_config=lora_config,
speculative_config=speculative_config,