[Mamba] Flashinfer selective_state_update (#36162)
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
+42
-11
@@ -45,6 +45,7 @@ from vllm.config import (
|
||||
KVTransferConfig,
|
||||
LoadConfig,
|
||||
LoRAConfig,
|
||||
MambaConfig,
|
||||
ModelConfig,
|
||||
MultiModalConfig,
|
||||
ObservabilityConfig,
|
||||
@@ -72,6 +73,7 @@ from vllm.config.cache import (
|
||||
from vllm.config.device import Device
|
||||
from vllm.config.kernel import IrOpPriorityConfig, MoEBackend
|
||||
from vllm.config.lora import MaxLoRARanks
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.config.model import (
|
||||
ConvertOption,
|
||||
HfOverrides,
|
||||
@@ -578,6 +580,7 @@ class EngineArgs:
|
||||
pooler_config: PoolerConfig | None = ModelConfig.pooler_config
|
||||
compilation_config: CompilationConfig = get_field(VllmConfig, "compilation_config")
|
||||
attention_config: AttentionConfig = get_field(VllmConfig, "attention_config")
|
||||
mamba_config: MambaConfig = get_field(VllmConfig, "mamba_config")
|
||||
kernel_config: KernelConfig = get_field(VllmConfig, "kernel_config")
|
||||
enable_flashinfer_autotune: bool = get_field(
|
||||
KernelConfig, "enable_flashinfer_autotune"
|
||||
@@ -610,10 +613,12 @@ class EngineArgs:
|
||||
mamba_ssm_cache_dtype: MambaDType = CacheConfig.mamba_ssm_cache_dtype
|
||||
mamba_block_size: int | None = get_field(CacheConfig, "mamba_block_size")
|
||||
mamba_cache_mode: MambaCacheMode = CacheConfig.mamba_cache_mode
|
||||
|
||||
mamba_backend: MambaBackendEnum = MambaBackendEnum.TRITON
|
||||
enable_mamba_cache_stochastic_rounding: bool = (
|
||||
CacheConfig.enable_mamba_cache_stochastic_rounding
|
||||
MambaConfig.enable_stochastic_rounding
|
||||
)
|
||||
mamba_cache_philox_rounds: int = CacheConfig.mamba_cache_philox_rounds
|
||||
mamba_cache_philox_rounds: int = MambaConfig.stochastic_rounding_philox_rounds
|
||||
|
||||
additional_config: dict[str, Any] = get_field(VllmConfig, "additional_config")
|
||||
|
||||
@@ -655,6 +660,8 @@ class EngineArgs:
|
||||
self.compilation_config = CompilationConfig(**self.compilation_config)
|
||||
if isinstance(self.attention_config, dict):
|
||||
self.attention_config = AttentionConfig(**self.attention_config)
|
||||
if isinstance(self.mamba_config, dict):
|
||||
self.mamba_config = MambaConfig(**self.mamba_config)
|
||||
if isinstance(self.kernel_config, dict):
|
||||
self.kernel_config = KernelConfig(**self.kernel_config)
|
||||
if isinstance(self.eplb_config, dict):
|
||||
@@ -825,6 +832,22 @@ class EngineArgs:
|
||||
"--attention-backend", **attention_kwargs["backend"]
|
||||
)
|
||||
|
||||
# Mamba arguments
|
||||
mamba_kwargs = get_kwargs(MambaConfig)
|
||||
mamba_group = parser.add_argument_group(
|
||||
title="MambaConfig",
|
||||
description=MambaConfig.__doc__,
|
||||
)
|
||||
mamba_group.add_argument("--mamba-backend", **mamba_kwargs["backend"])
|
||||
mamba_group.add_argument(
|
||||
"--enable-mamba-cache-stochastic-rounding",
|
||||
**mamba_kwargs["enable_stochastic_rounding"],
|
||||
)
|
||||
mamba_group.add_argument(
|
||||
"--mamba-cache-philox-rounds",
|
||||
**mamba_kwargs["stochastic_rounding_philox_rounds"],
|
||||
)
|
||||
|
||||
# Structured outputs arguments
|
||||
structured_outputs_kwargs = get_kwargs(StructuredOutputsConfig)
|
||||
structured_outputs_group = parser.add_argument_group(
|
||||
@@ -1050,13 +1073,6 @@ class EngineArgs:
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-mode", **cache_kwargs["mamba_cache_mode"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--enable-mamba-cache-stochastic-rounding",
|
||||
**cache_kwargs["enable_mamba_cache_stochastic_rounding"],
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--mamba-cache-philox-rounds", **cache_kwargs["mamba_cache_philox_rounds"]
|
||||
)
|
||||
cache_group.add_argument(
|
||||
"--kv-offloading-size", **cache_kwargs["kv_offloading_size"]
|
||||
)
|
||||
@@ -1622,8 +1638,6 @@ class EngineArgs:
|
||||
mamba_ssm_cache_dtype=self.mamba_ssm_cache_dtype,
|
||||
mamba_block_size=self.mamba_block_size,
|
||||
mamba_cache_mode=self.mamba_cache_mode,
|
||||
enable_mamba_cache_stochastic_rounding=self.enable_mamba_cache_stochastic_rounding,
|
||||
mamba_cache_philox_rounds=self.mamba_cache_philox_rounds,
|
||||
kv_offloading_size=self.kv_offloading_size,
|
||||
kv_offloading_backend=self.kv_offloading_backend,
|
||||
)
|
||||
@@ -1934,6 +1948,22 @@ class EngineArgs:
|
||||
self.attention_backend
|
||||
)
|
||||
|
||||
# Mamba config overrides
|
||||
mamba_config = copy.deepcopy(self.mamba_config)
|
||||
# Convert string to enum if needed (CLI parsing returns a string)
|
||||
if isinstance(self.mamba_backend, str):
|
||||
mamba_config.backend = MambaBackendEnum[self.mamba_backend.upper()]
|
||||
else:
|
||||
mamba_config.backend = self.mamba_backend
|
||||
if self.enable_mamba_cache_stochastic_rounding:
|
||||
mamba_config.enable_stochastic_rounding = (
|
||||
self.enable_mamba_cache_stochastic_rounding
|
||||
)
|
||||
if self.mamba_cache_philox_rounds:
|
||||
mamba_config.stochastic_rounding_philox_rounds = (
|
||||
self.mamba_cache_philox_rounds
|
||||
)
|
||||
|
||||
# Kernel config overrides
|
||||
kernel_config = copy.deepcopy(self.kernel_config)
|
||||
if self.enable_flashinfer_autotune is not None:
|
||||
@@ -2032,6 +2062,7 @@ class EngineArgs:
|
||||
load_config=load_config,
|
||||
offload_config=offload_config,
|
||||
attention_config=attention_config,
|
||||
mamba_config=mamba_config,
|
||||
kernel_config=kernel_config,
|
||||
lora_config=lora_config,
|
||||
speculative_config=speculative_config,
|
||||
|
||||
Reference in New Issue
Block a user