[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -196,15 +196,16 @@ class CacheConfig:
|
||||
caching is enabled.
|
||||
"""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM history buffer length B for standard Mamba2 decode. Kimi-K3
|
||||
speculative decoding does not use B. Default 16."""
|
||||
"""ReplaySSM logical history length B for Mamba2. Triton uses B physical
|
||||
rows and FlashInfer uses B+1. Kimi-K3 speculative decode does not use B.
|
||||
Default 16."""
|
||||
use_replayssm: bool = False
|
||||
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
|
||||
the per-step full-state store, writing the checkpoint back only on flush.
|
||||
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
|
||||
mamba backend; standard (non-speculative) decode only. In align mode flushes
|
||||
are most efficient when mamba_block_size is a multiple of replayssm_buffer_len,
|
||||
but this is not required."""
|
||||
or FlashInfer mamba backend; standard (non-speculative) decode only. In align
|
||||
mode flushes are most efficient when mamba_block_size is a multiple of
|
||||
replayssm_buffer_len, but this is not required."""
|
||||
use_kda_recoverssm: bool = field(default=False, init=False)
|
||||
"""Whether Kimi-K3 KDA uses RecoverSSM speculative decode."""
|
||||
|
||||
|
||||
+17
-2
@@ -2889,13 +2889,28 @@ class VllmConfig:
|
||||
raise ValueError(
|
||||
"RecoverSSM currently requires pipeline_parallel_size=1"
|
||||
)
|
||||
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError("RecoverSSM requires --mamba-backend triton")
|
||||
elif self.cache_config.mamba_cache_mode == "all":
|
||||
raise ValueError(
|
||||
"--use-replayssm supports prefix caching only in align mode; "
|
||||
"pass --mamba-cache-mode align"
|
||||
)
|
||||
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError("--use-replayssm requires --mamba-backend triton")
|
||||
elif self.mamba_config.backend == MambaBackendEnum.FLASHINFER:
|
||||
if self.cache_config.mamba_cache_mode == "align":
|
||||
raise ValueError(
|
||||
"FlashInfer ReplaySSM does not support "
|
||||
"--mamba-cache-mode align yet; use none"
|
||||
)
|
||||
elif self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError(
|
||||
"--use-replayssm requires --mamba-backend triton or flashinfer"
|
||||
)
|
||||
elif self.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"Triton ReplaySSM requires Model Runner V1; use "
|
||||
"--mamba-backend flashinfer or Model Runner V1"
|
||||
)
|
||||
if (
|
||||
self.kv_transfer_config is not None
|
||||
and self.kv_transfer_config.is_kv_transfer_instance
|
||||
|
||||
Reference in New Issue
Block a user