[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)

Signed-off-by: Johnny-Liou <a897111@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Johnny-Liou
2026-07-24 09:39:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> tomeras91 Cyrus Leung
parent d02df748bf
commit 866fea2b99
23 changed files with 2954 additions and 31 deletions
+11
View File
@@ -145,6 +145,17 @@ class CacheConfig:
- "align": only cache the mamba state of the last token of each scheduler step and
when the token is at position i * block_size.
"""
replayssm_buffer_len: int = Field(default=16, gt=0)
"""ReplaySSM history buffer length B: with use_replayssm, standard decode
caches recent SSM inputs in a size-B ring buffer and flushes the checkpoint
state to HBM every B steps. Default 16."""
use_replayssm: bool = False
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
the per-step full-state store, writing the checkpoint back only on flush.
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
mamba backend; standard (non-speculative) decode only. In align mode flushes
are most efficient when mamba_block_size is a multiple of replayssm_buffer_len,
but this is not required."""
# Will be set after profiling.
num_gpu_blocks: int | None = field(default=None, init=False)
+4
View File
@@ -1696,6 +1696,10 @@ class ModelConfig:
def supports_mamba_prefix_caching(self) -> bool:
return self._model_info.supports_mamba_prefix_caching
@property
def supports_replayssm(self) -> bool:
return self._model_info.supports_replayssm
@property
def use_mla(self) -> bool:
return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE
+32 -1
View File
@@ -39,7 +39,7 @@ from .kv_events import KVEventsConfig
from .kv_transfer import KVTransferConfig
from .load import LoadConfig
from .lora import LoRAConfig
from .mamba import MambaConfig
from .mamba import MambaBackendEnum, MambaConfig
from .model import ModelConfig
from .observability import ObservabilityConfig
from .offload import OffloadConfig
@@ -2308,6 +2308,37 @@ class VllmConfig:
)
return self
@model_validator(mode="after")
def validate_mamba_cached_kernel(self) -> "VllmConfig":
if not self.cache_config.use_replayssm:
return self
# ReplaySSM adds a 3-tensor ring to the mamba state; only models that
# opt in (supports_replayssm) build a consistent shape on both the layer
# and config paths. Reject others so the mamba page size cannot desync.
if self.model_config is not None and not self.model_config.supports_replayssm:
raise ValueError(
"--use-replayssm is only supported for Nemotron-H models "
f"(got architecture {self.model_config.architecture!r})"
)
if self.cache_config.mamba_cache_mode == "all":
raise ValueError(
"--use-replayssm supports prefix caching only in align mode; "
"pass --mamba-cache-mode align"
)
if self.num_speculative_tokens > 0:
raise ValueError("--use-replayssm does not support speculative decoding")
if self.mamba_config.backend != MambaBackendEnum.TRITON:
raise ValueError("--use-replayssm requires --mamba-backend triton")
if (
self.kv_transfer_config is not None
and self.kv_transfer_config.is_kv_transfer_instance
):
raise ValueError(
"--use-replayssm is incompatible with KV connectors "
"(P/D disaggregation, KV cache offload)"
)
return self
_current_vllm_config: VllmConfig | None = None
_current_prefix: str | None = None