[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)
Signed-off-by: Johnny-Liou <a897111@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 4.8
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
tomeras91
Cyrus Leung
parent
d02df748bf
commit
866fea2b99
@@ -145,6 +145,17 @@ class CacheConfig:
|
||||
- "align": only cache the mamba state of the last token of each scheduler step and
|
||||
when the token is at position i * block_size.
|
||||
"""
|
||||
replayssm_buffer_len: int = Field(default=16, gt=0)
|
||||
"""ReplaySSM history buffer length B: with use_replayssm, standard decode
|
||||
caches recent SSM inputs in a size-B ring buffer and flushes the checkpoint
|
||||
state to HBM every B steps. Default 16."""
|
||||
use_replayssm: bool = False
|
||||
"""Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip
|
||||
the per-step full-state store, writing the checkpoint back only on flush.
|
||||
Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton
|
||||
mamba backend; standard (non-speculative) decode only. In align mode flushes
|
||||
are most efficient when mamba_block_size is a multiple of replayssm_buffer_len,
|
||||
but this is not required."""
|
||||
|
||||
# Will be set after profiling.
|
||||
num_gpu_blocks: int | None = field(default=None, init=False)
|
||||
|
||||
@@ -1696,6 +1696,10 @@ class ModelConfig:
|
||||
def supports_mamba_prefix_caching(self) -> bool:
|
||||
return self._model_info.supports_mamba_prefix_caching
|
||||
|
||||
@property
|
||||
def supports_replayssm(self) -> bool:
|
||||
return self._model_info.supports_replayssm
|
||||
|
||||
@property
|
||||
def use_mla(self) -> bool:
|
||||
return self.is_deepseek_mla and not envs.VLLM_MLA_DISABLE
|
||||
|
||||
+32
-1
@@ -39,7 +39,7 @@ from .kv_events import KVEventsConfig
|
||||
from .kv_transfer import KVTransferConfig
|
||||
from .load import LoadConfig
|
||||
from .lora import LoRAConfig
|
||||
from .mamba import MambaConfig
|
||||
from .mamba import MambaBackendEnum, MambaConfig
|
||||
from .model import ModelConfig
|
||||
from .observability import ObservabilityConfig
|
||||
from .offload import OffloadConfig
|
||||
@@ -2308,6 +2308,37 @@ class VllmConfig:
|
||||
)
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_mamba_cached_kernel(self) -> "VllmConfig":
|
||||
if not self.cache_config.use_replayssm:
|
||||
return self
|
||||
# ReplaySSM adds a 3-tensor ring to the mamba state; only models that
|
||||
# opt in (supports_replayssm) build a consistent shape on both the layer
|
||||
# and config paths. Reject others so the mamba page size cannot desync.
|
||||
if self.model_config is not None and not self.model_config.supports_replayssm:
|
||||
raise ValueError(
|
||||
"--use-replayssm is only supported for Nemotron-H models "
|
||||
f"(got architecture {self.model_config.architecture!r})"
|
||||
)
|
||||
if self.cache_config.mamba_cache_mode == "all":
|
||||
raise ValueError(
|
||||
"--use-replayssm supports prefix caching only in align mode; "
|
||||
"pass --mamba-cache-mode align"
|
||||
)
|
||||
if self.num_speculative_tokens > 0:
|
||||
raise ValueError("--use-replayssm does not support speculative decoding")
|
||||
if self.mamba_config.backend != MambaBackendEnum.TRITON:
|
||||
raise ValueError("--use-replayssm requires --mamba-backend triton")
|
||||
if (
|
||||
self.kv_transfer_config is not None
|
||||
and self.kv_transfer_config.is_kv_transfer_instance
|
||||
):
|
||||
raise ValueError(
|
||||
"--use-replayssm is incompatible with KV connectors "
|
||||
"(P/D disaggregation, KV cache offload)"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
_current_vllm_config: VllmConfig | None = None
|
||||
_current_prefix: str | None = None
|
||||
|
||||
Reference in New Issue
Block a user