[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)

Signed-off-by: Johnny-Liou <a897111@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Johnny-Liou
2026-07-24 09:39:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> tomeras91 Cyrus Leung
parent d02df748bf
commit 866fea2b99
23 changed files with 2954 additions and 31 deletions
+26
View File
@@ -106,6 +106,7 @@ class InputBatch:
is_pooling_model: bool = False,
cp_kv_cache_interleave_size: int = 1,
reasoning_config: ReasoningConfig | None = None,
use_replayssm: bool = False,
slot_mapping_modes: list[SlotMappingMode] | None = None,
):
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
@@ -170,6 +171,17 @@ class InputBatch:
)
self.num_computed_tokens_cpu = self.num_computed_tokens_cpu_tensor.numpy()
# Mamba2 ReplaySSM decode ring origin (num_computed at each request's
# last full-state write); populated only when the feature is on.
self.use_replayssm = use_replayssm
self.replayssm_decode_base_cpu_tensor = torch.zeros(
(max_num_reqs,),
device="cpu",
dtype=torch.int32,
pin_memory=PIN_MEMORY,
)
self.replayssm_decode_base = self.replayssm_decode_base_cpu_tensor.numpy()
# Block table.
self.block_table = MultiGroupBlockTable(
max_num_reqs=max_num_reqs,
@@ -377,6 +389,11 @@ class InputBatch:
# Number of tokens without spec decode tokens.
self.num_tokens_no_spec[req_index] = request.num_tokens
if self.use_replayssm:
# Ring origin = full context at (re)admission (prompt + any resumed
# output), so a resumed request re-anchors past the prompt.
self.replayssm_decode_base[req_index] = request.num_tokens
self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
self.block_table.add_row(request.block_ids, req_index)
@@ -597,6 +614,11 @@ class InputBatch:
self.num_prompt_tokens[i2],
self.num_prompt_tokens[i1],
)
if self.use_replayssm:
self.replayssm_decode_base[i1], self.replayssm_decode_base[i2] = (
self.replayssm_decode_base[i2],
self.replayssm_decode_base[i1],
)
self.num_computed_tokens_cpu[i1], self.num_computed_tokens_cpu[i2] = (
self.num_computed_tokens_cpu[i2],
self.num_computed_tokens_cpu[i1],
@@ -754,6 +776,10 @@ class InputBatch:
last_req_index
]
self.num_prompt_tokens[empty_index] = self.num_prompt_tokens[last_req_index]
if self.use_replayssm:
self.replayssm_decode_base[empty_index] = self.replayssm_decode_base[
last_req_index
]
self.num_computed_tokens_cpu[empty_index] = self.num_computed_tokens_cpu[
last_req_index
]