[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)

Signed-off-by: Johnny-Liou <a897111@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Johnny-Liou
2026-07-24 09:39:49 -07:00
committed by GitHub
co-authored by Claude Opus 4.8 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> tomeras91 Cyrus Leung
parent d02df748bf
commit 866fea2b99
23 changed files with 2954 additions and 31 deletions
+9
View File
@@ -727,6 +727,7 @@ class GPUModelRunner(
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
reasoning_config=self.vllm_config.reasoning_config,
use_replayssm=self.cache_config.use_replayssm,
)
# Separate cuda stream for overlapping transfer of sampled token ids from
@@ -2413,6 +2414,12 @@ class GPUModelRunner(
if self.model_config.rswa_window is not None:
rswa_prefix_lens = num_prompt_tokens_cpu
replayssm_decode_base_cpu = None
if self.cache_config.use_replayssm:
replayssm_decode_base_cpu = (
self.input_batch.replayssm_decode_base_cpu_tensor[:num_reqs_padded]
)
cm_base = CommonAttentionMetadata(
query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1],
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1],
@@ -2420,6 +2427,7 @@ class GPUModelRunner(
_seq_lens_cpu=seq_lens_cpu,
_num_computed_tokens_cpu=num_computed_tokens_cpu,
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
replayssm_decode_base_cpu=replayssm_decode_base_cpu,
num_reqs=num_reqs_padded,
num_actual_tokens=num_tokens_padded,
max_query_len=max_query_len,
@@ -7262,6 +7270,7 @@ class GPUModelRunner(
is_pooling_model=self.is_pooling_model,
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
reasoning_config=self.vllm_config.reasoning_config,
use_replayssm=self.cache_config.use_replayssm,
slot_mapping_modes=slot_mapping_modes,
)