[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)
Signed-off-by: Johnny-Liou <a897111@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 4.8
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
tomeras91
Cyrus Leung
parent
d02df748bf
commit
866fea2b99
@@ -727,6 +727,7 @@ class GPUModelRunner(
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
use_replayssm=self.cache_config.use_replayssm,
|
||||
)
|
||||
|
||||
# Separate cuda stream for overlapping transfer of sampled token ids from
|
||||
@@ -2413,6 +2414,12 @@ class GPUModelRunner(
|
||||
if self.model_config.rswa_window is not None:
|
||||
rswa_prefix_lens = num_prompt_tokens_cpu
|
||||
|
||||
replayssm_decode_base_cpu = None
|
||||
if self.cache_config.use_replayssm:
|
||||
replayssm_decode_base_cpu = (
|
||||
self.input_batch.replayssm_decode_base_cpu_tensor[:num_reqs_padded]
|
||||
)
|
||||
|
||||
cm_base = CommonAttentionMetadata(
|
||||
query_start_loc=self.query_start_loc.gpu[: num_reqs_padded + 1],
|
||||
query_start_loc_cpu=self.query_start_loc.cpu[: num_reqs_padded + 1],
|
||||
@@ -2420,6 +2427,7 @@ class GPUModelRunner(
|
||||
_seq_lens_cpu=seq_lens_cpu,
|
||||
_num_computed_tokens_cpu=num_computed_tokens_cpu,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound,
|
||||
replayssm_decode_base_cpu=replayssm_decode_base_cpu,
|
||||
num_reqs=num_reqs_padded,
|
||||
num_actual_tokens=num_tokens_padded,
|
||||
max_query_len=max_query_len,
|
||||
@@ -7262,6 +7270,7 @@ class GPUModelRunner(
|
||||
is_pooling_model=self.is_pooling_model,
|
||||
cp_kv_cache_interleave_size=self.parallel_config.cp_kv_cache_interleave_size,
|
||||
reasoning_config=self.vllm_config.reasoning_config,
|
||||
use_replayssm=self.cache_config.use_replayssm,
|
||||
slot_mapping_modes=slot_mapping_modes,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user