[Kernel] ReplaySSM: cache SSM inputs for faster Mamba2 standard decode (#48018)
Signed-off-by: Johnny-Liou <a897111@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Claude Opus 4.8
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
tomeras91
Cyrus Leung
parent
d02df748bf
commit
866fea2b99
@@ -106,6 +106,7 @@ class InputBatch:
|
||||
is_pooling_model: bool = False,
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
reasoning_config: ReasoningConfig | None = None,
|
||||
use_replayssm: bool = False,
|
||||
slot_mapping_modes: list[SlotMappingMode] | None = None,
|
||||
):
|
||||
self.thinking_budget_state_holder = maybe_create_thinking_budget_state_holder(
|
||||
@@ -170,6 +171,17 @@ class InputBatch:
|
||||
)
|
||||
self.num_computed_tokens_cpu = self.num_computed_tokens_cpu_tensor.numpy()
|
||||
|
||||
# Mamba2 ReplaySSM decode ring origin (num_computed at each request's
|
||||
# last full-state write); populated only when the feature is on.
|
||||
self.use_replayssm = use_replayssm
|
||||
self.replayssm_decode_base_cpu_tensor = torch.zeros(
|
||||
(max_num_reqs,),
|
||||
device="cpu",
|
||||
dtype=torch.int32,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
self.replayssm_decode_base = self.replayssm_decode_base_cpu_tensor.numpy()
|
||||
|
||||
# Block table.
|
||||
self.block_table = MultiGroupBlockTable(
|
||||
max_num_reqs=max_num_reqs,
|
||||
@@ -377,6 +389,11 @@ class InputBatch:
|
||||
# Number of tokens without spec decode tokens.
|
||||
self.num_tokens_no_spec[req_index] = request.num_tokens
|
||||
|
||||
if self.use_replayssm:
|
||||
# Ring origin = full context at (re)admission (prompt + any resumed
|
||||
# output), so a resumed request re-anchors past the prompt.
|
||||
self.replayssm_decode_base[req_index] = request.num_tokens
|
||||
|
||||
self.num_computed_tokens_cpu[req_index] = request.num_computed_tokens
|
||||
self.block_table.add_row(request.block_ids, req_index)
|
||||
|
||||
@@ -597,6 +614,11 @@ class InputBatch:
|
||||
self.num_prompt_tokens[i2],
|
||||
self.num_prompt_tokens[i1],
|
||||
)
|
||||
if self.use_replayssm:
|
||||
self.replayssm_decode_base[i1], self.replayssm_decode_base[i2] = (
|
||||
self.replayssm_decode_base[i2],
|
||||
self.replayssm_decode_base[i1],
|
||||
)
|
||||
self.num_computed_tokens_cpu[i1], self.num_computed_tokens_cpu[i2] = (
|
||||
self.num_computed_tokens_cpu[i2],
|
||||
self.num_computed_tokens_cpu[i1],
|
||||
@@ -754,6 +776,10 @@ class InputBatch:
|
||||
last_req_index
|
||||
]
|
||||
self.num_prompt_tokens[empty_index] = self.num_prompt_tokens[last_req_index]
|
||||
if self.use_replayssm:
|
||||
self.replayssm_decode_base[empty_index] = self.replayssm_decode_base[
|
||||
last_req_index
|
||||
]
|
||||
self.num_computed_tokens_cpu[empty_index] = self.num_computed_tokens_cpu[
|
||||
last_req_index
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user