[Mamba] Add FlashInfer ReplaySSM backend (#52506)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Andrii Skliar <andreyws96@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
Andrii Skliar
2026-09-01 10:26:43 -04:00
committed by GitHub
co-authored by Andrii Skliar
parent 25efcfa788
commit adebc41b7e
19 changed files with 1029 additions and 105 deletions
+19 -3
View File
@@ -633,7 +633,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
cls=self.pcp_manager_cls,
)
initialize_mamba_ssu_backend(
self.vllm_config.mamba_config, self.kv_cache_config
self.vllm_config.mamba_config,
self.kv_cache_config,
use_replayssm=self.vllm_config.cache_config.use_replayssm,
)
if self.adaptive_verification is not None:
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
@@ -707,6 +709,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
context_len: int = 0,
skip_eplb: bool = False,
is_profile: bool = False,
valid_dummy_state_slots: bool = False,
**kwargs,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
if skip_attn and not is_profile:
@@ -764,6 +767,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
skip_attn_for_dummy_run=skip_attn,
is_profile=is_profile,
context_len=context_len,
valid_dummy_state_slots=valid_dummy_state_slots,
)
self.kv_connector.set_disabled(False)
@@ -1390,9 +1394,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
return block_tables, slot_mappings
def prepare_dummy_attn(
self, input_batch: InputBatch
self, input_batch: InputBatch, valid_state_slots: bool = False
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
block_tables = self.block_tables.get_dummy_block_tables(input_batch.num_reqs)
if valid_state_slots:
state_slots = torch.arange(
1,
input_batch.num_reqs + 1,
dtype=torch.int32,
device=self.device,
)
for block_table in block_tables:
block_table[:, 0].copy_(state_slots)
slot_mappings = pcp.maybe_get_pcp_dummy_slot_mappings(
self.pcp_manager, self.block_tables, input_batch.num_tokens
)
@@ -1525,6 +1538,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
skip_attn_for_dummy_run: bool = False,
is_profile: bool = False,
context_len: int = 0,
valid_dummy_state_slots: bool = False,
) -> ModelRunnerOutput | IntermediateTensors | None:
if not dummy_run:
# Update the request states.
@@ -1624,7 +1638,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
max_query_len=batch_desc.max_query_len,
)
if not skip_attn_for_dummy_run:
block_tables, slot_mappings = self.prepare_dummy_attn(input_batch)
block_tables, slot_mappings = self.prepare_dummy_attn(
input_batch, valid_dummy_state_slots
)
if context_len:
set_dummy_context(
input_batch,