[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -633,7 +633,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
cls=self.pcp_manager_cls,
|
||||
)
|
||||
initialize_mamba_ssu_backend(
|
||||
self.vllm_config.mamba_config, self.kv_cache_config
|
||||
self.vllm_config.mamba_config,
|
||||
self.kv_cache_config,
|
||||
use_replayssm=self.vllm_config.cache_config.use_replayssm,
|
||||
)
|
||||
if self.adaptive_verification is not None:
|
||||
self.compilation_config.cudagraph_mode = CUDAGraphMode.FULL_AND_PIECEWISE
|
||||
@@ -707,6 +709,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
context_len: int = 0,
|
||||
skip_eplb: bool = False,
|
||||
is_profile: bool = False,
|
||||
valid_dummy_state_slots: bool = False,
|
||||
**kwargs,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
if skip_attn and not is_profile:
|
||||
@@ -764,6 +767,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
skip_attn_for_dummy_run=skip_attn,
|
||||
is_profile=is_profile,
|
||||
context_len=context_len,
|
||||
valid_dummy_state_slots=valid_dummy_state_slots,
|
||||
)
|
||||
self.kv_connector.set_disabled(False)
|
||||
|
||||
@@ -1390,9 +1394,18 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
return block_tables, slot_mappings
|
||||
|
||||
def prepare_dummy_attn(
|
||||
self, input_batch: InputBatch
|
||||
self, input_batch: InputBatch, valid_state_slots: bool = False
|
||||
) -> tuple[tuple[torch.Tensor, ...], torch.Tensor]:
|
||||
block_tables = self.block_tables.get_dummy_block_tables(input_batch.num_reqs)
|
||||
if valid_state_slots:
|
||||
state_slots = torch.arange(
|
||||
1,
|
||||
input_batch.num_reqs + 1,
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
for block_table in block_tables:
|
||||
block_table[:, 0].copy_(state_slots)
|
||||
slot_mappings = pcp.maybe_get_pcp_dummy_slot_mappings(
|
||||
self.pcp_manager, self.block_tables, input_batch.num_tokens
|
||||
)
|
||||
@@ -1525,6 +1538,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
skip_attn_for_dummy_run: bool = False,
|
||||
is_profile: bool = False,
|
||||
context_len: int = 0,
|
||||
valid_dummy_state_slots: bool = False,
|
||||
) -> ModelRunnerOutput | IntermediateTensors | None:
|
||||
if not dummy_run:
|
||||
# Update the request states.
|
||||
@@ -1624,7 +1638,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_query_len=batch_desc.max_query_len,
|
||||
)
|
||||
if not skip_attn_for_dummy_run:
|
||||
block_tables, slot_mappings = self.prepare_dummy_attn(input_batch)
|
||||
block_tables, slot_mappings = self.prepare_dummy_attn(
|
||||
input_batch, valid_dummy_state_slots
|
||||
)
|
||||
if context_len:
|
||||
set_dummy_context(
|
||||
input_batch,
|
||||
|
||||
Reference in New Issue
Block a user