[Mamba] Add FlashInfer ReplaySSM support for MTP (#52928)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
7e583e615c
commit
deb148e730
@@ -227,6 +227,7 @@ class MambaStateShapeCalculator:
|
||||
tp_world_size: int,
|
||||
logical_window: int,
|
||||
backend: MambaBackendEnum,
|
||||
num_speculative_tokens: int = 0,
|
||||
) -> tuple[
|
||||
tuple[int, int],
|
||||
tuple[int, int, int],
|
||||
@@ -241,8 +242,8 @@ class MambaStateShapeCalculator:
|
||||
"""
|
||||
ring_buffer_len = logical_window
|
||||
if backend == MambaBackendEnum.FLASHINFER:
|
||||
# FlashInfer keeps the live window and appended token together.
|
||||
ring_buffer_len += 1
|
||||
# FlashInfer keeps the live window and current verify window together.
|
||||
ring_buffer_len += 1 + num_speculative_tokens
|
||||
local_nheads, head_dim, state_size = base_shapes[1]
|
||||
local_ngroups = divide(n_groups, tp_world_size)
|
||||
return (
|
||||
|
||||
Reference in New Issue
Block a user