[Mamba] Add FlashInfer ReplaySSM backend (#52506)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Andrii Skliar <andreyws96@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
This commit is contained in:
co-authored by
Andrii Skliar
parent
25efcfa788
commit
adebc41b7e
@@ -10,6 +10,7 @@ import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.cache import MambaDType
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.config.model import ModelDType
|
||||
from vllm.distributed import divide
|
||||
from vllm.logger import init_logger
|
||||
@@ -214,20 +215,25 @@ class MambaStateShapeCalculator:
|
||||
base_shapes: tuple[tuple[int, ...], ...],
|
||||
n_groups: int,
|
||||
tp_world_size: int,
|
||||
replayssm_buffer_len: int,
|
||||
logical_window: int,
|
||||
backend: MambaBackendEnum,
|
||||
) -> tuple[tuple[int, ...], ...]:
|
||||
"""Append the ReplaySSM ring shapes (x_cache, dt_cache, B_cache) to a
|
||||
base ``(conv, ssm)`` tuple. ``base_shapes[1]`` is the ssm shape
|
||||
``(nheads // tp, head_dim, state_size)``; B_cache uses the un-extended
|
||||
``n_groups``.
|
||||
"""Append the physical ReplaySSM ring shapes.
|
||||
|
||||
``base_shapes[1]`` is ``(nheads // tp, head_dim, state_size)``;
|
||||
B_cache uses the un-extended ``n_groups``.
|
||||
"""
|
||||
ring_buffer_len = logical_window
|
||||
if backend == MambaBackendEnum.FLASHINFER:
|
||||
# FlashInfer keeps the live window and appended token together.
|
||||
ring_buffer_len += 1
|
||||
local_nheads, head_dim, state_size = base_shapes[1]
|
||||
local_ngroups = divide(n_groups, tp_world_size)
|
||||
return (
|
||||
*base_shapes,
|
||||
(local_nheads, replayssm_buffer_len, head_dim),
|
||||
(local_nheads, replayssm_buffer_len),
|
||||
(local_ngroups, replayssm_buffer_len, state_size),
|
||||
(local_nheads, ring_buffer_len, head_dim),
|
||||
(local_nheads, ring_buffer_len),
|
||||
(local_ngroups, ring_buffer_len, state_size),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user