[V1][Hybrid] Mamba Prefix Caching with align mode (#30877)
Signed-off-by: huanghaoyan.hhy <huanghaoyan.hhy@alibaba-inc.com> Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
co-authored by
Chen Zhang
parent
fec9da0af4
commit
5206e5e28c
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeAlias
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.cache import MambaDType
|
||||
@@ -223,3 +227,94 @@ class MambaStateShapeCalculator:
|
||||
conv_state_k_shape,
|
||||
recurrent_state_shape,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MambaCopySpec:
|
||||
"""
|
||||
Data class specifying the memory-copy parameters for Mamba states used for
|
||||
prefix caching in align mode.
|
||||
|
||||
Attributes:
|
||||
start_addr (int): Starting address for the memory copy operation.
|
||||
num_elements (int): Number of elements to copy from the starting address.
|
||||
"""
|
||||
|
||||
start_addr: int
|
||||
num_elements: int
|
||||
|
||||
|
||||
MambaStateCopyFunc: TypeAlias = Callable[
|
||||
[torch.Tensor, list[int], int, int], MambaCopySpec
|
||||
]
|
||||
"""
|
||||
Type alias for a function that computes a MambaCopySpec for copying state slices.
|
||||
Parameters:
|
||||
state: torch.Tensor - the Mamba state tensor (e.g., conv or temporal states).
|
||||
block_ids: list[int] - the list of block indices for the state to copy.
|
||||
cur_block_idx: int - current block index within `block_ids` to copy from.
|
||||
num_accepted_tokens: int - number of accepted tokens used to compute the copy offset.
|
||||
Range: 1 .. 1 + num_speculative_tokens (inclusive).
|
||||
"""
|
||||
|
||||
|
||||
def get_conv_copy_spec(
|
||||
state: torch.Tensor,
|
||||
block_ids: list[int],
|
||||
cur_block_idx: int,
|
||||
num_accepted_tokens: int,
|
||||
) -> MambaCopySpec:
|
||||
"""Return a MambaCopySpec for copying a convolutional state slice."""
|
||||
src_block_id = block_ids[cur_block_idx]
|
||||
src_state = state[src_block_id, num_accepted_tokens - 1 :]
|
||||
return MambaCopySpec(
|
||||
start_addr=src_state.data_ptr(), num_elements=src_state.numel()
|
||||
)
|
||||
|
||||
|
||||
def get_temporal_copy_spec(
|
||||
state: torch.Tensor,
|
||||
block_ids: list[int],
|
||||
cur_block_idx: int,
|
||||
num_accepted_tokens: int,
|
||||
) -> MambaCopySpec:
|
||||
"""Return a MambaCopySpec for copying a temporal state slice."""
|
||||
src_block_id = block_ids[cur_block_idx + num_accepted_tokens - 1]
|
||||
src_state = state[src_block_id]
|
||||
return MambaCopySpec(
|
||||
start_addr=src_state.data_ptr(), num_elements=src_state.numel()
|
||||
)
|
||||
|
||||
|
||||
get_full_copy_spec = get_temporal_copy_spec
|
||||
|
||||
|
||||
class MambaStateCopyFuncCalculator:
|
||||
@classmethod
|
||||
def linear_attention_state_copy_func(cls):
|
||||
return (get_temporal_copy_spec,)
|
||||
|
||||
@classmethod
|
||||
def mamba1_state_copy_func(cls):
|
||||
return (get_conv_copy_spec, get_temporal_copy_spec)
|
||||
|
||||
@classmethod
|
||||
def mamba2_state_copy_func(cls):
|
||||
return get_conv_copy_spec, get_temporal_copy_spec
|
||||
|
||||
@classmethod
|
||||
def short_conv_state_copy_func(cls):
|
||||
return (get_conv_copy_spec,)
|
||||
|
||||
@classmethod
|
||||
def gated_delta_net_state_copy_func(cls):
|
||||
return (get_conv_copy_spec, get_temporal_copy_spec)
|
||||
|
||||
@classmethod
|
||||
def kda_state_copy_func(cls):
|
||||
return (
|
||||
get_conv_copy_spec,
|
||||
get_conv_copy_spec,
|
||||
get_conv_copy_spec,
|
||||
get_temporal_copy_spec,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user