[K3] support recoverssm for K3 (#51855)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Benjamin Chislett <bchislett@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f27ae25473
commit
70afdedc10
@@ -71,10 +71,12 @@ class MambaBase(AttentionLayerBase):
|
||||
page_size_padded=page_size_padded,
|
||||
mamba_type=self.mamba_type,
|
||||
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
|
||||
# RecoverSSM verifies the whole window off one checkpoint, so it
|
||||
# never writes the baseline's per-draft-token state slots.
|
||||
num_speculative_blocks=(
|
||||
vllm_config.speculative_config.num_speculative_tokens
|
||||
if vllm_config.speculative_config
|
||||
else 0
|
||||
0
|
||||
if vllm_config.cache_config.use_kda_recoverssm
|
||||
else vllm_config.num_speculative_tokens
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -136,6 +136,15 @@ class MambaStateDtypeCalculator:
|
||||
state_dtype = get_kv_cache_torch_dtype(mamba_cache_dtype, model_dtype)
|
||||
return (state_dtype, torch.float32)
|
||||
|
||||
@classmethod
|
||||
def append_kda_recoverssm_record(
|
||||
cls,
|
||||
base_dtypes: tuple[torch.dtype, ...],
|
||||
model_dtype: ModelDType | torch.dtype,
|
||||
) -> tuple[torch.dtype, ...]:
|
||||
activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype)
|
||||
return (*base_dtypes, torch.float32, activation_dtype)
|
||||
|
||||
|
||||
class MambaStateShapeCalculator:
|
||||
@classmethod
|
||||
@@ -293,6 +302,27 @@ class MambaStateShapeCalculator:
|
||||
recurrent_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim)
|
||||
return (conv_state_shape, recurrent_state_shape)
|
||||
|
||||
@classmethod
|
||||
def append_kda_recoverssm_record(
|
||||
cls,
|
||||
base_shapes: tuple[tuple[int, int], tuple[int, int, int]],
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
tp_world_size: int,
|
||||
spec_query_len: int,
|
||||
) -> tuple[
|
||||
tuple[int, int],
|
||||
tuple[int, int, int],
|
||||
tuple[int, int, int],
|
||||
tuple[int, int, int],
|
||||
]:
|
||||
local_num_heads = divide(num_heads, tp_world_size)
|
||||
return (
|
||||
*base_shapes,
|
||||
(local_num_heads, spec_query_len, head_dim),
|
||||
(local_num_heads, spec_query_len, 2 * head_dim),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MambaCopySpec:
|
||||
|
||||
@@ -75,6 +75,12 @@ MambaStateShapes: TypeAlias = (
|
||||
| tuple[tuple[int, int, int]]
|
||||
| tuple[tuple[int, int], tuple[int, int]]
|
||||
| tuple[tuple[int, int], tuple[int, int, int]]
|
||||
| tuple[
|
||||
tuple[int, int],
|
||||
tuple[int, int, int],
|
||||
tuple[int, int, int],
|
||||
tuple[int, int, int],
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -1099,8 +1105,8 @@ def supports_mamba_prefix_caching(
|
||||
|
||||
@runtime_checkable
|
||||
class SupportsReplaySSM(Protocol):
|
||||
"""The interface for models whose Mamba2 layers support ReplaySSM cached
|
||||
standard decode.
|
||||
"""The interface for models whose recurrent layers support ReplaySSM
|
||||
cached decode.
|
||||
|
||||
This is currently experimental.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user