[K3] support recoverssm for K3 (#51855)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-08-17 19:34:14 +08:00
committed by GitHub
co-authored by OpenAI Codex Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f27ae25473
commit 70afdedc10
19 changed files with 2235 additions and 75 deletions
+5 -3
View File
@@ -71,10 +71,12 @@ class MambaBase(AttentionLayerBase):
page_size_padded=page_size_padded,
mamba_type=self.mamba_type,
mamba_cache_mode=vllm_config.cache_config.mamba_cache_mode,
# RecoverSSM verifies the whole window off one checkpoint, so it
# never writes the baseline's per-draft-token state slots.
num_speculative_blocks=(
vllm_config.speculative_config.num_speculative_tokens
if vllm_config.speculative_config
else 0
0
if vllm_config.cache_config.use_kda_recoverssm
else vllm_config.num_speculative_tokens
),
)
@@ -136,6 +136,15 @@ class MambaStateDtypeCalculator:
state_dtype = get_kv_cache_torch_dtype(mamba_cache_dtype, model_dtype)
return (state_dtype, torch.float32)
@classmethod
def append_kda_recoverssm_record(
cls,
base_dtypes: tuple[torch.dtype, ...],
model_dtype: ModelDType | torch.dtype,
) -> tuple[torch.dtype, ...]:
activation_dtype = get_kv_cache_torch_dtype("auto", model_dtype)
return (*base_dtypes, torch.float32, activation_dtype)
class MambaStateShapeCalculator:
@classmethod
@@ -293,6 +302,27 @@ class MambaStateShapeCalculator:
recurrent_state_shape = (divide(num_heads, tp_world_size), head_dim, head_dim)
return (conv_state_shape, recurrent_state_shape)
@classmethod
def append_kda_recoverssm_record(
cls,
base_shapes: tuple[tuple[int, int], tuple[int, int, int]],
num_heads: int,
head_dim: int,
tp_world_size: int,
spec_query_len: int,
) -> tuple[
tuple[int, int],
tuple[int, int, int],
tuple[int, int, int],
tuple[int, int, int],
]:
local_num_heads = divide(num_heads, tp_world_size)
return (
*base_shapes,
(local_num_heads, spec_query_len, head_dim),
(local_num_heads, spec_query_len, 2 * head_dim),
)
@dataclass
class MambaCopySpec: