[K3] support recoverssm for K3 (#51855)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Benjamin Chislett <bchislett@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jiangyun Zhu
2026-08-17 19:34:14 +08:00
committed by GitHub
co-authored by OpenAI Codex Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f27ae25473
commit 70afdedc10
19 changed files with 2235 additions and 75 deletions
+73 -22
View File
@@ -282,23 +282,37 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
def get_state_dtype(
self,
) -> tuple[torch.dtype, torch.dtype]:
) -> tuple[torch.dtype, ...]:
if self.model_config is None or self.cache_config is None:
raise ValueError("model_config and cache_config must be set")
return MambaStateDtypeCalculator.kda_state_dtype(
base_dtypes = MambaStateDtypeCalculator.kda_state_dtype(
self.model_config.dtype, self.cache_config.mamba_cache_dtype
)
if self.cache_config.use_kda_recoverssm:
return MambaStateDtypeCalculator.append_kda_recoverssm_record(
base_dtypes, self.model_config.dtype
)
return base_dtypes
def get_state_shape(
self,
) -> tuple[tuple[int, ...], tuple[int, ...]]:
return MambaStateShapeCalculator.kda_state_shape(
) -> tuple[tuple[int, ...], ...]:
base_shapes = MambaStateShapeCalculator.kda_state_shape(
self.tp_size,
self.num_heads,
self.head_dim,
conv_kernel_size=self.conv_size,
num_spec=self.num_spec,
)
if self.cache_config.use_kda_recoverssm:
return MambaStateShapeCalculator.append_kda_recoverssm_record(
base_shapes,
self.num_heads,
self.head_dim,
tp_world_size=self.tp_size,
spec_query_len=1 + self.num_spec,
)
return base_shapes
def __init__(
self,
@@ -308,6 +322,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
run_gemm_rs: bool = False,
) -> None:
super().__init__(config, vllm_config, prefix)
self.use_recoverssm = self.cache_config.use_kda_recoverssm
if self.cache_config.use_replayssm and not self.use_recoverssm:
raise ValueError(
"Kimi-K3 supports --use-replayssm only with speculative decoding"
)
self.spec_query_len = 1 + self.num_spec
kda_config = config.linear_attn_config # type: ignore[attr-defined]
assert kda_config is not None, "linear_attn_config must be set"
@@ -371,7 +391,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
# Keep a width-major copy for fused decode without changing the layout
# consumed by the prefill and fallback decode kernels.
conv_state_dtype, _ = self.get_state_dtype()
conv_state_dtype = self.get_state_dtype()[0]
decode_conv1d_weight = None
if is_fused_kda_decode_supported(
self.local_num_heads,
@@ -567,7 +587,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
g1 = g1[:, :num_actual_tokens]
beta = beta[:, :num_actual_tokens]
conv_state, recurrent_state = self.kv_cache
conv_state, recurrent_state, *recoverssm_records = self.kv_cache
# The convolution kernels consume (..., dim, width - 1).
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
@@ -634,7 +654,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
spec_conv_indices = spec_state_indices_tensor[:, 0][: m.num_spec_decodes]
spec_max_query_len = spec_state_indices_tensor.size(-1)
spec_max_query_len = (
self.spec_query_len
if self.use_recoverssm
else spec_state_indices_tensor.size(-1)
)
spec_conv_out = torch.empty_like(mixed_qkv_spec)
mixed_qkv_spec = causal_conv1d_update(
mixed_qkv_spec,
@@ -659,21 +683,48 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if m.num_prefills == 0 and m.num_decodes == 0
else None
)
core_attn_out_spec, _ = fused_recurrent_kda(
q=q_spec,
k=k_spec,
v=v_spec,
raw_g=g1_spec,
raw_beta=beta_spec,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
cu_seqlens=spec_cu_seqlens,
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
out=spec_out,
)
if self.use_recoverssm:
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
kda_recoverssm_verify,
)
if len(recoverssm_records) != 2:
raise ValueError(
"KDA RecoverSSM requires correction and key/gate buffers"
)
core_attn_out_spec = kda_recoverssm_verify(
q=q_spec,
k=k_spec,
v=v_spec,
raw_g=g1_spec,
raw_beta=beta_spec,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
checkpoint_state=recurrent_state,
correction_cache=recoverssm_records[0],
kg_cache=recoverssm_records[1],
query_start_loc=spec_cu_seqlens,
state_indices=spec_state_indices_tensor[: m.num_spec_decodes, 0],
spec_query_len=self.spec_query_len,
out=spec_out,
)
else:
core_attn_out_spec, _ = fused_recurrent_kda(
q=q_spec,
k=k_spec,
v=v_spec,
raw_g=g1_spec,
raw_beta=beta_spec,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
cu_seqlens=spec_cu_seqlens,
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
out=spec_out,
)
# Prefill or plain-decode path.
core_attn_out_non_spec = None
+176 -12
View File
@@ -6,13 +6,18 @@ The request classification and cudagraph staging intentionally mirror
``GDNAttentionMetadataBuilder``. Kimi-K3 builds the metadata required by its
prefill KDA kernel internally, so this builder omits the shared FLA chunk
metadata construction.
For Kimi-K3 speculative decoding, ``--use-replayssm`` selects the simplified
RecoverSSM path implemented here instead of the Mamba2 ReplaySSM kernel.
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from functools import cache
from typing import TYPE_CHECKING
import torch
from vllm.config import VllmConfig
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
@@ -22,6 +27,10 @@ from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMMetadata,
RecoverSSMPostprocessMetadata,
)
from vllm.v1.attention.backends.utils import (
NULL_BLOCK_ID,
compute_causal_conv1d_metadata,
@@ -29,6 +38,11 @@ from vllm.v1.attention.backends.utils import (
)
from vllm.v1.kv_cache_interface import MambaSpec
if TYPE_CHECKING:
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
KDARecoverSSMCommitContext,
)
@cache
def _metadata_launch_pdl() -> bool:
@@ -229,11 +243,103 @@ def stage_spec_decode_metadata(
@dataclass
class KimiK3KDAMetadata(GDNAttentionMetadata):
pass
class KDARecoverSSMAlignMetadata:
block_table: torch.Tensor
num_computed_tokens: torch.Tensor
block_size: int
@dataclass
class KDARecoverSSMCommitMetadata:
state_indices: torch.Tensor
query_start_loc: torch.Tensor
request_indices: torch.Tensor | None
align: KDARecoverSSMAlignMetadata | None
@dataclass
class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata):
recoverssm_commit: KDARecoverSSMCommitMetadata | None = None
recoverssm_context: "KDARecoverSSMCommitContext | None" = field(
default=None, repr=False, compare=False
)
def commit_recoverssm_state(
self, num_accepted_tokens: torch.Tensor
) -> RecoverSSMPostprocessMetadata | None:
commit = self.recoverssm_commit
if commit is None:
return None
context = self.recoverssm_context
assert context is not None
align = commit.align
context.commit(
num_accepted_tokens,
commit.state_indices[: self.num_spec_decodes, 0],
commit.query_start_loc[: self.num_spec_decodes + 1],
request_indices=commit.request_indices,
block_table=align.block_table if align is not None else None,
num_computed_tokens=(
align.num_computed_tokens if align is not None else None
),
mamba_block_size=align.block_size if align is not None else None,
)
if align is None:
return None
return RecoverSSMPostprocessMetadata(
num_spec_decodes=self.num_spec_decodes,
request_indices=commit.request_indices,
block_table=align.block_table,
num_computed_tokens=align.num_computed_tokens,
block_size=align.block_size,
)
class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
def __init__(
self,
kv_cache_spec: MambaSpec,
layer_names: list[str],
vllm_config: VllmConfig,
device: torch.device,
) -> None:
super().__init__(kv_cache_spec, layer_names, vllm_config, device)
self.use_recoverssm = vllm_config.cache_config.use_kda_recoverssm
self.spec_state_slots = 1 if self.use_recoverssm else self.num_spec + 1
self.recoverssm_num_accepted_tokens: torch.Tensor | None = None
self.recoverssm_context: KDARecoverSSMCommitContext | None = None
if self.use_recoverssm:
max_num_reqs = vllm_config.scheduler_config.max_num_seqs
self.spec_state_indices_tensor = torch.empty(
(max_num_reqs, 1),
dtype=torch.int32,
device=device,
)
self.recoverssm_num_accepted_tokens = torch.ones(
max_num_reqs,
dtype=torch.int32,
device=device,
)
def _get_recoverssm_context(self) -> "KDARecoverSSMCommitContext":
context = self.recoverssm_context
if context is not None:
return context
from vllm.models.kimi_k3.nvidia.ops.recoverssm import (
KDARecoverSSMCommitContext,
)
forward_context = self.vllm_config.compilation_config.static_forward_context
layers = [forward_context[layer_name] for layer_name in self.layer_names]
context = KDARecoverSSMCommitContext.create(
layers,
spec_query_len=1 + self.vllm_config.num_speculative_tokens,
max_num_reqs=self.vllm_config.scheduler_config.max_num_seqs,
)
self.recoverssm_context = context
return context
def build( # type: ignore[override]
self,
common_prefix_len: int,
@@ -263,14 +369,26 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
num_spec_decodes = 0
else:
spec_sequence_masks_cpu = num_decode_draft_tokens_cpu >= 0
# A nonnegative entry identifies a spec request. If no draft token
# was scheduled, process the whole batch as non-spec instead.
if num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item() == 0:
if self.use_recoverssm:
assert m.is_prefilling is not None
assert m.is_prefilling.device.type == "cpu"
active_decode_mask_cpu = (~m.is_prefilling) & (
query_start_loc_cpu.diff() > 0
)
spec_sequence_masks_cpu |= active_decode_mask_cpu
# Native KDA can use its regular decode path when no draft token
# was scheduled. RecoverSSM must preserve its extended conv window.
if (
not self.use_recoverssm
and num_decode_draft_tokens_cpu[spec_sequence_masks_cpu].sum().item()
== 0
):
spec_sequence_masks_cpu = None
num_spec_decodes = 0
else:
num_spec_decodes = spec_sequence_masks_cpu.sum().item()
spec_request_indices = None
if num_spec_decodes == 0:
# The runner orders ordinary decodes before prefills.
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
@@ -289,6 +407,16 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
assert spec_sequence_masks_cpu is not None
assert num_accepted_tokens is not None
query_lens_cpu = query_start_loc_cpu.diff()
if (
self.use_recoverssm
and torch.any(
query_lens_cpu[spec_sequence_masks_cpu] > self.num_spec + 1
).item()
):
raise ValueError(
"KDA RecoverSSM speculative decode query length exceeds "
f"its activation capacity ({self.num_spec + 1})"
)
num_query_tokens = query_start_loc_cpu[-1].item()
# Exclude zero-length cudagraph padding from request-indexed
@@ -325,7 +453,7 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
non_spec_token_indx = None
# Real requests precede trailing cudagraph padding.
spec_state_indices_tensor = block_table_tensor[
:num_spec_decodes, : self.num_spec + 1
:num_spec_decodes, : self.spec_state_slots
]
non_spec_state_indices_tensor = None
# Padding trails real requests, so this prefix already contains
@@ -339,6 +467,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
spec_sequence_masks_gpu = async_tensor_h2d(
spec_sequence_masks_cpu, device=query_start_loc.device
)
if self.use_recoverssm:
spec_request_indices = async_tensor_h2d(
spec_sequence_masks_cpu.nonzero(as_tuple=True)[0],
dtype=torch.int32,
device=query_start_loc.device,
)
spec_token_masks = torch.repeat_interleave(
spec_sequence_masks_gpu,
query_lens,
@@ -351,10 +485,10 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
non_spec_token_indx = index[:num_non_spec_tokens]
spec_token_indx = index[num_non_spec_tokens:]
# Spec requests carry one state slot per speculative step;
# non-spec requests use only their current state slot.
# Native spec uses one state slot per step. RecoverSSM keeps
# only the current checkpoint slot.
spec_state_indices_tensor = block_table_tensor[
spec_sequence_masks_cpu, : self.num_spec + 1
spec_sequence_masks_cpu, : self.spec_state_slots
]
non_spec_state_indices_tensor = block_table_tensor[
active_non_spec_mask_cpu, 0
@@ -400,12 +534,18 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
num_accepted_tokens = num_accepted_tokens[spec_sequence_masks_cpu]
if self.use_recoverssm:
assert self.recoverssm_num_accepted_tokens is not None
num_accepted_tokens = self.recoverssm_num_accepted_tokens[
:num_spec_decodes
]
# Unlike the shared GDN layer, Kimi-K3's prefill KDA wrapper prepares
# its own chunk indices. Only causal-convolution metadata is needed here.
nums_dict, batch_ptr, token_chunk_offset_ptr = None, None, None
if num_prefills > 0:
has_initial_state = m.compute_num_computed_tokens() > 0
if spec_sequence_masks_cpu is not None:
if num_spec_decodes > 0:
has_initial_state = has_initial_state[active_non_spec_mask_cpu]
assert non_spec_query_start_loc_cpu is not None
nums_dict, batch_ptr, token_chunk_offset_ptr = (
@@ -426,7 +566,7 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
and num_spec_decodes > 0
and num_prefills == 0
and num_decodes == 0
and num_spec_decodes <= self.decode_cudagraph_max_bs
and batch_size <= self.spec_state_indices_tensor.shape[0]
and num_spec_decode_tokens <= self.decode_cudagraph_max_bs
):
# Equivalent PyTorch staging:
@@ -463,6 +603,24 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
:batch_size
]
recoverssm_commit = None
if self.use_recoverssm and num_spec_decodes > 0:
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
align = None
if self.kv_cache_spec.mamba_cache_mode == "align":
align = KDARecoverSSMAlignMetadata(
block_table=m.block_table_tensor,
num_computed_tokens=m.compute_num_computed_tokens(),
block_size=self.kv_cache_spec.block_size,
)
recoverssm_commit = KDARecoverSSMCommitMetadata(
state_indices=spec_state_indices_tensor,
query_start_loc=spec_query_start_loc,
request_indices=spec_request_indices,
align=align,
)
return KimiK3KDAMetadata(
num_prefills=num_prefills,
num_prefill_tokens=num_prefill_tokens,
@@ -480,6 +638,12 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder):
spec_token_indx=spec_token_indx,
non_spec_token_indx=non_spec_token_indx,
num_accepted_tokens=num_accepted_tokens,
recoverssm_commit=recoverssm_commit,
recoverssm_context=(
self._get_recoverssm_context()
if recoverssm_commit is not None
else None
),
nums_dict=nums_dict,
batch_ptr=batch_ptr,
token_chunk_offset_ptr=token_chunk_offset_ptr,
+28 -5
View File
@@ -62,12 +62,14 @@ from vllm.model_executor.models.interfaces import (
EagleModelMixin,
HasInnerState,
IsHybrid,
MambaStateShapes,
MixtureOfExperts,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
SupportsQuant,
SupportsReplaySSM,
)
from vllm.model_executor.models.kimi_k25 import KimiK25MediaPixelInputs
from vllm.model_executor.models.kimi_k25_vit import (
@@ -1559,7 +1561,13 @@ class KimiLinearModel(nn.Module, EagleModelMixin, SupportsQuant):
class KimiLinearForCausalLM(
nn.Module, HasInnerState, SupportsPP, MixtureOfExperts, IsHybrid, SupportsEagle3
nn.Module,
HasInnerState,
SupportsPP,
MixtureOfExperts,
IsHybrid,
SupportsEagle3,
SupportsReplaySSM,
):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -1613,15 +1621,20 @@ class KimiLinearForCausalLM(
def get_mamba_state_dtype_from_config(
cls,
vllm_config: "VllmConfig",
) -> tuple[torch.dtype, torch.dtype]:
return MambaStateDtypeCalculator.kda_state_dtype(
) -> tuple[torch.dtype, ...]:
dtypes = MambaStateDtypeCalculator.kda_state_dtype(
vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
)
if vllm_config.cache_config.use_kda_recoverssm:
dtypes = MambaStateDtypeCalculator.append_kda_recoverssm_record(
dtypes, vllm_config.model_config.dtype
)
return dtypes
@classmethod
def get_mamba_state_shape_from_config(
cls, vllm_config: "VllmConfig"
) -> tuple[tuple[int, int], tuple[int, int, int]]:
) -> MambaStateShapes:
parallel_config = vllm_config.parallel_config
hf_config = vllm_config.model_config.hf_config
tp_size = parallel_config.tensor_parallel_size
@@ -1630,13 +1643,22 @@ class KimiLinearForCausalLM(
if vllm_config.speculative_config
else 0
)
return MambaStateShapeCalculator.kda_state_shape(
shapes = MambaStateShapeCalculator.kda_state_shape(
tp_size,
hf_config.linear_attn_config["num_heads"],
hf_config.linear_attn_config["head_dim"],
conv_kernel_size=hf_config.linear_attn_config["short_conv_kernel_size"],
num_spec=num_spec,
)
if vllm_config.cache_config.use_kda_recoverssm:
return MambaStateShapeCalculator.append_kda_recoverssm_record(
shapes,
hf_config.linear_attn_config["num_heads"],
hf_config.linear_attn_config["head_dim"],
tp_world_size=tp_size,
spec_query_len=1 + num_spec,
)
return shapes
@classmethod
def get_mamba_state_copy_func(
@@ -1697,6 +1719,7 @@ class KimiK3ForConditionalGeneration(
SupportsEagle3,
HasInnerState,
IsHybrid,
SupportsReplaySSM,
):
"""Kimi-K3 model with Kimi-K2.5 vision and KimiLinear text."""
File diff suppressed because it is too large Load Diff