[ROCm][Perf] Kimi-K3 enable prefill checkpoints on ROCm (#58344)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
kliuae
2026-10-02 06:58:04 +00:00
committed by GitHub
parent 4056c8ac1f
commit 0154979656
7 changed files with 748 additions and 14 deletions
+56 -6
View File
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import replace
import torch
from einops import rearrange
from torch import nn
@@ -38,7 +40,11 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
KimiK3ROCmKDAPrefillCheckpointExporter,
)
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
KDA_CHECKPOINT_ALIGNMENT,
is_fused_kda_chunk_supported,
)
from vllm.models.kimi_k3.amd.ops.kda_decode import (
@@ -55,6 +61,7 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.kv_cache_interface import MambaSpec
logger = init_logger(__name__)
@@ -79,6 +86,23 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
num_spec=self.num_spec,
)
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec:
spec = super().get_kv_cache_spec(vllm_config)
assert isinstance(spec, MambaSpec)
# Only the fused path exports the checkpoint snapshots
can_checkpoint = (
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
)
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
return replace(
spec,
num_prefill_checkpoint_blocks=int(can_checkpoint),
prefill_checkpoint_alignment=(
KDA_CHECKPOINT_ALIGNMENT if can_checkpoint else None
),
)
def __init__(
self,
config: KimiLinearConfig,
@@ -226,6 +250,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
"Kimi-K3 KDA prefill backend: %s",
"fused" if self.use_fused_chunk else "triton",
)
# TODO: Remove the extra config arg once tested
self.use_prefill_checkpoint = (
additional_config.get("kda_prefill_checkpoint", True)
if isinstance(additional_config, dict)
else True
)
self._checkpoint_exporter = KimiK3ROCmKDAPrefillCheckpointExporter(
state_len=self.conv_size - 1
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
decode_norm_weight = None
@@ -511,10 +544,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
assert non_spec_state_indices_tensor is not None
assert has_initial_state is not None
# A mixed non-spec batch is decode-first: the decodes are
# length-1 sequences, and the chunk kernel returns NaN for those.
# Send them to the recurrent kernel and give the chunk kernel the
# prefill tail only.
# Decode-first, so peel the length-1 decodes to the recurrent
# kernel. A spec batch interleaves its non-spec rows, so the
# peel cannot apply and the chunk kernel takes them.
core_attn_out_decode = None
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
# Without a spec split the non-spec tokens are exactly
@@ -522,7 +554,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
# can write their slice straight into it. With one, the tokens
# are interleaved and have to be scattered by index afterwards.
direct = spec_sequence_masks is None
use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None
if split_non_spec:
assert non_spec_query_start_loc is not None
nd_tok = m.num_decode_tokens
@@ -559,6 +590,22 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
prefill_has_initial_state = has_initial_state
nd_tok = 0
checkpoint = getattr(m, "checkpoint", None)
checkpoint_state = None
checkpoint_offsets = None
checkpoint_state_indices = None
if checkpoint is not None:
assert prefill_query_start_loc is not None
checkpoint_state = recurrent_state
checkpoint_offsets = checkpoint.checkpoint_offsets
checkpoint_state_indices = checkpoint.state_indices
self._checkpoint_exporter.export(
checkpoint,
raw_qkv=mixed_qkv_ns[nd_tok:],
conv_state=conv_state,
cu_seqlens=prefill_query_start_loc,
)
(
core_attn_out_non_spec,
_,
@@ -578,8 +625,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
cu_seqlens=prefill_query_start_loc,
chunk_indices=m.chunk_indices,
chunk_offsets=m.chunk_offsets,
use_fused_chunk=use_fused_chunk,
use_fused_chunk=self.use_fused_chunk,
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
checkpoint_state_indices=checkpoint_state_indices,
)
# chunk_kda_prefill updates `recurrent_state` in place, so
# there is no final state to write back here.
+56
View File
@@ -7,16 +7,23 @@ The request classification and cudagraph staging intentionally mirror
differently on device rather than on the host.
"""
from dataclasses import replace
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionBackend,
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
logger = init_logger(__name__)
@@ -86,6 +93,55 @@ def prepare_chunk_metadata_device(
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
def build( # type: ignore[override]
self,
common_prefix_len: int,
common_attn_metadata: CommonAttentionMetadata,
num_accepted_tokens: torch.Tensor | None = None,
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
fast_build: bool = False,
) -> GDNAttentionMetadata:
attn_metadata = super().build(
common_prefix_len,
common_attn_metadata,
num_accepted_tokens,
num_decode_draft_tokens_cpu,
fast_build,
)
if attn_metadata.checkpoint is None:
return attn_metadata
# The shared builder keys checkpoint rows on every request; the ROCm
# prefill path indexes them by prefill_query_start_loc, so rebuild with
# those rows.
return replace(
attn_metadata,
checkpoint=self._build_checkpoint_metadata(
common_attn_metadata, attn_metadata, num_decode_draft_tokens_cpu
),
)
def _build_checkpoint_metadata(
self,
m: CommonAttentionMetadata,
attn_metadata: GDNAttentionMetadata,
num_decode_draft_tokens_cpu: torch.Tensor | None,
) -> MambaPrefillCheckpointMetadata | None:
if attn_metadata.num_prefills == 0:
return None
# request_rows must line up with prefill_query_start_loc, either the
# prefill tail of a decode-first batch, or every non-spec row
# including padding
num_decodes = attn_metadata.num_decodes
request_rows = list(
range(num_decodes, num_decodes + attn_metadata.num_prefills)
)
if attn_metadata.spec_sequence_masks is not None:
assert num_decode_draft_tokens_cpu is not None
request_rows = (
(num_decode_draft_tokens_cpu < 0).nonzero().flatten().tolist()
)
return self.checkpoint_builder.build(m, request_rows)
def _build_chunk_metadata(
self,
prefill_query_start_loc: torch.Tensor,
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Conv half of the ROCm Kimi-K3 prefill checkpoint."""
from dataclasses import dataclass
import torch
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.kda_checkpoint import (
FlashKDAPrefillCheckpointExporter,
)
@dataclass(frozen=True)
class KimiK3ROCmKDAPrefillCheckpointExporter(FlashKDAPrefillCheckpointExporter):
"""Conv window only; `fused_kda_chunk` writes the recurrent half itself.
`state_len` is `conv_kernel_size - 1`, not the spec-widened conv row.
"""
def export( # type: ignore[override]
self,
checkpoint: MambaPrefillCheckpointMetadata,
*,
raw_qkv: torch.Tensor,
conv_state: torch.Tensor,
cu_seqlens: torch.Tensor,
) -> None:
assert self.state_len is not None
# An empty recurrent checkpoint masks the kernel's recurrent half off.
super().export(
checkpoint,
raw_qkv=raw_qkv,
conv_state=conv_state,
recurrent_checkpoint=conv_state[:, :0],
recurrent_state=conv_state,
cu_seqlens=cu_seqlens,
)
+11 -1
View File
@@ -16,6 +16,7 @@ from vllm.third_party.flash_linear_attention.ops.index import (
prepare_chunk_indices,
prepare_chunk_offsets,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
CHUNK_SIZE = 64
HEAD_DIM = 128
@@ -28,6 +29,15 @@ _DEEP_LEN = 12
_MAX_GROUPS = 32
def _hip_checkpoint_state_indices(state_indices: torch.Tensor) -> torch.Tensor:
"""NULL_BLOCK_ID marks no checkpoint; the HIP kernel skips negative rows."""
return (
torch.where(state_indices != NULL_BLOCK_ID, state_indices, -1)
.to(torch.int32)
.contiguous()
)
@cache
def is_fused_kda_chunk_supported() -> bool:
from vllm.platforms.rocm import on_gfx950
@@ -295,7 +305,7 @@ def fused_kda_chunk(
None if checkpoint_offsets is None else checkpoint_offsets.to(torch.int32),
None
if checkpoint_state_indices is None
else checkpoint_state_indices.to(torch.int32).contiguous(),
else _hip_checkpoint_state_indices(checkpoint_state_indices),
state_cache,
None if state_indices is None else state_indices.to(torch.int32).contiguous(),
has_initial_state,
+7 -1
View File
@@ -155,7 +155,13 @@ def chunk_kda_prefill(
aqk=ws["aqk"],
decay=ws["decay"],
# v is dead by this point, so it doubles as the output buffer.
out=out if out is not None else v,
out=out
if out is not None
else (
v
if v.is_contiguous()
else torch.empty_like(v, memory_format=torch.contiguous_format)
),
scale=scale,
cu_seqlens=cu_seqlens,
initial_state=initial_state,