[ROCm][Perf] Kimi-K3 enable prefill checkpoints on ROCm (#58344)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from einops import rearrange
|
||||
from torch import nn
|
||||
@@ -38,7 +40,11 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
from vllm.models.kimi_k3.amd.ops.kda_checkpoint import (
|
||||
KimiK3ROCmKDAPrefillCheckpointExporter,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
KDA_CHECKPOINT_ALIGNMENT,
|
||||
is_fused_kda_chunk_supported,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_decode import (
|
||||
@@ -55,6 +61,7 @@ from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated
|
||||
from vllm.transformers_utils.configs.kimi_linear import KimiLinearConfig
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -79,6 +86,23 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
num_spec=self.num_spec,
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
assert isinstance(spec, MambaSpec)
|
||||
# Only the fused path exports the checkpoint snapshots
|
||||
can_checkpoint = (
|
||||
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
|
||||
)
|
||||
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
|
||||
return replace(
|
||||
spec,
|
||||
num_prefill_checkpoint_blocks=int(can_checkpoint),
|
||||
prefill_checkpoint_alignment=(
|
||||
KDA_CHECKPOINT_ALIGNMENT if can_checkpoint else None
|
||||
),
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
@@ -226,6 +250,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
"Kimi-K3 KDA prefill backend: %s",
|
||||
"fused" if self.use_fused_chunk else "triton",
|
||||
)
|
||||
# TODO: Remove the extra config arg once tested
|
||||
self.use_prefill_checkpoint = (
|
||||
additional_config.get("kda_prefill_checkpoint", True)
|
||||
if isinstance(additional_config, dict)
|
||||
else True
|
||||
)
|
||||
self._checkpoint_exporter = KimiK3ROCmKDAPrefillCheckpointExporter(
|
||||
state_len=self.conv_size - 1
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
decode_norm_weight = None
|
||||
@@ -511,10 +544,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
assert non_spec_state_indices_tensor is not None
|
||||
assert has_initial_state is not None
|
||||
|
||||
# A mixed non-spec batch is decode-first: the decodes are
|
||||
# length-1 sequences, and the chunk kernel returns NaN for those.
|
||||
# Send them to the recurrent kernel and give the chunk kernel the
|
||||
# prefill tail only.
|
||||
# Decode-first, so peel the length-1 decodes to the recurrent
|
||||
# kernel. A spec batch interleaves its non-spec rows, so the
|
||||
# peel cannot apply and the chunk kernel takes them.
|
||||
core_attn_out_decode = None
|
||||
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
|
||||
# Without a spec split the non-spec tokens are exactly
|
||||
@@ -522,7 +554,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
# can write their slice straight into it. With one, the tokens
|
||||
# are interleaved and have to be scattered by index afterwards.
|
||||
direct = spec_sequence_masks is None
|
||||
use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None
|
||||
if split_non_spec:
|
||||
assert non_spec_query_start_loc is not None
|
||||
nd_tok = m.num_decode_tokens
|
||||
@@ -559,6 +590,22 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
prefill_has_initial_state = has_initial_state
|
||||
nd_tok = 0
|
||||
|
||||
checkpoint = getattr(m, "checkpoint", None)
|
||||
checkpoint_state = None
|
||||
checkpoint_offsets = None
|
||||
checkpoint_state_indices = None
|
||||
if checkpoint is not None:
|
||||
assert prefill_query_start_loc is not None
|
||||
checkpoint_state = recurrent_state
|
||||
checkpoint_offsets = checkpoint.checkpoint_offsets
|
||||
checkpoint_state_indices = checkpoint.state_indices
|
||||
self._checkpoint_exporter.export(
|
||||
checkpoint,
|
||||
raw_qkv=mixed_qkv_ns[nd_tok:],
|
||||
conv_state=conv_state,
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
)
|
||||
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
_,
|
||||
@@ -578,8 +625,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
chunk_indices=m.chunk_indices,
|
||||
chunk_offsets=m.chunk_offsets,
|
||||
use_fused_chunk=use_fused_chunk,
|
||||
use_fused_chunk=self.use_fused_chunk,
|
||||
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
|
||||
checkpoint_state=checkpoint_state,
|
||||
checkpoint_offsets=checkpoint_offsets,
|
||||
checkpoint_state_indices=checkpoint_state_indices,
|
||||
)
|
||||
# chunk_kda_prefill updates `recurrent_state` in place, so
|
||||
# there is no final state to write back here.
|
||||
|
||||
@@ -7,16 +7,23 @@ The request classification and cudagraph staging intentionally mirror
|
||||
differently on device rather than on the host.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionBackend,
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import CommonAttentionMetadata
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -86,6 +93,55 @@ def prepare_chunk_metadata_device(
|
||||
|
||||
|
||||
class KimiK3ROCmKDAMetadataBuilder(GDNAttentionMetadataBuilder):
|
||||
def build( # type: ignore[override]
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
num_accepted_tokens: torch.Tensor | None = None,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None = None,
|
||||
fast_build: bool = False,
|
||||
) -> GDNAttentionMetadata:
|
||||
attn_metadata = super().build(
|
||||
common_prefix_len,
|
||||
common_attn_metadata,
|
||||
num_accepted_tokens,
|
||||
num_decode_draft_tokens_cpu,
|
||||
fast_build,
|
||||
)
|
||||
if attn_metadata.checkpoint is None:
|
||||
return attn_metadata
|
||||
# The shared builder keys checkpoint rows on every request; the ROCm
|
||||
# prefill path indexes them by prefill_query_start_loc, so rebuild with
|
||||
# those rows.
|
||||
return replace(
|
||||
attn_metadata,
|
||||
checkpoint=self._build_checkpoint_metadata(
|
||||
common_attn_metadata, attn_metadata, num_decode_draft_tokens_cpu
|
||||
),
|
||||
)
|
||||
|
||||
def _build_checkpoint_metadata(
|
||||
self,
|
||||
m: CommonAttentionMetadata,
|
||||
attn_metadata: GDNAttentionMetadata,
|
||||
num_decode_draft_tokens_cpu: torch.Tensor | None,
|
||||
) -> MambaPrefillCheckpointMetadata | None:
|
||||
if attn_metadata.num_prefills == 0:
|
||||
return None
|
||||
# request_rows must line up with prefill_query_start_loc, either the
|
||||
# prefill tail of a decode-first batch, or every non-spec row
|
||||
# including padding
|
||||
num_decodes = attn_metadata.num_decodes
|
||||
request_rows = list(
|
||||
range(num_decodes, num_decodes + attn_metadata.num_prefills)
|
||||
)
|
||||
if attn_metadata.spec_sequence_masks is not None:
|
||||
assert num_decode_draft_tokens_cpu is not None
|
||||
request_rows = (
|
||||
(num_decode_draft_tokens_cpu < 0).nonzero().flatten().tolist()
|
||||
)
|
||||
return self.checkpoint_builder.build(m, request_rows)
|
||||
|
||||
def _build_chunk_metadata(
|
||||
self,
|
||||
prefill_query_start_loc: torch.Tensor,
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Conv half of the ROCm Kimi-K3 prefill checkpoint."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.kda_checkpoint import (
|
||||
FlashKDAPrefillCheckpointExporter,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class KimiK3ROCmKDAPrefillCheckpointExporter(FlashKDAPrefillCheckpointExporter):
|
||||
"""Conv window only; `fused_kda_chunk` writes the recurrent half itself.
|
||||
|
||||
`state_len` is `conv_kernel_size - 1`, not the spec-widened conv row.
|
||||
"""
|
||||
|
||||
def export( # type: ignore[override]
|
||||
self,
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
*,
|
||||
raw_qkv: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
cu_seqlens: torch.Tensor,
|
||||
) -> None:
|
||||
assert self.state_len is not None
|
||||
# An empty recurrent checkpoint masks the kernel's recurrent half off.
|
||||
super().export(
|
||||
checkpoint,
|
||||
raw_qkv=raw_qkv,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=conv_state[:, :0],
|
||||
recurrent_state=conv_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
)
|
||||
@@ -16,6 +16,7 @@ from vllm.third_party.flash_linear_attention.ops.index import (
|
||||
prepare_chunk_indices,
|
||||
prepare_chunk_offsets,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
CHUNK_SIZE = 64
|
||||
HEAD_DIM = 128
|
||||
@@ -28,6 +29,15 @@ _DEEP_LEN = 12
|
||||
_MAX_GROUPS = 32
|
||||
|
||||
|
||||
def _hip_checkpoint_state_indices(state_indices: torch.Tensor) -> torch.Tensor:
|
||||
"""NULL_BLOCK_ID marks no checkpoint; the HIP kernel skips negative rows."""
|
||||
return (
|
||||
torch.where(state_indices != NULL_BLOCK_ID, state_indices, -1)
|
||||
.to(torch.int32)
|
||||
.contiguous()
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def is_fused_kda_chunk_supported() -> bool:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
@@ -295,7 +305,7 @@ def fused_kda_chunk(
|
||||
None if checkpoint_offsets is None else checkpoint_offsets.to(torch.int32),
|
||||
None
|
||||
if checkpoint_state_indices is None
|
||||
else checkpoint_state_indices.to(torch.int32).contiguous(),
|
||||
else _hip_checkpoint_state_indices(checkpoint_state_indices),
|
||||
state_cache,
|
||||
None if state_indices is None else state_indices.to(torch.int32).contiguous(),
|
||||
has_initial_state,
|
||||
|
||||
@@ -155,7 +155,13 @@ def chunk_kda_prefill(
|
||||
aqk=ws["aqk"],
|
||||
decay=ws["decay"],
|
||||
# v is dead by this point, so it doubles as the output buffer.
|
||||
out=out if out is not None else v,
|
||||
out=out
|
||||
if out is not None
|
||||
else (
|
||||
v
|
||||
if v.is_contiguous()
|
||||
else torch.empty_like(v, memory_format=torch.contiguous_format)
|
||||
),
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
initial_state=initial_state,
|
||||
|
||||
Reference in New Issue
Block a user