[ROCm][Perf][Kimi-K3] Use AITER Gluon chunk_kda for KDA prefill (#59552)

Signed-off-by: Omar Muhammad <omar.muhammad@amd.com>
Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Omar Muhammad
2026-10-16 22:58:56 +00:00
committed by GitHub
co-authored by Nicholas Susanto Douglas Lehr Kevin H. Luu
parent b088eba944
commit 28e92eeff2
6 changed files with 197 additions and 20 deletions
+13 -9
View File
@@ -95,7 +95,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
assert isinstance(spec, MambaSpec)
# Only the fused path exports the checkpoint snapshots
can_checkpoint = (
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
self.use_prefill_checkpoint
and self.kda_prefill_backend == "fused"
and self.use_safe_gate
)
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
@@ -242,13 +244,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
"The fused KDA chunk kernel requires gfx950 and a build that "
"includes it."
)
self.use_fused_chunk = backend == "fused" or (
backend == "auto" and is_fused_kda_chunk_supported()
)
logger.info_once(
"Kimi-K3 KDA prefill backend: %s",
"fused" if self.use_fused_chunk else "triton",
)
self.kda_prefill_backend = backend
if backend == "auto":
if self.use_safe_gate and rocm_aiter_ops.is_kda_gluon_prefill_available():
self.kda_prefill_backend = "gluon"
elif is_fused_kda_chunk_supported():
self.kda_prefill_backend = "fused"
else:
self.kda_prefill_backend = "triton"
logger.info_once("Kimi-K3 KDA prefill backend: %s", self.kda_prefill_backend)
# TODO: Remove the extra config arg once tested
self.use_prefill_checkpoint = (
additional_config.get("kda_prefill_checkpoint", True)
@@ -694,7 +698,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
cu_seqlens=prefill_query_start_loc,
chunk_indices=m.chunk_indices,
chunk_offsets=m.chunk_offsets,
use_fused_chunk=self.use_fused_chunk,
backend=self.kda_prefill_backend,
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
+39 -5
View File
@@ -9,6 +9,7 @@ path.
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
@@ -24,6 +25,11 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
logger = init_logger(__name__)
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
def chunk_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
@@ -40,7 +46,7 @@ def chunk_kda_prefill(
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_fused_chunk: bool = False,
backend: str = "triton",
out: torch.Tensor | None = None,
checkpoint_state: torch.Tensor | None = None,
checkpoint_offsets: torch.Tensor | None = None,
@@ -68,9 +74,10 @@ def chunk_kda_prefill(
cu_seqlens: int32 cumulative sequence lengths.
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
chunk_offsets: int32 per-sequence first chunk index.
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
two-kernel ROCm path; used only when every one of its
preconditions holds, otherwise the Triton path runs unchanged) or
``"gluon"`` (AITER's ``chunk_kda``).
out: buffer the result must land in. Honoured by both backends, so the
caller can hand in a slice of its own output and skip a copy.
checkpoint_state: destination for mid-prefill recurrent state
@@ -89,6 +96,11 @@ def chunk_kda_prefill(
The output and, when requested, the final recurrent state.
"""
if backend not in ("triton", "fused", "gluon"):
raise ValueError(
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
f"got {backend!r}."
)
if scale is None:
scale = k.shape[-1] ** -0.5
@@ -96,7 +108,7 @@ def chunk_kda_prefill(
# needs the raw projections and a bounded gate rather than the pre-normalized
# tensors the Triton path takes.
fused = (
use_fused_chunk
backend == "fused"
and use_qk_l2norm_in_kernel
and cu_seqlens is not None
and lower_bound is not None
@@ -116,6 +128,28 @@ def chunk_kda_prefill(
if state_indices is None or has_initial_state is None:
raise ValueError("state_cache needs state_indices and has_initial_state")
if backend == "gluon":
return rocm_aiter_ops.kda_chunk_prefill(
q=q,
k=k,
v=v,
g=raw_g,
beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
scale=scale,
out=out,
initial_state=initial_state,
output_final_state=output_final_state,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
# Only the fused walk addresses the paged rows directly. The Triton path
# gathers the rows it needs and scatters the results back here, so callers
# pass the cache the same way for either backend.