[ROCm][Perf][Kimi-K3] Use AITER Gluon chunk_kda for KDA prefill (#59552)

Signed-off-by: Omar Muhammad <omar.muhammad@amd.com>
Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Omar Muhammad
2026-10-16 22:58:56 +00:00
committed by GitHub
co-authored by Nicholas Susanto Douglas Lehr Kevin H. Luu
parent b088eba944
commit 28e92eeff2
6 changed files with 197 additions and 20 deletions
@@ -19,9 +19,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_kda_layer(worker: Worker) -> KimiK3DeltaAttention | None:
def _get_kda_layer(
worker: Worker, module: str = "vllm.models.kimi_k3.nvidia.kda"
) -> KimiK3DeltaAttention | None:
# Kimi model construction already imports kda. Avoid importing it here.
kda = sys.modules.get("vllm.models.kimi_k3.nvidia.kda")
kda = sys.modules.get(module)
if kda is None:
return None
@@ -172,9 +174,59 @@ def _warm_recurrent_kda(
)
def _warm_gluon_chunk_kda(layer, input_dtype: torch.dtype) -> None:
from vllm.models.kimi_k3.amd.ops.kda_prefill import (
chunk_kda_prefill,
get_gluon_chunk_kda_warmup_profiles,
)
if layer.kda_prefill_backend != "gluon":
return
kv_cache = layer.kv_cache
if not isinstance(kv_cache, (list, tuple)) or len(kv_cache) < 2:
return
state = kv_cache[1]
if not isinstance(state, torch.Tensor) or not state.numel():
return
logger.info("Warming up Kimi-K3 Gluon chunk KDA prefill kernels.")
h, d = int(layer.local_num_heads), int(layer.head_dim)
for num_seqs, seq_len in get_gluon_chunk_kda_warmup_profiles():
num_tokens = num_seqs * seq_len
shape = (1, num_tokens, h, d)
chunk_kda_prefill(
q=torch.zeros(shape, dtype=input_dtype, device=state.device),
k=torch.zeros(shape, dtype=input_dtype, device=state.device),
v=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_g=torch.zeros(shape, dtype=input_dtype, device=state.device),
raw_beta=torch.zeros(
(1, num_tokens, h), dtype=input_dtype, device=state.device
),
A_log=layer.A_log,
g_bias=layer.dt_bias,
lower_bound=layer.gate_lower_bound,
state_cache=state,
state_indices=torch.zeros(num_seqs, dtype=torch.int32, device=state.device),
has_initial_state=torch.zeros(
num_seqs, dtype=torch.bool, device=state.device
),
use_qk_l2norm_in_kernel=True,
cu_seqlens=torch.arange(
0, num_tokens + 1, seq_len, dtype=torch.int32, device=state.device
),
backend="gluon",
out=torch.empty(shape, dtype=input_dtype, device=state.device),
)
@torch.inference_mode()
def kimi_k3_triton_warmup(worker: Worker) -> None:
"""Warm Kimi-K3 Triton kernels reachable by this server."""
if current_platform.is_rocm():
layer = _get_kda_layer(worker, "vllm.models.kimi_k3.amd.kda")
if layer is not None:
_warm_gluon_chunk_kda(layer, worker.model_config.dtype)
return
if not current_platform.is_cuda():
return