Files
28e92eeff2 [ROCm][Perf][Kimi-K3] Use AITER Gluon chunk_kda for KDA prefill (#59552)
Signed-off-by: Omar Muhammad <omar.muhammad@amd.com>
Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
2026-10-16 22:58:56 +00:00

237 lines
8.9 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""KDA prefill backend selection for ROCm.
The Kimi-K3 KDA layer calls :func:`chunk_kda_prefill`, which either runs the
fused HIP kernels in ``kda_chunk`` or falls back to the vendored Triton chunk
path.
"""
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
can_use_fused_kda_chunk,
fused_kda_chunk,
fused_kda_prologue,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import chunk_kda_with_fused_gate
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
logger = init_logger(__name__)
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
def chunk_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
g_bias: torch.Tensor | None = None,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
lower_bound: float | None = None,
use_qk_l2norm_in_kernel: bool = False,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
backend: str = "triton",
out: torch.Tensor | None = None,
checkpoint_state: torch.Tensor | None = None,
checkpoint_offsets: torch.Tensor | None = None,
checkpoint_state_indices: torch.Tensor | None = None,
state_cache: torch.Tensor | None = None,
state_indices: torch.Tensor | None = None,
has_initial_state: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Run chunk KDA from raw gate and beta projections.
Args:
q: query tensor, ``[1, T, H, K]``.
k: key tensor, ``[1, T, H, K]``.
v: value tensor, ``[1, T, H, V]``.
raw_g: raw gate projection, before the activation.
raw_beta: raw beta projection, before the activation.
A_log: log of the per-head gate decay.
g_bias: optional per-head gate bias.
scale: scale applied to the query-key products. Defaults to
``k.shape[-1] ** -0.5``.
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
output_final_state: whether to return the final recurrent state.
lower_bound: optional floor applied to the gate.
use_qk_l2norm_in_kernel: fold the q/k L2 norm into the kernel.
cu_seqlens: int32 cumulative sequence lengths.
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
chunk_offsets: int32 per-sequence first chunk index.
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
two-kernel ROCm path; used only when every one of its
preconditions holds, otherwise the Triton path runs unchanged) or
``"gluon"`` (AITER's ``chunk_kda``).
out: buffer the result must land in. Honoured by both backends, so the
caller can hand in a slice of its own output and skip a copy.
checkpoint_state: destination for mid-prefill recurrent state
snapshots, letting a later prefix-cache hit resume from a mamba
block boundary. See :func:`fused_kda_chunk`.
checkpoint_offsets: per-sequence token offset to snapshot at, ``0``
for none.
checkpoint_state_indices: optional per-sequence destination row.
state_cache: the paged recurrent state. When given, the fused backend
reads and writes it in place and neither a gather nor a scatter is
needed around this call; the returned final state is ``None``.
state_indices: per-sequence cache row.
has_initial_state: per-sequence flag; false starts from a zero state.
Returns:
The output and, when requested, the final recurrent state.
"""
if backend not in ("triton", "fused", "gluon"):
raise ValueError(
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
f"got {backend!r}."
)
if scale is None:
scale = k.shape[-1] ** -0.5
# The fused prologue folds the q/k L2 norm and the gate activation in, so it
# needs the raw projections and a bounded gate rather than the pre-normalized
# tensors the Triton path takes.
fused = (
backend == "fused"
and use_qk_l2norm_in_kernel
and cu_seqlens is not None
and lower_bound is not None
and g_bias is not None
and can_use_fused_kda_chunk(k.shape[-1], v.shape[-1], k.dtype, FLA_CHUNK_SIZE)
)
if checkpoint_offsets is not None and not fused:
raise NotImplementedError(
"The KDA prefill checkpoint export needs kda_prefill_backend=fused for ROCm"
)
# Checked here rather than in the backend so both paths reject the same
# arguments.
if state_cache is not None:
if initial_state is not None or output_final_state:
raise ValueError("state_cache replaces initial_state/output_final_state")
if state_indices is None or has_initial_state is None:
raise ValueError("state_cache needs state_indices and has_initial_state")
if backend == "gluon":
return rocm_aiter_ops.kda_chunk_prefill(
q=q,
k=k,
v=v,
g=raw_g,
beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
scale=scale,
out=out,
initial_state=initial_state,
output_final_state=output_final_state,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
# Only the fused walk addresses the paged rows directly. The Triton path
# gathers the rows it needs and scatters the results back here, so callers
# pass the cache the same way for either backend.
scatter_to: torch.Tensor | None = None
if state_cache is not None and not fused:
initial_state = gather_initial_states(
state_cache, state_indices, has_initial_state
)
output_final_state = True
scatter_to, state_cache = state_cache, None
if fused:
# Restated for the type checker; `fused` already implies all three.
assert cu_seqlens is not None and lower_bound is not None
assert g_bias is not None
logger.info_once(
"Kimi-K3 KDA prefill: dispatching the fused ROCm chunk kernel."
)
ws = fused_kda_prologue(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
scale=scale,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
)
return fused_kda_chunk(
qg=ws["qg"],
w=ws["w"],
u=ws["u"],
kg_t=ws["kg_t"],
aqk=ws["aqk"],
decay=ws["decay"],
# v is dead by this point, so it doubles as the output buffer.
out=out
if out is not None
else (
v
if v.is_contiguous()
else torch.empty_like(v, memory_format=torch.contiguous_format)
),
scale=scale,
cu_seqlens=cu_seqlens,
initial_state=initial_state,
output_final_state=output_final_state,
chunk_offsets=chunk_offsets,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
checkpoint_state_indices=checkpoint_state_indices,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
o, final_state = chunk_kda_with_fused_gate(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
g_bias=g_bias,
scale=scale,
initial_state=initial_state,
output_final_state=output_final_state,
lower_bound=lower_bound,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
if out is not None and o.data_ptr() != out.data_ptr():
out.copy_(o)
o = out
if scatter_to is not None:
assert state_indices is not None
scatter_to[state_indices.long()] = final_state
return o, None
return o, final_state