Revert "[ROCm][Perf] Kimi-K3 Fused kernels for KDA prefill" (#53294)

Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Kevin H. Luu
2026-08-21 13:34:13 -07:00
committed by GitHub
co-authored by Andreas Karatzas
parent 0a3a4f7f35
commit 592e06f2ae
9 changed files with 18 additions and 2716 deletions
+10 -38
View File
@@ -41,16 +41,13 @@ from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
is_fused_kda_chunk_supported,
)
from vllm.models.kimi_k3.amd.ops.kda_decode import (
is_fused_kda_decode_supported,
make_decode_conv1d_weight_loader,
make_decode_norm_weight_loader,
)
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
@@ -213,21 +210,10 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if isinstance(additional_config, dict)
else "auto"
)
assert backend in ("auto", "triton", "fused"), (
"The ROCm Kimi-K3 KDA prefill backend must be one of "
f"'auto', 'triton' or 'fused', got {backend!r}."
)
if backend == "fused" and not is_fused_kda_chunk_supported():
raise RuntimeError(
"The fused KDA chunk kernel requires gfx950 and a build that "
"includes it."
)
self.use_fused_chunk = backend == "fused" or (
backend == "auto" and is_fused_kda_chunk_supported()
)
logger.info_once(
"Kimi-K3 KDA prefill backend: %s",
"fused" if self.use_fused_chunk else "triton",
backend = "triton" if backend == "auto" else backend
assert backend == "triton", (
"The ROCm Kimi-K3 KDA layer only supports the Triton KDA prefill "
f"backend, got {backend!r}."
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
@@ -510,12 +496,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
# prefill tail only.
core_attn_out_decode = None
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
# Without a spec split the non-spec tokens are exactly
# core_attn_out[:, :num_actual_tokens] in order, so both kernels
# can write their slice straight into it. With one, the tokens
# are interleaved and have to be scattered by index afterwards.
direct = spec_sequence_masks is None
use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None
if split_non_spec:
assert non_spec_query_start_loc is not None
nd_tok = m.num_decode_tokens
@@ -533,7 +513,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
ssm_state_indices=non_spec_state_indices_tensor[
: m.num_decodes
],
out=core_attn_out[:, :nd_tok] if direct else None,
)
q_ns = q_ns[:, nd_tok:]
k_ns = k_ns[:, nd_tok:]
@@ -550,7 +529,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
prefill_query_start_loc = non_spec_query_start_loc
prefill_state_indices = non_spec_state_indices_tensor
prefill_has_initial_state = has_initial_state
nd_tok = 0
initial_state = gather_initial_states(
recurrent_state,
@@ -560,7 +538,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_kda_prefill(
) = chunk_kda_with_fused_gate(
q=q_ns,
k=k_ns,
v=v_ns,
@@ -575,16 +553,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
cu_seqlens=prefill_query_start_loc,
chunk_indices=m.chunk_indices,
chunk_offsets=m.chunk_offsets,
use_fused_chunk=use_fused_chunk,
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
)
# Init cache
recurrent_state[prefill_state_indices] = last_recurrent_state
if direct:
# Both slices already landed in core_attn_out.
core_attn_out_non_spec = core_attn_out[:, :num_actual_tokens]
elif split_non_spec:
if split_non_spec:
# Restore decode-first token order for the merge below.
core_attn_out_non_spec = torch.cat(
[core_attn_out_decode, core_attn_out_non_spec], dim=1
@@ -636,10 +609,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
else:
assert core_attn_out_spec is not None
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
-251
View File
@@ -1,251 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
path with a single launch that keeps the per-chunk state in registers, so the
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
"""
from functools import cache
import torch
from vllm.third_party.flash_linear_attention.ops.index import (
prepare_chunk_indices,
prepare_chunk_offsets,
)
CHUNK_SIZE = 64
HEAD_DIM = 128
# Chunk-group split policy constants; see _chunk_groups.
_BLOCKS_P1 = 2 # (kV + kK) / kBV
_BLOCKS_P2 = 1 # kV / kBV
_MIN_LEN = 4
_DEEP_LEN = 12
_MAX_GROUPS = 32
@cache
def is_fused_kda_chunk_supported() -> bool:
from vllm.platforms.rocm import on_gfx950
if not hasattr(torch.ops._C, "fused_kda_chunk"):
return False
# TODO: Verify on other archs, currently only validated on gfx950
return on_gfx950()
def can_use_fused_kda_chunk(
head_k_dim: int,
head_v_dim: int,
dtype: torch.dtype,
chunk_size: int,
) -> bool:
return (
head_k_dim == HEAD_DIM
and head_v_dim == HEAD_DIM
and chunk_size == CHUNK_SIZE
and dtype == torch.bfloat16
and is_fused_kda_chunk_supported()
)
def fused_kda_prologue(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
scale: float,
lower_bound: float,
cu_seqlens: torch.Tensor,
conv_weight: torch.Tensor | None = None,
conv_state: torch.Tensor | None = None,
conv_state_indices: torch.Tensor | None = None,
conv_has_initial_state: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
) -> dict[str, torch.Tensor]:
"""Run the whole chunk-path prologue in one launch.
Replaces the two L2 norms, the gate cumsum, both intra-chunk passes and the
w/u recompute. Returns the operands the fused chunk kernel consumes.
The kernel can also apply the depthwise conv and its silu, in which case
``q``/``k``/``v`` are the three raw bands of the QKV projection and
``conv_state`` is updated in place. This path is currently not used.
"""
# `beta` and `g` reach the layer as last-dim slices of the fused QKVGFAB
# projection and carry its row stride. The kernel reads both with an
# explicit per-token stride, so neither is copied here.
_, t_total, num_heads, _ = q.shape
if chunk_indices is None:
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
chunk_indices = chunk_indices.to(torch.int32)
num_chunks = chunk_indices.shape[0]
dev = q.device
# q may be a strided band view, so the workspaces are sized rather than
# cloned from it.
def _like(last: int) -> torch.Tensor:
return torch.empty(1, t_total, num_heads, last, dtype=q.dtype, device=q.device)
ws = dict(
qg=_like(HEAD_DIM),
w=_like(HEAD_DIM),
u=_like(HEAD_DIM),
kg_t=torch.empty(
num_chunks, num_heads, HEAD_DIM, CHUNK_SIZE, dtype=q.dtype, device=dev
),
aqk=torch.empty(1, t_total, num_heads, CHUNK_SIZE, dtype=q.dtype, device=dev),
decay=torch.empty(
num_chunks, num_heads, HEAD_DIM, dtype=torch.float32, device=dev
),
)
torch.ops._C.fused_kda_prologue(
q,
k,
v,
raw_g,
raw_beta,
A_log.reshape(-1),
dt_bias.reshape(-1),
ws["qg"],
ws["w"],
ws["u"],
ws["kg_t"],
ws["aqk"],
ws["decay"],
cu_seqlens.to(torch.int32),
chunk_indices,
conv_weight,
conv_state,
conv_state_indices,
conv_has_initial_state,
scale,
lower_bound,
)
return ws
@cache
def _num_cus(device: int) -> int:
from vllm.platforms import current_platform
return current_platform.get_device_properties(device).multi_processor_count
@cache
def _chunk_groups(chunks_per_seq: int, num_seqs: int, num_heads: int) -> int:
"""How many parallel chunk groups to cut each sequence into.
The scan composes the group operators with the exact transfer
``M_g = prod_c (diag(d_c) - w_c^T kg_c)``, so the composition error does
not grow with group length and the choice is purely a machine fit:
* ``fill1``/``fill2``: the largest G whose pass-one / pass-two grid still
fits in one scheduling batch. Rows are launched in blocks of at most
``kBV = 128``, so pass one (``kV + kK`` rows) needs two blocks per
``(n, h, g)`` and pass two one. Past a pass's fill point its block count
grows as fast as its depth shrinks and the rung stops paying.
* ``_DEEP_LEN``: unless the groups are still long, where pass two's halving
is worth letting pass one spill to a second batch.
* ``_MIN_LEN``: below this many chunks per group the per-group fixed cost
exceeds what the shorter walk saves.
``G == 2`` is depth-neutral -- two passes of ``nt/2`` -- and only doubles
staging, so the split is taken only from ``G >= 4``.
"""
cus = _num_cus(torch.accelerator.current_device_index())
nh = max(num_seqs * num_heads, 1)
fill1 = cus // (_BLOCKS_P1 * nh)
fill2 = cus // (_BLOCKS_P2 * nh)
cand = fill1
if fill2 > 0 and chunks_per_seq // fill2 >= _DEEP_LEN:
cand = fill2
cand = min(cand, chunks_per_seq // _MIN_LEN, _MAX_GROUPS)
return cand if cand >= 4 else 1
def _kda_group_workspace(
groups: int, nh: int, device: torch.device
) -> torch.Tensor | None:
"""One fp32 buffer the kernel carves into bg / sin_ / ag / mgT.
Kept caller-owned so the memory stays inside the caching allocator and
vLLM's memory profiling accounts for it; the kernel's carve order must
match the layout here.
"""
if groups <= 1:
return None
planes = groups * nh
plane = HEAD_DIM * HEAD_DIM
floats = 2 * planes * plane + planes * HEAD_DIM # bg, sin_, ag
floats += (planes * plane + 1) // 2 # mgT, bf16
return torch.empty(floats, dtype=torch.float32, device=device)
def fused_kda_chunk(
qg: torch.Tensor,
w: torch.Tensor,
u: torch.Tensor,
kg_t: torch.Tensor,
aqk: torch.Tensor,
decay: torch.Tensor,
out: torch.Tensor,
scale: float,
cu_seqlens: torch.Tensor,
initial_state: torch.Tensor | None,
output_final_state: bool,
chunk_offsets: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Run the chunk recurrence and the output projection in one launch.
Args:
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
"""
num_seqs = cu_seqlens.numel() - 1
final_state = None
if output_final_state:
final_state = torch.empty(
num_seqs,
u.shape[2],
u.shape[3],
qg.shape[3],
dtype=torch.float32,
device=u.device,
)
cu_seqlens = cu_seqlens.to(torch.int32)
if chunk_offsets is None:
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
# The chunk walk is serial in chunks. Splitting each sequence into `groups`
# stretches that run in parallel costs a second pass and a scan, and only
# pays once the chain is long enough to amortise both.
chunks_per_seq = kg_t.shape[0] // max(num_seqs, 1)
groups = _chunk_groups(chunks_per_seq, num_seqs, u.shape[2])
group_state = _kda_group_workspace(groups, num_seqs * u.shape[2], u.device)
torch.ops._C.fused_kda_chunk(
qg,
w,
u,
kg_t,
aqk,
decay,
initial_state,
final_state,
out,
cu_seqlens,
chunk_offsets,
scale,
group_state,
groups,
)
return out, final_state
-126
View File
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""KDA prefill backend selection for ROCm.
The Kimi-K3 KDA layer calls :func:`chunk_kda_prefill`, which either runs the
fused HIP kernels in ``kda_chunk`` or falls back to the vendored Triton chunk
path.
"""
import torch
from vllm.logger import init_logger
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
can_use_fused_kda_chunk,
fused_kda_chunk,
fused_kda_prologue,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import chunk_kda_with_fused_gate
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
logger = init_logger(__name__)
def chunk_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
g_bias: torch.Tensor | None = None,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
lower_bound: float | None = None,
use_qk_l2norm_in_kernel: bool = False,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_fused_chunk: bool = False,
out: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Run chunk KDA from raw gate and beta projections.
Args:
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
out: buffer the result must land in. Honoured by both backends, so the
caller can hand in a slice of its own output and skip a copy.
Returns:
The output and, when requested, the final recurrent state.
"""
if scale is None:
scale = k.shape[-1] ** -0.5
# The fused prologue folds the q/k L2 norm and the gate activation in, so it
# needs the raw projections and a bounded gate rather than the pre-normalized
# tensors the Triton path takes.
fused = (
use_fused_chunk
and use_qk_l2norm_in_kernel
and cu_seqlens is not None
and lower_bound is not None
and g_bias is not None
and can_use_fused_kda_chunk(k.shape[-1], v.shape[-1], k.dtype, FLA_CHUNK_SIZE)
)
if fused:
# Restated for the type checker; `fused` already implies all three.
assert cu_seqlens is not None and lower_bound is not None
assert g_bias is not None
logger.info_once(
"Kimi-K3 KDA prefill: dispatching the fused ROCm chunk kernel."
)
ws = fused_kda_prologue(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
scale=scale,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
)
return fused_kda_chunk(
qg=ws["qg"],
w=ws["w"],
u=ws["u"],
kg_t=ws["kg_t"],
aqk=ws["aqk"],
decay=ws["decay"],
# v is dead by this point, so it doubles as the output buffer.
out=out if out is not None else v,
scale=scale,
cu_seqlens=cu_seqlens,
initial_state=initial_state,
output_final_state=output_final_state,
chunk_offsets=chunk_offsets,
)
o, final_state = chunk_kda_with_fused_gate(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
g_bias=g_bias,
scale=scale,
initial_state=initial_state,
output_final_state=output_final_state,
lower_bound=lower_bound,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
if out is not None and o.data_ptr() != out.data_ptr():
out.copy_(o)
o = out
return o, final_state