[ROCm][Perf] Kimi-K3 Fused kernels for KDA prefill reland (#54038)

Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Signed-off-by: kliuae <17350011+kliuae@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
kliuae
2026-09-10 15:37:54 +08:00
committed by GitHub
co-authored by Claude TJian
parent be1cb9834b
commit 9521c60bdc
9 changed files with 3818 additions and 33 deletions
+21 -5
View File
@@ -1181,11 +1181,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
endif()
# HIP counterpart of the FUSED_KDA_DECODE block above: same `fused_kda_decode`
# op, same VLLM_ENABLE_FUSED_KDA_DECODE guard in torch_bindings.cpp. Built for
# the CDNA archs the kernel supports (gfx942 / gfx950), so it is only emitted
# when one of those is in the arch list. A multi-arch build that includes them
# still emits this source for every arch in that list. The runtime gate in
# kda_decode.py keeps the kernel off unsupported devices.
# op, same VLLM_ENABLE_FUSED_KDA_DECODE guard in torch_bindings.cpp
# Built for the CDNA archs the kernel supports
# (gfx942 / gfx950), so it is only emitted when one of those is in the arch
# list. A multi-arch build that includes them still emits this source for
# every arch in that list. The runtime gate in kda_decode.py keeps the kernel
# off unsupported devices. Fused prefill KDA kernels is Only measured on
# gfx950, so it is only built when gfx950 is in the arch list.
if(VLLM_GPU_LANG STREQUAL "HIP")
set(FUSED_KDA_DECODE_HIP_ARCHS ${VLLM_GPU_ARCHES})
list(FILTER FUSED_KDA_DECODE_HIP_ARCHS INCLUDE REGEX "gfx942|gfx950")
@@ -1196,6 +1198,16 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
message(STATUS
"Building fused KDA decode for HIP archs: ${FUSED_KDA_DECODE_HIP_ARCHS}")
endif()
set(FUSED_KDA_CHUNK_HIP_ARCHS ${VLLM_GPU_ARCHES})
list(FILTER FUSED_KDA_CHUNK_HIP_ARCHS INCLUDE REGEX "gfx950")
if(FUSED_KDA_CHUNK_HIP_ARCHS)
set(FUSED_KDA_CHUNK_HIP_SRC
"csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu")
list(APPEND VLLM_STABLE_EXT_SRC "${FUSED_KDA_CHUNK_HIP_SRC}")
message(STATUS
"Building fused KDA chunk for HIP archs: ${FUSED_KDA_CHUNK_HIP_ARCHS}")
endif()
endif()
message(STATUS "Enabling C_stable extension.")
@@ -1246,6 +1258,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_FUSED_KDA_DECODE=1)
endif()
if(FUSED_KDA_CHUNK_HIP_SRC)
target_compile_definitions(_C_stable_libtorch PRIVATE
VLLM_ENABLE_FUSED_KDA_CHUNK=1)
endif()
endif()
# On ROCm, _C_stable_libtorch calls raw HIP APIs (e.g. hipGetDevice in
File diff suppressed because it is too large Load Diff
+33
View File
@@ -406,6 +406,39 @@ void fused_gdn_decode_post_conv_mtp(
#endif
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
void fused_kda_prologue(
torch::stable::Tensor const& q, torch::stable::Tensor const& k,
torch::stable::Tensor const& v, torch::stable::Tensor const& raw_g,
torch::stable::Tensor const& raw_beta, torch::stable::Tensor const& a_log,
torch::stable::Tensor const& dt_bias, torch::stable::Tensor& qg,
torch::stable::Tensor& w, torch::stable::Tensor& u,
torch::stable::Tensor& kg_t, torch::stable::Tensor& aqk,
torch::stable::Tensor& decay, torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& chunk_indices,
std::optional<torch::stable::Tensor> conv_weight,
std::optional<torch::stable::Tensor> conv_state,
std::optional<torch::stable::Tensor> conv_state_indices,
std::optional<torch::stable::Tensor> conv_has_initial_state, double scale,
double lower_bound);
void fused_kda_chunk(
torch::stable::Tensor const& qg, torch::stable::Tensor const& w,
torch::stable::Tensor const& u, torch::stable::Tensor const& kg_t,
torch::stable::Tensor const& aqk, torch::stable::Tensor const& decay,
std::optional<torch::stable::Tensor> initial_state,
std::optional<torch::stable::Tensor> final_state,
torch::stable::Tensor& out, torch::stable::Tensor const& cu_seqlens,
torch::stable::Tensor const& chunk_offsets, double scale,
std::optional<torch::stable::Tensor> group_state, int64_t groups,
std::optional<torch::stable::Tensor> checkpoint_state,
std::optional<torch::stable::Tensor> checkpoint_offsets,
std::optional<torch::stable::Tensor> checkpoint_state_indices,
std::optional<torch::stable::Tensor> state_cache,
std::optional<torch::stable::Tensor> state_indices,
std::optional<torch::stable::Tensor> has_initial_state);
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
std::optional<torch::stable::Tensor> delta,
+25
View File
@@ -537,6 +537,26 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
"str output_gate_activation='silu') -> ()");
#endif
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
ops.def(
"fused_kda_prologue("
"Tensor q, Tensor k, Tensor v, Tensor raw_g, Tensor raw_beta, "
"Tensor A_log, Tensor dt_bias, Tensor! qg, Tensor! w, Tensor! u, "
"Tensor! kg_t, Tensor! aqk, Tensor! decay, Tensor cu_seqlens, "
"Tensor chunk_indices, Tensor? conv_weight, Tensor(e!)? conv_state, "
"Tensor? conv_state_indices, Tensor? conv_has_initial_state, "
"float scale, float lower_bound) -> ()");
ops.def(
"fused_kda_chunk("
"Tensor qg, Tensor w, Tensor u, Tensor kg_t, Tensor aqk, Tensor decay, "
"Tensor? initial_state, Tensor(a!)? final_state, Tensor! out, "
"Tensor cu_seqlens, Tensor chunk_offsets, float scale, "
"Tensor(b!)? group_state, int groups, "
"Tensor(c!)? checkpoint_state=None, Tensor? checkpoint_offsets=None, "
"Tensor? checkpoint_state_indices=None, Tensor(d!)? state_cache=None, "
"Tensor? state_indices=None, Tensor? has_initial_state=None) -> ()");
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
ops.def(
"kimi_k3_attn_res("
@@ -825,6 +845,11 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
TORCH_BOX(&fused_gdn_decode_post_conv_mtp));
#endif
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
ops.impl("fused_kda_prologue", TORCH_BOX(&fused_kda_prologue));
ops.impl("fused_kda_chunk", TORCH_BOX(&fused_kda_chunk));
#endif
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
#endif
+859
View File
@@ -0,0 +1,859 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The fused ROCm KDA chunk kernel must match the Triton chunk path.
The fused kernel reassociates nothing, but it keeps the chunk states in
registers and rounds them to bfloat16 only where the Triton path does, so the
outputs agree to bfloat16 rounding and the final states nearly exactly.
"""
import pytest
import torch
from vllm.platforms import current_platform
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx950
return on_gfx950()
pytestmark = pytest.mark.skipif(
not _on_gfx950(),
reason="The fused KDA chunk kernel is only built for gfx950",
)
HEAD_DIM = 128
NUM_HEADS = 12
LOWER_BOUND = -5.0
def _requires_kernel() -> None:
if not hasattr(torch.ops._C, "fused_kda_chunk"):
pytest.skip("vLLM was built without the fused KDA chunk kernel")
def _inputs(seqlens: list[int], seed: int) -> dict:
torch.manual_seed(seed)
dev, dtype = "cuda", torch.bfloat16
total = sum(seqlens)
cu = torch.tensor(
[0] + torch.tensor(seqlens).cumsum(0).tolist(), device=dev, dtype=torch.int32
)
shape = (1, total, NUM_HEADS, HEAD_DIM)
return dict(
q=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
k=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
raw_g=torch.randn(shape, device=dev, dtype=dtype),
raw_beta=torch.randn(1, total, NUM_HEADS, device=dev, dtype=dtype),
A_log=torch.randn(NUM_HEADS, device=dev, dtype=torch.float32) * 0.5,
dt_bias=torch.randn(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32)
* 0.5,
h0=torch.randn(
len(seqlens), NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
)
* 0.1,
cu=cu,
)
def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
return chunk_kda_prefill(
q=inp["q"].clone(),
k=inp["k"].clone(),
v=inp["v"].clone(),
raw_g=inp["raw_g"].clone(),
raw_beta=inp["raw_beta"],
A_log=inp["A_log"],
g_bias=inp["dt_bias"],
lower_bound=LOWER_BOUND,
initial_state=inp["h0"].clone(),
output_final_state=True,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
use_fused_chunk=use_fused,
out=out,
**kw,
)
def _force_groups(monkeypatch, groups: int) -> None:
"""Pin the group count, which is otherwise a function of the device."""
import vllm.models.kimi_k3.amd.ops.kda_chunk as kda_chunk
monkeypatch.setattr(kda_chunk, "_chunk_groups", lambda *_: groups)
@pytest.mark.parametrize(
"seqlens",
[
[1], # shorter than a chunk
[64], # exactly one chunk
[1024],
[4096], # many chunks, one sequence
[513, 64, 1, 1200], # ragged, including a partial tail chunk
[512] * 6,
],
)
def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
_requires_kernel()
inp = _inputs(seqlens, seed=len(seqlens))
o_ref, state_ref = _run(inp, use_fused=False)
o_fused, state_fused = _run(inp, use_fused=True)
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize("use_fused", [False, True])
def test_out_argument_receives_the_result(use_fused: bool) -> None:
"""``out`` is a promise about where the result lands.
The layer passes a slice of its own output buffer so neither path has to
copy afterwards, and it then skips the copy based on the returned tensor
aliasing that buffer. Both must hold for the fused path and for the Triton
fallback, which allocates its own buffer internally.
"""
if use_fused:
_requires_kernel()
seqlens = [321, 64, 700]
inp = _inputs(seqlens, seed=13)
o_ref, _ = _run(inp, use_fused=use_fused)
# A non-zero offset into a larger buffer, as the layer produces when a
# mixed batch puts decode tokens first.
total = sum(seqlens)
buf = torch.empty(
1, total + 8, NUM_HEADS, HEAD_DIM, device="cuda", dtype=o_ref.dtype
)
view = buf[:, 8:]
o_got, _ = _run(inp, use_fused=use_fused, out=view)
assert o_got.data_ptr() == view.data_ptr()
torch.testing.assert_close(view.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
def test_fused_chunk_accepts_projection_views() -> None:
"""`beta` and `g` arrive as last-dim slices of the fused QKVGFAB projection.
Those views carry the projection's row stride, so a kernel that indexes them
densely reads the wrong elements or rejects the tensor outright. Build the
inputs the way the layer does rather than as standalone tensors.
"""
_requires_kernel()
seqlens = [777, 64, 391]
inp = _inputs(seqlens, seed=7)
total = sum(seqlens)
# One wide projection, split on the last dim exactly like KimiK3DeltaAttention.
qkv_width = 3 * NUM_HEADS * HEAD_DIM
projection = torch.randn(
total, qkv_width + NUM_HEADS, device="cuda", dtype=torch.bfloat16
)
beta_view = projection.split([qkv_width, NUM_HEADS], dim=-1)[1].unsqueeze(0)
assert not beta_view.is_contiguous()
inp["raw_beta"] = beta_view.contiguous()
o_ref, state_ref = _run(inp, use_fused=False)
inp["raw_beta"] = beta_view
o_fused, state_fused = _run(inp, use_fused=True)
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
def test_fused_chunk_survives_correlated_keys_and_real_gate_bias() -> None:
"""The input regime the served model actually produces.
This is what forces the intra-chunk inverse to keep fp32 MFMA operands: at
bf16 the Neumann doubling's intermediate powers cancel catastrophically
here, diverging by orders of magnitude and driving the final state to inf.
Three properties of real traffic combine here and in none of the tests
above, all measured from tensors captured off a live TP8 server:
* repetitive text gives near-duplicate L2-normalized keys, so ``k_i . k_j``
approaches 1 and the Neumann terms stop shrinking;
* the checkpoint's ``dt_bias`` lies in ``[-7.8, -1.4]``, which makes the
within-chunk gate cumsum span ~300 powers of two;
* ``beta`` saturates near 0.95.
``_inputs`` draws all three from ``randn``, landing in the benign regime.
"""
_requires_kernel()
torch.manual_seed(11)
dev, dtype = "cuda", torch.bfloat16
total = 1088
shape = (1, total, NUM_HEADS, HEAD_DIM)
base = torch.randn(1, 1, NUM_HEADS, HEAD_DIM, device=dev, dtype=dtype)
def correlated():
return (base + torch.randn(shape, device=dev, dtype=dtype) * 0.02).contiguous()
inp = dict(
q=correlated(),
k=correlated(),
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
raw_g=torch.randn(shape, device=dev, dtype=dtype),
raw_beta=torch.full((1, total, NUM_HEADS), 3.0, device=dev, dtype=dtype),
A_log=torch.rand(NUM_HEADS, device=dev, dtype=torch.float32) * 0.6 - 0.42,
dt_bias=torch.rand(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32) * 6.4
- 7.8,
h0=torch.zeros(
1, NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
),
cu=torch.tensor([0, total], device=dev, dtype=torch.int32),
)
o_ref, state_ref = _run(inp, use_fused=False)
o_fused, state_fused = _run(inp, use_fused=True)
# Looser than the tests above: near-identical keys plus a saturated beta
# are past anything real text produces, and the inverse is rounded to bf16
# before the w/u GEMMs. The tolerance still catches divergence, which in
# this regime is orders of magnitude rather than a few times the bound.
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=5e-2, atol=5e-2)
torch.testing.assert_close(state_fused, state_ref, rtol=5e-2, atol=5e-2)
@pytest.mark.parametrize(
"seqlens",
[
[1], # every conv tap comes from the cache
[2], # cache and batch both contribute
[64],
[1024],
[513, 64, 1, 1200], # ragged, tail chunk shorter than the conv width
],
)
@pytest.mark.parametrize("has_state", [False, True])
def test_fused_conv_matches_reference_conv(seqlens: list[int], has_state: bool) -> None:
"""The prologue's optional fused convolution must match a separate conv.
The layer does not use this path -- recomputing the conv once per load
phase costs more than the ``causal_conv1d_fn`` launches it saves -- so it
is driven through the ops directly rather than through
``chunk_kda_prefill``, which does not expose it.
The reference is computed in torch rather than with ``causal_conv1d_fn``,
which is not self-consistent for these shapes on ROCm (it leaves part of
its output unwritten, so repeated identical calls disagree).
The cases that matter are the ones touching the sequence boundary, where
the fused path takes taps from the conv cache instead of the batch:
sequences shorter than the kernel width, and a cache that is or is not
populated. The rolled-forward cache is checked too, since a wrong shift
only shows up on the next step.
"""
import torch.nn.functional as F
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
fused_kda_chunk,
fused_kda_prologue,
)
_requires_kernel()
dev, dtype = "cuda", torch.bfloat16
lp = NUM_HEADS * HEAD_DIM
width, n = 4, len(seqlens)
inp = _inputs(seqlens, seed=7 + len(seqlens) + int(has_state))
total = int(inp["cu"][-1])
torch.manual_seed(11 + int(has_state))
# The layer hands the prologue a strided view of the QKVGFAB projection.
projection = torch.randn(total, 3 * lp + 64, device=dev, dtype=dtype) * 0.5
mixed = projection[:, : 3 * lp]
# conv1d is built with bias=False and fp32 weights; the fused kernel takes
# the width-major [qkv, width, channel] mirror the decode path already
# keeps.
weight = torch.randn(3, width, lp, device=dev, dtype=torch.float32) * 0.3
store = torch.zeros(n, width - 1, 3 * lp, device=dev, dtype=dtype)
if has_state:
store.normal_(0.0, 0.5)
state = store.transpose(-1, -2)
idx = torch.arange(n, device=dev, dtype=torch.int32)
has_init = torch.full((n,), has_state, device=dev, dtype=torch.bool)
# Reference conv, per sequence, in fp32 then rounded like a bf16 output.
conv_ref = torch.empty(total, 3 * lp, device=dev, dtype=dtype)
state_ref = torch.empty_like(store)
flat_w = weight.permute(0, 2, 1).reshape(3 * lp, width)
for i, (b, e) in enumerate(zip(inp["cu"][:-1].tolist(), inp["cu"][1:].tolist())):
pre = (
store[i].float()
if has_state
else torch.zeros(width - 1, 3 * lp, device=dev, dtype=torch.float32)
)
xs = torch.cat([pre, mixed[b:e].float()], 0)
acc = torch.zeros(e - b, 3 * lp, device=dev, dtype=torch.float32)
for w in range(width):
acc += xs[w : w + (e - b)] * flat_w[:, w].unsqueeze(0)
conv_ref[b:e] = F.silu(acc).to(dtype)
state_ref[i] = xs[-(width - 1) :].to(dtype)
def _bands(x, contiguous):
out = [
b.unsqueeze(0).unflatten(-1, (NUM_HEADS, HEAD_DIM))
for b in x.split(lp, dim=-1)
]
return [b.contiguous() for b in out] if contiguous else out
# Compare through both kernels rather than the prologue workspaces: `aqk`
# is deliberately left unwritten past each chunk's valid rows, so those
# elements are whatever the allocator last held.
scale = HEAD_DIM**-0.5
def _through_kernels(q, k, v, conv: dict | None):
ws = fused_kda_prologue(
q=q,
k=k,
v=v,
raw_g=inp["raw_g"],
raw_beta=inp["raw_beta"],
A_log=inp["A_log"],
dt_bias=inp["dt_bias"],
scale=scale,
lower_bound=LOWER_BOUND,
cu_seqlens=inp["cu"],
**(conv or {}),
)
return fused_kda_chunk(
qg=ws["qg"],
w=ws["w"],
u=ws["u"],
kg_t=ws["kg_t"],
aqk=ws["aqk"],
decay=ws["decay"],
out=torch.empty_like(ws["u"]),
scale=scale,
cu_seqlens=inp["cu"],
initial_state=inp["h0"].clone(),
output_final_state=True,
)
qr, kr, vr = _bands(conv_ref, True)
o_ref, s_ref = _through_kernels(qr, kr, vr, None)
got_state = state.clone()
qf, kf, vf = _bands(mixed, False)
o_got, s_got = _through_kernels(
qf,
kf,
vf,
dict(
conv_weight=weight,
conv_state=got_state,
conv_state_indices=idx,
conv_has_initial_state=has_init,
),
)
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(s_got, s_ref, rtol=1e-3, atol=1e-3)
torch.testing.assert_close(
got_state.transpose(-1, -2).float(), state_ref.float(), rtol=0, atol=0
)
@pytest.mark.parametrize("groups", [2, 4, 8])
@pytest.mark.parametrize("seqlens", [[64], [64, 64], [100, 64], [513, 64, 1]])
def test_chunk_group_scan_matches_serial_walk(
groups: int, seqlens: list[int], monkeypatch
) -> None:
"""Splitting the chunk walk into parallel groups must not change the result.
Two bugs lived here, both invisible while the path was gated to >= 128
chunks per sequence:
* a group with no chunks returned before writing its ``B_g``, so the scan
folded whatever the workspace last held into the final state — these
shapes have fewer chunks than groups, so most groups are empty;
* the second pass started every group after the first from a zero state
instead of the one the scan composed, which only looked right because a
strongly decaying gate makes the dropped term small.
"""
_requires_kernel()
inp = _inputs(seqlens, seed=5 + groups)
_force_groups(monkeypatch, 1)
o_ref, state_ref = _run(inp, use_fused=True)
_force_groups(monkeypatch, groups)
o_got, state_got = _run(inp, use_fused=True)
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
@pytest.mark.parametrize("seqlens", [[320], [513, 64, 1, 1200], [64, 64, 64]])
def test_accepts_chunk_metadata_from_the_attention_builder(
seqlens: list[int],
) -> None:
"""The layer passes precomputed chunk metadata, not None.
``KimiK3ROCmKDAMetadataBuilder`` builds these on device to avoid a blocking
device->host copy. Passing ``None`` instead takes a different path that
builds them locally, so drive the op with the builder's own tensors.
"""
_requires_kernel()
from vllm.models.kimi_k3.amd.kda_metadata import prepare_chunk_metadata_device
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
inp = _inputs(seqlens, seed=17)
cu = inp["cu"]
chunk_indices, chunk_offsets = prepare_chunk_metadata_device(
cu, cu.cpu(), FLA_CHUNK_SIZE
)
o_ref, state_ref = _run(inp, use_fused=True)
o_got, state_got = _run(
inp, use_fused=True, chunk_indices=chunk_indices, chunk_offsets=chunk_offsets
)
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
def _checkpoint_buffer(rows: int) -> torch.Tensor:
"""A destination filled with NaN, so an unwritten row is unmistakable."""
return torch.full(
(rows, NUM_HEADS, HEAD_DIM, HEAD_DIM),
float("nan"),
device="cuda",
dtype=torch.float32,
)
def _prefix_inputs(inp: dict, seq: int, num_tokens: int) -> dict:
"""Sequence ``seq`` truncated to its first ``num_tokens`` tokens."""
bos = int(inp["cu"][seq])
tokens = slice(bos, bos + num_tokens)
prefix = dict(inp)
for key in ("q", "k", "v", "raw_g", "raw_beta"):
prefix[key] = inp[key][:, tokens].contiguous()
prefix["h0"] = inp["h0"][seq : seq + 1].contiguous()
prefix["cu"] = torch.tensor([0, num_tokens], device="cuda", dtype=torch.int32)
return prefix
def _run_with_checkpoints(
inp: dict, offsets: list[int], rows: int | None = None, **kw
) -> tuple[torch.Tensor, tuple]:
ckpt = _checkpoint_buffer(len(offsets) if rows is None else rows)
result = _run(
inp,
use_fused=True,
checkpoint_state=ckpt,
checkpoint_offsets=torch.tensor(offsets, device="cuda", dtype=torch.int32),
**kw,
)
return ckpt, result
@pytest.mark.parametrize(
"seqlens,offsets",
[
([256], [64]), # the first interior boundary
([256], [192]), # the last one
([1024], [512]),
([320, 320], [64, 256]), # every sequence checkpoints
([513, 64, 1, 1200], [448, 0, 0, 640]), # ragged, most opted out
],
)
def test_checkpoint_matches_a_prefill_that_stops_at_the_offset(
seqlens: list[int], offsets: list[int], monkeypatch
) -> None:
"""A checkpoint is only useful if resuming from it is indistinguishable.
The scheduler serves the exported state on a later prefix-cache hit as if
the prompt had been prefilled up to that block and no further, so the
kernel has to hand back exactly what a prefill truncated there produces --
not merely something close. The walk passes through that state anyway, so
equality is bit-exact and the test asserts it as such.
Sequences with offset 0 opt out, and their rows must stay untouched.
"""
_requires_kernel()
_force_groups(monkeypatch, 1)
inp = _inputs(seqlens, seed=23 + len(seqlens))
ckpt, _ = _run_with_checkpoints(inp, offsets)
for i, offset in enumerate(offsets):
if offset == 0:
assert ckpt[i].isnan().all()
continue
_, truncated = _run(_prefix_inputs(inp, i, offset), use_fused=True)
torch.testing.assert_close(ckpt[i], truncated[0], rtol=0, atol=0)
def test_checkpoint_leaves_the_output_and_final_state_alone() -> None:
"""Exporting a checkpoint must not perturb the walk it is taken from."""
_requires_kernel()
seqlens = [768, 320]
inp = _inputs(seqlens, seed=29)
o_ref, state_ref = _run(inp, use_fused=True)
_, (o_got, state_got) = _run_with_checkpoints(inp, [512, 128])
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=0, atol=0)
torch.testing.assert_close(state_got, state_ref, rtol=0, atol=0)
@pytest.mark.parametrize("offset", [100, 63, 65, 1024])
def test_unusable_checkpoint_offsets_write_nothing(offset: int) -> None:
"""Offsets the walk never reaches must leave the destination untouched.
The recurrent state only exists at chunk boundaries, so an offset that is
not a multiple of ``KDA_CHECKPOINT_ALIGNMENT`` -- or that lies past the end
of the sequence -- cannot be honoured. Rounding it to a nearby boundary
would hand the scheduler a state for the wrong token count, so the kernel
stores nothing and leaves the caller to filter these out before the block
is hashed.
"""
_requires_kernel()
from vllm.models.kimi_k3.amd.ops.kda_chunk import KDA_CHECKPOINT_ALIGNMENT
assert offset % KDA_CHECKPOINT_ALIGNMENT != 0 or offset > 512
inp = _inputs([512], seed=31)
ckpt, _ = _run_with_checkpoints(inp, [offset])
assert ckpt.isnan().all()
def test_checkpoint_state_indices_pick_the_destination_row() -> None:
"""With indices the export lands straight in the paged state cache.
FlashKDA writes one row per sequence into a staging buffer that the caller
then scatters. The ROCm walk holds the state in registers already, so it
can be told which cache block each sequence owns and skip the staging
buffer and its scatter launch entirely. A negative row opts a sequence out,
the same way a zero offset does.
"""
_requires_kernel()
seqlens = [256, 256, 256]
rows = 8
inp = _inputs(seqlens, seed=37)
indices = [5, -1, 2]
ckpt, _ = _run_with_checkpoints(
inp,
[64, 64, 128],
rows=rows,
checkpoint_state_indices=torch.tensor(
indices, device="cuda", dtype=torch.int32
),
)
written = {5: (0, 64), 2: (2, 128)}
for row in range(rows):
if row not in written:
assert ckpt[row].isnan().all(), f"row {row} was written"
continue
seq, offset = written[row]
_, truncated = _run(_prefix_inputs(inp, seq, offset), use_fused=True)
torch.testing.assert_close(ckpt[row], truncated[0], rtol=0, atol=0)
@pytest.mark.parametrize("groups", [2, 4])
def test_checkpoint_survives_the_group_split(groups: int, monkeypatch) -> None:
"""The split walks each group from a composed state, not from zero.
Only the second pass holds the true state at an interior boundary, so a
checkpoint taken from the first pass would be the group-local state and
silently wrong.
"""
_requires_kernel()
inp = _inputs([1024], seed=41)
_force_groups(monkeypatch, 1)
_, serial = _run(_prefix_inputs(inp, 0, 512), use_fused=True)
_force_groups(monkeypatch, groups)
ckpt, _ = _run_with_checkpoints(inp, [512])
torch.testing.assert_close(ckpt[0], serial[0], rtol=1e-3, atol=1e-3)
def test_triton_fallback_refuses_a_checkpoint_export() -> None:
"""Dropping the export silently would serve an unwritten state.
The scheduler allocates and hashes the checkpoint block on the strength of
the layer opting in, so a backend that cannot write it has to say so rather
than return a plausible output.
"""
inp = _inputs([256], seed=43)
with pytest.raises(NotImplementedError):
_run(
inp,
use_fused=False,
checkpoint_state=_checkpoint_buffer(1),
checkpoint_offsets=torch.tensor([64], device="cuda", dtype=torch.int32),
)
# ---------------------------------------------------------------------------
# Paged recurrent state
#
# Each sequence reads its initial state from a row of the paged cache and
# writes its final state back to the same row. These check that the result
# matches the dense initial_state/final_state pair, that a sequence marked
# cold starts from zeros rather than from whatever its row held, and that rows
# no sequence owns are left untouched.
# ---------------------------------------------------------------------------
def _paged_cache(inp: dict, warm: list[bool], rows: list[int], slots: int):
"""A cache holding each sequence's h0 at its row, a fill pattern elsewhere."""
cache = torch.full(
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
-7.0,
device="cuda",
dtype=torch.float32,
)
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
for n, row in enumerate(rows):
# A cold row keeps its fill pattern, which must not reach the walk.
if warm[n]:
cache[row] = inp["h0"][n]
return cache, idx, torch.tensor(warm, device="cuda", dtype=torch.bool)
def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
return chunk_kda_prefill(
q=inp["q"].clone(),
k=inp["k"].clone(),
v=inp["v"].clone(),
raw_g=inp["raw_g"].clone(),
raw_beta=inp["raw_beta"],
A_log=inp["A_log"],
g_bias=inp["dt_bias"],
lower_bound=LOWER_BOUND,
state_cache=cache,
state_indices=idx,
has_initial_state=warm,
use_qk_l2norm_in_kernel=True,
cu_seqlens=inp["cu"],
use_fused_chunk=use_fused,
**kw,
)
@pytest.mark.parametrize("use_fused", [False, True])
@pytest.mark.parametrize("seqlens", [[512], [2048], [320, 64], [513, 64, 1]])
def test_paged_state_matches_the_dense_pair(
use_fused: bool, seqlens: list[int]
) -> None:
"""A paged cache must give the same result as the dense pair."""
_requires_kernel()
inp = _inputs(seqlens, seed=11)
n = len(seqlens)
rows = [3 * i + 5 for i in range(n)]
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
before = cache.clone()
o_ref, ht_ref = _run(inp, use_fused=use_fused)
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
assert ht_got is None, "the final state is in the cache, not returned"
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
untouched = torch.ones(cache.shape[0], dtype=torch.bool, device="cuda")
untouched[idx.long()] = False
assert torch.equal(cache[untouched], before[untouched]), (
"a row no sequence owns was written"
)
@pytest.mark.parametrize("use_fused", [False, True])
def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
"""`has_initial_state=False` must ignore whatever the row already holds."""
_requires_kernel()
seqlens = [256, 256, 256]
inp = _inputs(seqlens, seed=12)
warm = [True, False, True]
rows = [4, 9, 1]
cache, idx, warm_t = _paged_cache(inp, warm, rows, slots=16)
# The dense reference zeroes exactly the cold sequences' initial states.
ref = dict(inp)
ref["h0"] = inp["h0"].clone()
ref["h0"][1] = 0.0
o_ref, ht_ref = _run(ref, use_fused=use_fused)
o_got, _ = _run_paged(inp, cache, idx, warm_t, use_fused=use_fused)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
@pytest.mark.parametrize("groups", [2, 4])
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
"""At G>1 the read and the write are different blocks of one launch."""
_requires_kernel()
_force_groups(monkeypatch, groups)
inp = _inputs([1024, 512], seed=13)
rows = [6, 2]
cache, idx, warm = _paged_cache(inp, [True, True], rows, slots=12)
o_ref, ht_ref = _run(inp, use_fused=True)
o_got, _ = _run_paged(inp, cache, idx, warm, use_fused=True)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
def test_paged_state_and_checkpoint_export_together() -> None:
"""The checkpoint lands in its own row while the walk owns the state row."""
_requires_kernel()
inp = _inputs([1024], seed=14)
cache, idx, warm = _paged_cache(inp, [True], [3], slots=12)
ckidx = torch.tensor([8], device="cuda", dtype=torch.int32)
o_ref, ht_ref = _run(inp, use_fused=True)
prefix = _prefix_inputs(inp, 0, 512)
_, ck_ref = _run(prefix, use_fused=True)
o_got, _ = _run_paged(
inp,
cache,
idx,
warm,
use_fused=True,
checkpoint_state=cache,
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
checkpoint_state_indices=ckidx,
)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[ckidx.long()], ck_ref, rtol=5e-3, atol=5e-3)
def test_paged_state_rejects_a_dense_initial_state() -> None:
"""The two ways of passing the state are mutually exclusive."""
_requires_kernel()
inp = _inputs([256], seed=15)
cache, idx, warm = _paged_cache(inp, [True], [1], slots=4)
with pytest.raises(ValueError, match="replaces initial_state"):
_run_paged(
inp,
cache,
idx,
warm,
use_fused=True,
initial_state=inp["h0"].clone(),
)
@pytest.mark.parametrize("use_fused", [False, True])
def test_padded_cache_stride_is_honoured(use_fused: bool) -> None:
"""A mamba page may pad stride(0); only each state row is dense.
The real `recurrent_state` is not a contiguous tensor, so a kernel that
assumes a packed ``H * V * K`` row stride addresses the wrong rows. The
other cases in this file all build a contiguous cache, which cannot
exercise that.
"""
_requires_kernel()
seqlens = [1024, 512]
inp = _inputs(seqlens, seed=21)
n = len(seqlens)
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
slots, pad = 16, 512
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
cache = raw.as_strided(
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
)
assert not cache.is_contiguous() and cache[0].is_contiguous()
rows = [11, 3]
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
warm = torch.ones(n, dtype=torch.bool, device="cuda")
for i, r in enumerate(rows):
cache[r] = inp["h0"][i]
before = raw.clone()
o_ref, ht_ref = _run(inp, use_fused=use_fused)
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
assert ht_got is None
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
# The padding between rows, and every unowned row, must be untouched.
untouched = torch.ones(slots, dtype=torch.bool, device="cuda")
untouched[idx.long()] = False
assert torch.equal(raw[untouched], before[untouched])
assert torch.equal(raw[idx.long(), row:], before[idx.long(), row:]), (
"the kernel wrote into the inter-row padding"
)
def test_padded_checkpoint_stride_is_honoured() -> None:
"""The checkpoint destination is the paged cache, which pads stride(0).
Every other checkpoint case here builds a contiguous buffer, so none of
them would catch a kernel that assumes a packed ``H * V * K`` row stride
and writes the snapshot into a neighbouring block.
"""
_requires_kernel()
seqlens = [1024]
inp = _inputs(seqlens, seed=31)
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
slots, pad = 16, 512
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
cache = raw.as_strided(
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
)
assert not cache.is_contiguous()
state_rows = [11]
idx = torch.tensor(state_rows, device="cuda", dtype=torch.int32)
warm = torch.ones(1, dtype=torch.bool, device="cuda")
cache[state_rows[0]] = inp["h0"][0]
ck_rows = [4]
before = raw.clone()
prefix = _prefix_inputs(inp, 0, 512)
_, ck_ref = _run(prefix, use_fused=True)
o_ref, ht_ref = _run(inp, use_fused=True)
o_got, _ = _run_paged(
inp,
cache,
idx,
warm,
use_fused=True,
checkpoint_state=cache,
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
checkpoint_state_indices=torch.tensor(
ck_rows, device="cuda", dtype=torch.int32
),
)
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
torch.testing.assert_close(cache[ck_rows[0]], ck_ref[0], rtol=5e-3, atol=5e-3)
touched = torch.zeros(slots, dtype=torch.bool, device="cuda")
touched[state_rows + ck_rows] = True
assert torch.equal(raw[~touched], before[~touched])
assert torch.equal(raw[touched, row:], before[touched, row:]), (
"the kernel wrote into the inter-row padding"
)
+17 -5
View File
@@ -778,9 +778,9 @@ class EngineArgs:
fail_on_environ_validation: bool = False
gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None
kda_prefill_backend: Literal["auto", "triton", "flashkda", "flashinfer"] | None = (
None
)
kda_prefill_backend: (
Literal["auto", "triton", "flashkda", "flashinfer", "fused"] | None
) = None
kda_decode_backend: Literal["auto", "native", "flashinfer", "triton"] | None = None
def __post_init__(self):
@@ -1761,9 +1761,10 @@ class EngineArgs:
parser.add_argument(
"--kda-prefill-backend",
dest="kda_prefill_backend",
choices=["auto", "triton", "flashkda", "flashinfer"],
choices=["auto", "triton", "flashkda", "flashinfer", "fused"],
default=None,
help="Select KDA prefill backend.",
help="Select KDA prefill backend. 'flashkda' is CUDA-only and "
"'fused' is ROCm-only; 'auto' picks a supported backend.",
)
parser.add_argument(
"--kda-decode-backend",
@@ -2597,6 +2598,17 @@ class EngineArgs:
if self.gdn_prefill_backend is not None:
self.additional_config["gdn_prefill_backend"] = self.gdn_prefill_backend
if self.kda_prefill_backend is not None:
if (
self.kda_prefill_backend == "flashkda"
and not current_platform.is_cuda()
):
raise ValueError(
"--kda-prefill-backend=flashkda is only available on CUDA."
)
if self.kda_prefill_backend == "fused" and not current_platform.is_rocm():
raise ValueError(
"--kda-prefill-backend=fused is only available on ROCm."
)
self.additional_config["kda_prefill_backend"] = self.kda_prefill_backend
if self.kda_decode_backend is not None:
self.additional_config["kda_decode_backend"] = self.kda_decode_backend
+44 -23
View File
@@ -35,19 +35,19 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
causal_conv1d_fn,
causal_conv1d_update,
)
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
is_fused_kda_chunk_supported,
)
from vllm.models.kimi_k3.amd.ops.kda_decode import (
is_fused_kda_decode_supported,
make_decode_conv1d_weight_loader,
make_decode_norm_weight_loader,
)
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
chunk_kda_with_fused_gate,
fused_recurrent_kda,
fused_recurrent_kda_packed_decode,
)
@@ -210,10 +210,21 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if isinstance(additional_config, dict)
else "auto"
)
backend = "triton" if backend == "auto" else backend
assert backend == "triton", (
"The ROCm Kimi-K3 KDA layer only supports the Triton KDA prefill "
f"backend, got {backend!r}."
assert backend in ("auto", "triton", "fused"), (
"The ROCm Kimi-K3 KDA prefill backend must be one of "
f"'auto', 'triton' or 'fused', got {backend!r}."
)
if backend == "fused" and not is_fused_kda_chunk_supported():
raise RuntimeError(
"The fused KDA chunk kernel requires gfx950 and a build that "
"includes it."
)
self.use_fused_chunk = backend == "fused" or (
backend == "auto" and is_fused_kda_chunk_supported()
)
logger.info_once(
"Kimi-K3 KDA prefill backend: %s",
"fused" if self.use_fused_chunk else "triton",
)
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
@@ -498,6 +509,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
# prefill tail only.
core_attn_out_decode = None
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
# Without a spec split the non-spec tokens are exactly
# core_attn_out[:, :num_actual_tokens] in order, so both kernels
# can write their slice straight into it. With one, the tokens
# are interleaved and have to be scattered by index afterwards.
direct = spec_sequence_masks is None
use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None
if split_non_spec:
assert non_spec_query_start_loc is not None
nd_tok = m.num_decode_tokens
@@ -515,6 +532,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
ssm_state_indices=non_spec_state_indices_tensor[
: m.num_decodes
],
out=core_attn_out[:, :nd_tok] if direct else None,
)
q_ns = q_ns[:, nd_tok:]
k_ns = k_ns[:, nd_tok:]
@@ -531,16 +549,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
prefill_query_start_loc = non_spec_query_start_loc
prefill_state_indices = non_spec_state_indices_tensor
prefill_has_initial_state = has_initial_state
nd_tok = 0
initial_state = gather_initial_states(
recurrent_state,
prefill_state_indices,
prefill_has_initial_state,
)
(
core_attn_out_non_spec,
last_recurrent_state,
) = chunk_kda_with_fused_gate(
_,
) = chunk_kda_prefill(
q=q_ns,
k=k_ns,
v=v_ns,
@@ -549,17 +563,23 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
A_log=self.A_log,
g_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=initial_state,
output_final_state=True,
state_cache=recurrent_state,
state_indices=prefill_state_indices,
has_initial_state=prefill_has_initial_state,
use_qk_l2norm_in_kernel=True,
cu_seqlens=prefill_query_start_loc,
chunk_indices=m.chunk_indices,
chunk_offsets=m.chunk_offsets,
use_fused_chunk=use_fused_chunk,
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
)
# Init cache
recurrent_state[prefill_state_indices] = last_recurrent_state
# chunk_kda_prefill updates `recurrent_state` in place, so
# there is no final state to write back here.
if split_non_spec:
if direct:
# Both slices already landed in core_attn_out.
core_attn_out_non_spec = core_attn_out[:, :num_actual_tokens]
elif split_non_spec:
# Restore decode-first token order for the merge below.
core_attn_out_non_spec = torch.cat(
[core_attn_out_decode, core_attn_out_non_spec], dim=1
@@ -611,9 +631,10 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
elif core_attn_out_non_spec is not None:
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
0, :num_actual_tokens
]
else:
assert core_attn_out_spec is not None
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
+294
View File
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
path with a single launch that keeps the per-chunk state in registers, so the
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
"""
from functools import cache
import torch
from vllm.third_party.flash_linear_attention.ops.index import (
prepare_chunk_indices,
prepare_chunk_offsets,
)
CHUNK_SIZE = 64
HEAD_DIM = 128
KDA_CHECKPOINT_ALIGNMENT = CHUNK_SIZE
# Chunk-group split policy constants; see _chunk_groups.
_BLOCKS_P1 = 2 # (kV + kK) / kBV
_BLOCKS_P2 = 1 # kV / kBV
_MIN_LEN = 4
_DEEP_LEN = 12
_MAX_GROUPS = 32
@cache
def is_fused_kda_chunk_supported() -> bool:
from vllm.platforms.rocm import on_gfx950
if not hasattr(torch.ops._C, "fused_kda_chunk"):
return False
# TODO: Verify on other archs, currently only validated on gfx950
return on_gfx950()
def can_use_fused_kda_chunk(
head_k_dim: int,
head_v_dim: int,
dtype: torch.dtype,
chunk_size: int,
) -> bool:
return (
head_k_dim == HEAD_DIM
and head_v_dim == HEAD_DIM
and chunk_size == CHUNK_SIZE
and dtype == torch.bfloat16
and is_fused_kda_chunk_supported()
)
def fused_kda_prologue(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
dt_bias: torch.Tensor,
scale: float,
lower_bound: float,
cu_seqlens: torch.Tensor,
conv_weight: torch.Tensor | None = None,
conv_state: torch.Tensor | None = None,
conv_state_indices: torch.Tensor | None = None,
conv_has_initial_state: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
) -> dict[str, torch.Tensor]:
"""Run the whole chunk-path prologue in one launch.
Replaces the two L2 norms, the gate cumsum, both intra-chunk passes and the
w/u recompute. Returns the operands the fused chunk kernel consumes.
The kernel can also apply the depthwise conv and its silu, in which case
``q``/``k``/``v`` are the three raw bands of the QKV projection and
``conv_state`` is updated in place. This path is currently not used.
"""
# `beta` and `g` reach the layer as last-dim slices of the fused QKVGFAB
# projection and carry its row stride. The kernel reads both with an
# explicit per-token stride, so neither is copied here.
_, t_total, num_heads, _ = q.shape
if chunk_indices is None:
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
chunk_indices = chunk_indices.to(torch.int32)
num_chunks = chunk_indices.shape[0]
dev = q.device
# q may be a strided band view, so the workspaces are sized rather than
# cloned from it.
def _like(last: int) -> torch.Tensor:
return torch.empty(1, t_total, num_heads, last, dtype=q.dtype, device=q.device)
ws = dict(
qg=_like(HEAD_DIM),
w=_like(HEAD_DIM),
u=_like(HEAD_DIM),
kg_t=torch.empty(
num_chunks, num_heads, HEAD_DIM, CHUNK_SIZE, dtype=q.dtype, device=dev
),
aqk=torch.empty(1, t_total, num_heads, CHUNK_SIZE, dtype=q.dtype, device=dev),
decay=torch.empty(
num_chunks, num_heads, HEAD_DIM, dtype=torch.float32, device=dev
),
)
torch.ops._C.fused_kda_prologue(
q,
k,
v,
raw_g,
raw_beta,
A_log.reshape(-1),
dt_bias.reshape(-1),
ws["qg"],
ws["w"],
ws["u"],
ws["kg_t"],
ws["aqk"],
ws["decay"],
cu_seqlens.to(torch.int32),
chunk_indices,
conv_weight,
conv_state,
conv_state_indices,
conv_has_initial_state,
scale,
lower_bound,
)
return ws
@cache
def _num_cus(device: int) -> int:
from vllm.platforms import current_platform
return current_platform.get_device_properties(device).multi_processor_count
@cache
def _chunk_groups(chunks_per_seq: int, num_seqs: int, num_heads: int) -> int:
"""How many parallel chunk groups to cut each sequence into.
The scan composes the group operators with the exact transfer
``M_g = prod_c (diag(d_c) - w_c^T kg_c)``, so the composition error does
not grow with group length and the choice is purely a machine fit:
* ``fill1``/``fill2``: the largest G whose pass-one / pass-two grid still
fits in one scheduling batch. Rows are launched in blocks of at most
``kBV = 128``, so pass one (``kV + kK`` rows) needs two blocks per
``(n, h, g)`` and pass two one. Past a pass's fill point its block count
grows as fast as its depth shrinks and the rung stops paying.
* ``_DEEP_LEN``: unless the groups are still long, where pass two's halving
is worth letting pass one spill to a second batch.
* ``_MIN_LEN``: below this many chunks per group the per-group fixed cost
exceeds what the shorter walk saves.
``G == 2`` is depth-neutral -- two passes of ``nt/2`` -- and only doubles
staging, so the split is taken only from ``G >= 4``.
"""
cus = _num_cus(torch.accelerator.current_device_index())
nh = max(num_seqs * num_heads, 1)
fill1 = cus // (_BLOCKS_P1 * nh)
fill2 = cus // (_BLOCKS_P2 * nh)
cand = fill1
if fill2 > 0 and chunks_per_seq // fill2 >= _DEEP_LEN:
cand = fill2
cand = min(cand, chunks_per_seq // _MIN_LEN, _MAX_GROUPS)
return cand if cand >= 4 else 1
def _kda_group_workspace(
groups: int, nh: int, device: torch.device
) -> torch.Tensor | None:
"""One fp32 buffer the kernel carves into bg / sin_ / ag / mgT.
Kept caller-owned so the memory stays inside the caching allocator and
vLLM's memory profiling accounts for it; the kernel's carve order must
match the layout here.
"""
if groups <= 1:
return None
planes = groups * nh
plane = HEAD_DIM * HEAD_DIM
floats = 2 * planes * plane + planes * HEAD_DIM # bg, sin_, ag
floats += (planes * plane + 1) // 2 # mgT, bf16
return torch.empty(floats, dtype=torch.float32, device=device)
def fused_kda_chunk(
qg: torch.Tensor,
w: torch.Tensor,
u: torch.Tensor,
kg_t: torch.Tensor,
aqk: torch.Tensor,
decay: torch.Tensor,
out: torch.Tensor,
scale: float,
cu_seqlens: torch.Tensor,
initial_state: torch.Tensor | None,
output_final_state: bool,
chunk_offsets: torch.Tensor | None = None,
checkpoint_state: torch.Tensor | None = None,
checkpoint_offsets: torch.Tensor | None = None,
checkpoint_state_indices: torch.Tensor | None = None,
state_cache: torch.Tensor | None = None,
state_indices: torch.Tensor | None = None,
has_initial_state: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Run the chunk recurrence and the output projection in one launch.
Args:
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
checkpoint_state: destination for the mid-prefill state snapshots,
fp32 ``[rows, H, 128, 128]``. Without
``checkpoint_state_indices`` it is a staging buffer indexed by
sequence; with them it can be the paged state cache itself.
checkpoint_offsets: int32 ``[N]`` token offsets, each relative to its
sequence's first query token, at which to snapshot the recurrent
state. ``0`` disables the export for that sequence. A non-zero
offset must be a multiple of :data:`KDA_CHECKPOINT_ALIGNMENT`;
the kernel skips any that is not, so the caller has to filter
rather than rely on the store happening.
checkpoint_state_indices: optional int32 ``[N]`` destination row per
sequence. A negative row disables the export for that sequence.
state_cache: the paged recurrent state, fp32
``[slots, H, 128, 128]``. When given, the walk reads sequence
``n``'s initial state from row ``state_indices[n]`` and writes its
final state back to the same row, which removes the gather and the
scatter that otherwise bracket this kernel. Replaces
``initial_state`` / ``output_final_state``, and the returned final
state is then ``None`` because it is already in the cache.
state_indices: int32 ``[N]`` cache row per sequence.
has_initial_state: bool ``[N]``; a false entry starts that sequence
from a zero state and its cache row is read only, never before the
walk writes it.
"""
if state_cache is not None:
if initial_state is not None or output_final_state:
raise ValueError("state_cache replaces initial_state/output_final_state")
if state_indices is None or has_initial_state is None:
raise ValueError("state_cache needs state_indices and has_initial_state")
num_seqs = cu_seqlens.numel() - 1
final_state = None
if output_final_state:
final_state = torch.empty(
num_seqs,
u.shape[2],
u.shape[3],
qg.shape[3],
dtype=torch.float32,
device=u.device,
)
cu_seqlens = cu_seqlens.to(torch.int32)
if chunk_offsets is None:
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
# The chunk walk is serial in chunks. Splitting each sequence into `groups`
# stretches that run in parallel costs a second pass and a scan, and only
# pays once the chain is long enough to amortise both.
chunks_per_seq = kg_t.shape[0] // max(num_seqs, 1)
groups = _chunk_groups(chunks_per_seq, num_seqs, u.shape[2])
group_state = _kda_group_workspace(groups, num_seqs * u.shape[2], u.device)
torch.ops._C.fused_kda_chunk(
qg,
w,
u,
kg_t,
aqk,
decay,
initial_state,
final_state,
out,
cu_seqlens,
chunk_offsets,
scale,
group_state,
groups,
checkpoint_state,
None if checkpoint_offsets is None else checkpoint_offsets.to(torch.int32),
None
if checkpoint_state_indices is None
else checkpoint_state_indices.to(torch.int32),
state_cache,
None if state_indices is None else state_indices.to(torch.int32),
has_initial_state,
)
return out, final_state
+179
View File
@@ -0,0 +1,179 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""KDA prefill backend selection for ROCm.
The Kimi-K3 KDA layer calls :func:`chunk_kda_prefill`, which either runs the
fused HIP kernels in ``kda_chunk`` or falls back to the vendored Triton chunk
path.
"""
import torch
from vllm.logger import init_logger
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
can_use_fused_kda_chunk,
fused_kda_chunk,
fused_kda_prologue,
)
from vllm.models.kimi_k3.amd.ops.third_party.kda import chunk_kda_with_fused_gate
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
logger = init_logger(__name__)
def chunk_kda_prefill(
q: torch.Tensor,
k: torch.Tensor,
v: torch.Tensor,
raw_g: torch.Tensor,
raw_beta: torch.Tensor,
A_log: torch.Tensor,
g_bias: torch.Tensor | None = None,
scale: float | None = None,
initial_state: torch.Tensor | None = None,
output_final_state: bool = False,
lower_bound: float | None = None,
use_qk_l2norm_in_kernel: bool = False,
cu_seqlens: torch.Tensor | None = None,
chunk_indices: torch.Tensor | None = None,
chunk_offsets: torch.Tensor | None = None,
use_fused_chunk: bool = False,
out: torch.Tensor | None = None,
checkpoint_state: torch.Tensor | None = None,
checkpoint_offsets: torch.Tensor | None = None,
checkpoint_state_indices: torch.Tensor | None = None,
state_cache: torch.Tensor | None = None,
state_indices: torch.Tensor | None = None,
has_initial_state: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Run chunk KDA from raw gate and beta projections.
Args:
use_fused_chunk: request the two-kernel ROCm path. It is used only when
every one of its preconditions holds; otherwise the Triton path
runs unchanged.
out: buffer the result must land in. Honoured by both backends, so the
caller can hand in a slice of its own output and skip a copy.
checkpoint_state: destination for mid-prefill recurrent state
snapshots, letting a later prefix-cache hit resume from a mamba
block boundary. See :func:`fused_kda_chunk`.
checkpoint_offsets: per-sequence token offset to snapshot at, ``0``
for none.
checkpoint_state_indices: optional per-sequence destination row.
state_cache: the paged recurrent state. When given, the fused backend
reads and writes it in place and neither a gather nor a scatter is
needed around this call; the returned final state is ``None``.
state_indices: per-sequence cache row.
has_initial_state: per-sequence flag; false starts from a zero state.
Returns:
The output and, when requested, the final recurrent state.
"""
if scale is None:
scale = k.shape[-1] ** -0.5
# The fused prologue folds the q/k L2 norm and the gate activation in, so it
# needs the raw projections and a bounded gate rather than the pre-normalized
# tensors the Triton path takes.
fused = (
use_fused_chunk
and use_qk_l2norm_in_kernel
and cu_seqlens is not None
and lower_bound is not None
and g_bias is not None
and can_use_fused_kda_chunk(k.shape[-1], v.shape[-1], k.dtype, FLA_CHUNK_SIZE)
)
if checkpoint_offsets is not None and not fused:
raise NotImplementedError(
"The KDA prefill checkpoint export needs kda_prefill_backend=fused for ROCm"
)
# Checked here rather than in the backend so both paths reject the same
# arguments.
if state_cache is not None:
if initial_state is not None or output_final_state:
raise ValueError("state_cache replaces initial_state/output_final_state")
if state_indices is None or has_initial_state is None:
raise ValueError("state_cache needs state_indices and has_initial_state")
# Only the fused walk addresses the paged rows directly. The Triton path
# gathers the rows it needs and scatters the results back here, so callers
# pass the cache the same way for either backend.
scatter_to: torch.Tensor | None = None
if state_cache is not None and not fused:
initial_state = gather_initial_states(
state_cache, state_indices, has_initial_state
)
output_final_state = True
scatter_to, state_cache = state_cache, None
if fused:
# Restated for the type checker; `fused` already implies all three.
assert cu_seqlens is not None and lower_bound is not None
assert g_bias is not None
logger.info_once(
"Kimi-K3 KDA prefill: dispatching the fused ROCm chunk kernel."
)
ws = fused_kda_prologue(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
dt_bias=g_bias,
scale=scale,
lower_bound=lower_bound,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
)
return fused_kda_chunk(
qg=ws["qg"],
w=ws["w"],
u=ws["u"],
kg_t=ws["kg_t"],
aqk=ws["aqk"],
decay=ws["decay"],
# v is dead by this point, so it doubles as the output buffer.
out=out if out is not None else v,
scale=scale,
cu_seqlens=cu_seqlens,
initial_state=initial_state,
output_final_state=output_final_state,
chunk_offsets=chunk_offsets,
checkpoint_state=checkpoint_state,
checkpoint_offsets=checkpoint_offsets,
checkpoint_state_indices=checkpoint_state_indices,
state_cache=state_cache,
state_indices=state_indices,
has_initial_state=has_initial_state,
)
o, final_state = chunk_kda_with_fused_gate(
q=q,
k=k,
v=v,
raw_g=raw_g,
raw_beta=raw_beta,
A_log=A_log,
g_bias=g_bias,
scale=scale,
initial_state=initial_state,
output_final_state=output_final_state,
lower_bound=lower_bound,
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
cu_seqlens=cu_seqlens,
chunk_indices=chunk_indices,
chunk_offsets=chunk_offsets,
)
if out is not None and o.data_ptr() != out.data_ptr():
out.copy_(o)
o = out
if scatter_to is not None:
assert state_indices is not None
scatter_to[state_indices.long()] = final_state
return o, None
return o, final_state