[ROCm][Perf] Kimi-K3 Fused kernels for KDA prefill reland (#54038)
Signed-off-by: kliuae <kuanfu.liu@embeddedllm.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Signed-off-by: kliuae <17350011+kliuae@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
+21
-5
@@ -1181,11 +1181,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
endif()
|
||||
|
||||
# HIP counterpart of the FUSED_KDA_DECODE block above: same `fused_kda_decode`
|
||||
# op, same VLLM_ENABLE_FUSED_KDA_DECODE guard in torch_bindings.cpp. Built for
|
||||
# the CDNA archs the kernel supports (gfx942 / gfx950), so it is only emitted
|
||||
# when one of those is in the arch list. A multi-arch build that includes them
|
||||
# still emits this source for every arch in that list. The runtime gate in
|
||||
# kda_decode.py keeps the kernel off unsupported devices.
|
||||
# op, same VLLM_ENABLE_FUSED_KDA_DECODE guard in torch_bindings.cpp
|
||||
# Built for the CDNA archs the kernel supports
|
||||
# (gfx942 / gfx950), so it is only emitted when one of those is in the arch
|
||||
# list. A multi-arch build that includes them still emits this source for
|
||||
# every arch in that list. The runtime gate in kda_decode.py keeps the kernel
|
||||
# off unsupported devices. Fused prefill KDA kernels is Only measured on
|
||||
# gfx950, so it is only built when gfx950 is in the arch list.
|
||||
if(VLLM_GPU_LANG STREQUAL "HIP")
|
||||
set(FUSED_KDA_DECODE_HIP_ARCHS ${VLLM_GPU_ARCHES})
|
||||
list(FILTER FUSED_KDA_DECODE_HIP_ARCHS INCLUDE REGEX "gfx942|gfx950")
|
||||
@@ -1196,6 +1198,16 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
message(STATUS
|
||||
"Building fused KDA decode for HIP archs: ${FUSED_KDA_DECODE_HIP_ARCHS}")
|
||||
endif()
|
||||
|
||||
set(FUSED_KDA_CHUNK_HIP_ARCHS ${VLLM_GPU_ARCHES})
|
||||
list(FILTER FUSED_KDA_CHUNK_HIP_ARCHS INCLUDE REGEX "gfx950")
|
||||
if(FUSED_KDA_CHUNK_HIP_ARCHS)
|
||||
set(FUSED_KDA_CHUNK_HIP_SRC
|
||||
"csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu")
|
||||
list(APPEND VLLM_STABLE_EXT_SRC "${FUSED_KDA_CHUNK_HIP_SRC}")
|
||||
message(STATUS
|
||||
"Building fused KDA chunk for HIP archs: ${FUSED_KDA_CHUNK_HIP_ARCHS}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
message(STATUS "Enabling C_stable extension.")
|
||||
@@ -1246,6 +1258,10 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_FUSED_KDA_DECODE=1)
|
||||
endif()
|
||||
if(FUSED_KDA_CHUNK_HIP_SRC)
|
||||
target_compile_definitions(_C_stable_libtorch PRIVATE
|
||||
VLLM_ENABLE_FUSED_KDA_CHUNK=1)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# On ROCm, _C_stable_libtorch calls raw HIP APIs (e.g. hipGetDevice in
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -406,6 +406,39 @@ void fused_gdn_decode_post_conv_mtp(
|
||||
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
|
||||
void fused_kda_prologue(
|
||||
torch::stable::Tensor const& q, torch::stable::Tensor const& k,
|
||||
torch::stable::Tensor const& v, torch::stable::Tensor const& raw_g,
|
||||
torch::stable::Tensor const& raw_beta, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias, torch::stable::Tensor& qg,
|
||||
torch::stable::Tensor& w, torch::stable::Tensor& u,
|
||||
torch::stable::Tensor& kg_t, torch::stable::Tensor& aqk,
|
||||
torch::stable::Tensor& decay, torch::stable::Tensor const& cu_seqlens,
|
||||
torch::stable::Tensor const& chunk_indices,
|
||||
std::optional<torch::stable::Tensor> conv_weight,
|
||||
std::optional<torch::stable::Tensor> conv_state,
|
||||
std::optional<torch::stable::Tensor> conv_state_indices,
|
||||
std::optional<torch::stable::Tensor> conv_has_initial_state, double scale,
|
||||
double lower_bound);
|
||||
|
||||
void fused_kda_chunk(
|
||||
torch::stable::Tensor const& qg, torch::stable::Tensor const& w,
|
||||
torch::stable::Tensor const& u, torch::stable::Tensor const& kg_t,
|
||||
torch::stable::Tensor const& aqk, torch::stable::Tensor const& decay,
|
||||
std::optional<torch::stable::Tensor> initial_state,
|
||||
std::optional<torch::stable::Tensor> final_state,
|
||||
torch::stable::Tensor& out, torch::stable::Tensor const& cu_seqlens,
|
||||
torch::stable::Tensor const& chunk_offsets, double scale,
|
||||
std::optional<torch::stable::Tensor> group_state, int64_t groups,
|
||||
std::optional<torch::stable::Tensor> checkpoint_state,
|
||||
std::optional<torch::stable::Tensor> checkpoint_offsets,
|
||||
std::optional<torch::stable::Tensor> checkpoint_state_indices,
|
||||
std::optional<torch::stable::Tensor> state_cache,
|
||||
std::optional<torch::stable::Tensor> state_indices,
|
||||
std::optional<torch::stable::Tensor> has_initial_state);
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
|
||||
std::optional<torch::stable::Tensor> delta,
|
||||
|
||||
@@ -537,6 +537,26 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"str output_gate_activation='silu') -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
|
||||
ops.def(
|
||||
"fused_kda_prologue("
|
||||
"Tensor q, Tensor k, Tensor v, Tensor raw_g, Tensor raw_beta, "
|
||||
"Tensor A_log, Tensor dt_bias, Tensor! qg, Tensor! w, Tensor! u, "
|
||||
"Tensor! kg_t, Tensor! aqk, Tensor! decay, Tensor cu_seqlens, "
|
||||
"Tensor chunk_indices, Tensor? conv_weight, Tensor(e!)? conv_state, "
|
||||
"Tensor? conv_state_indices, Tensor? conv_has_initial_state, "
|
||||
"float scale, float lower_bound) -> ()");
|
||||
ops.def(
|
||||
"fused_kda_chunk("
|
||||
"Tensor qg, Tensor w, Tensor u, Tensor kg_t, Tensor aqk, Tensor decay, "
|
||||
"Tensor? initial_state, Tensor(a!)? final_state, Tensor! out, "
|
||||
"Tensor cu_seqlens, Tensor chunk_offsets, float scale, "
|
||||
"Tensor(b!)? group_state, int groups, "
|
||||
"Tensor(c!)? checkpoint_state=None, Tensor? checkpoint_offsets=None, "
|
||||
"Tensor? checkpoint_state_indices=None, Tensor(d!)? state_cache=None, "
|
||||
"Tensor? state_indices=None, Tensor? has_initial_state=None) -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.def(
|
||||
"kimi_k3_attn_res("
|
||||
@@ -825,6 +845,11 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
TORCH_BOX(&fused_gdn_decode_post_conv_mtp));
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_CHUNK
|
||||
ops.impl("fused_kda_prologue", TORCH_BOX(&fused_kda_prologue));
|
||||
ops.impl("fused_kda_chunk", TORCH_BOX(&fused_kda_chunk));
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,859 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The fused ROCm KDA chunk kernel must match the Triton chunk path.
|
||||
|
||||
The fused kernel reassociates nothing, but it keeps the chunk states in
|
||||
registers and rounds them to bfloat16 only where the Triton path does, so the
|
||||
outputs agree to bfloat16 rounding and the final states nearly exactly.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
reason="The fused KDA chunk kernel is only built for gfx950",
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
NUM_HEADS = 12
|
||||
LOWER_BOUND = -5.0
|
||||
|
||||
|
||||
def _requires_kernel() -> None:
|
||||
if not hasattr(torch.ops._C, "fused_kda_chunk"):
|
||||
pytest.skip("vLLM was built without the fused KDA chunk kernel")
|
||||
|
||||
|
||||
def _inputs(seqlens: list[int], seed: int) -> dict:
|
||||
torch.manual_seed(seed)
|
||||
dev, dtype = "cuda", torch.bfloat16
|
||||
total = sum(seqlens)
|
||||
cu = torch.tensor(
|
||||
[0] + torch.tensor(seqlens).cumsum(0).tolist(), device=dev, dtype=torch.int32
|
||||
)
|
||||
shape = (1, total, NUM_HEADS, HEAD_DIM)
|
||||
return dict(
|
||||
q=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
k=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
raw_g=torch.randn(shape, device=dev, dtype=dtype),
|
||||
raw_beta=torch.randn(1, total, NUM_HEADS, device=dev, dtype=dtype),
|
||||
A_log=torch.randn(NUM_HEADS, device=dev, dtype=torch.float32) * 0.5,
|
||||
dt_bias=torch.randn(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32)
|
||||
* 0.5,
|
||||
h0=torch.randn(
|
||||
len(seqlens), NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
|
||||
)
|
||||
* 0.1,
|
||||
cu=cu,
|
||||
)
|
||||
|
||||
|
||||
def _run(inp: dict, use_fused: bool, out: torch.Tensor | None = None, **kw):
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
|
||||
return chunk_kda_prefill(
|
||||
q=inp["q"].clone(),
|
||||
k=inp["k"].clone(),
|
||||
v=inp["v"].clone(),
|
||||
raw_g=inp["raw_g"].clone(),
|
||||
raw_beta=inp["raw_beta"],
|
||||
A_log=inp["A_log"],
|
||||
g_bias=inp["dt_bias"],
|
||||
lower_bound=LOWER_BOUND,
|
||||
initial_state=inp["h0"].clone(),
|
||||
output_final_state=True,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
use_fused_chunk=use_fused,
|
||||
out=out,
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
def _force_groups(monkeypatch, groups: int) -> None:
|
||||
"""Pin the group count, which is otherwise a function of the device."""
|
||||
import vllm.models.kimi_k3.amd.ops.kda_chunk as kda_chunk
|
||||
|
||||
monkeypatch.setattr(kda_chunk, "_chunk_groups", lambda *_: groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[
|
||||
[1], # shorter than a chunk
|
||||
[64], # exactly one chunk
|
||||
[1024],
|
||||
[4096], # many chunks, one sequence
|
||||
[513, 64, 1, 1200], # ragged, including a partial tail chunk
|
||||
[512] * 6,
|
||||
],
|
||||
)
|
||||
def test_fused_chunk_matches_triton(seqlens: list[int]) -> None:
|
||||
_requires_kernel()
|
||||
|
||||
inp = _inputs(seqlens, seed=len(seqlens))
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_fused, state_fused = _run(inp, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_out_argument_receives_the_result(use_fused: bool) -> None:
|
||||
"""``out`` is a promise about where the result lands.
|
||||
|
||||
The layer passes a slice of its own output buffer so neither path has to
|
||||
copy afterwards, and it then skips the copy based on the returned tensor
|
||||
aliasing that buffer. Both must hold for the fused path and for the Triton
|
||||
fallback, which allocates its own buffer internally.
|
||||
"""
|
||||
if use_fused:
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [321, 64, 700]
|
||||
inp = _inputs(seqlens, seed=13)
|
||||
o_ref, _ = _run(inp, use_fused=use_fused)
|
||||
|
||||
# A non-zero offset into a larger buffer, as the layer produces when a
|
||||
# mixed batch puts decode tokens first.
|
||||
total = sum(seqlens)
|
||||
buf = torch.empty(
|
||||
1, total + 8, NUM_HEADS, HEAD_DIM, device="cuda", dtype=o_ref.dtype
|
||||
)
|
||||
view = buf[:, 8:]
|
||||
o_got, _ = _run(inp, use_fused=use_fused, out=view)
|
||||
|
||||
assert o_got.data_ptr() == view.data_ptr()
|
||||
torch.testing.assert_close(view.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_fused_chunk_accepts_projection_views() -> None:
|
||||
"""`beta` and `g` arrive as last-dim slices of the fused QKVGFAB projection.
|
||||
|
||||
Those views carry the projection's row stride, so a kernel that indexes them
|
||||
densely reads the wrong elements or rejects the tensor outright. Build the
|
||||
inputs the way the layer does rather than as standalone tensors.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [777, 64, 391]
|
||||
inp = _inputs(seqlens, seed=7)
|
||||
total = sum(seqlens)
|
||||
|
||||
# One wide projection, split on the last dim exactly like KimiK3DeltaAttention.
|
||||
qkv_width = 3 * NUM_HEADS * HEAD_DIM
|
||||
projection = torch.randn(
|
||||
total, qkv_width + NUM_HEADS, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
beta_view = projection.split([qkv_width, NUM_HEADS], dim=-1)[1].unsqueeze(0)
|
||||
assert not beta_view.is_contiguous()
|
||||
|
||||
inp["raw_beta"] = beta_view.contiguous()
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
|
||||
inp["raw_beta"] = beta_view
|
||||
o_fused, state_fused = _run(inp, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def test_fused_chunk_survives_correlated_keys_and_real_gate_bias() -> None:
|
||||
"""The input regime the served model actually produces.
|
||||
|
||||
This is what forces the intra-chunk inverse to keep fp32 MFMA operands: at
|
||||
bf16 the Neumann doubling's intermediate powers cancel catastrophically
|
||||
here, diverging by orders of magnitude and driving the final state to inf.
|
||||
|
||||
Three properties of real traffic combine here and in none of the tests
|
||||
above, all measured from tensors captured off a live TP8 server:
|
||||
|
||||
* repetitive text gives near-duplicate L2-normalized keys, so ``k_i . k_j``
|
||||
approaches 1 and the Neumann terms stop shrinking;
|
||||
* the checkpoint's ``dt_bias`` lies in ``[-7.8, -1.4]``, which makes the
|
||||
within-chunk gate cumsum span ~300 powers of two;
|
||||
* ``beta`` saturates near 0.95.
|
||||
|
||||
``_inputs`` draws all three from ``randn``, landing in the benign regime.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
torch.manual_seed(11)
|
||||
dev, dtype = "cuda", torch.bfloat16
|
||||
total = 1088
|
||||
shape = (1, total, NUM_HEADS, HEAD_DIM)
|
||||
base = torch.randn(1, 1, NUM_HEADS, HEAD_DIM, device=dev, dtype=dtype)
|
||||
|
||||
def correlated():
|
||||
return (base + torch.randn(shape, device=dev, dtype=dtype) * 0.02).contiguous()
|
||||
|
||||
inp = dict(
|
||||
q=correlated(),
|
||||
k=correlated(),
|
||||
v=torch.randn(shape, device=dev, dtype=dtype) * 0.5,
|
||||
raw_g=torch.randn(shape, device=dev, dtype=dtype),
|
||||
raw_beta=torch.full((1, total, NUM_HEADS), 3.0, device=dev, dtype=dtype),
|
||||
A_log=torch.rand(NUM_HEADS, device=dev, dtype=torch.float32) * 0.6 - 0.42,
|
||||
dt_bias=torch.rand(NUM_HEADS * HEAD_DIM, device=dev, dtype=torch.float32) * 6.4
|
||||
- 7.8,
|
||||
h0=torch.zeros(
|
||||
1, NUM_HEADS, HEAD_DIM, HEAD_DIM, device=dev, dtype=torch.float32
|
||||
),
|
||||
cu=torch.tensor([0, total], device=dev, dtype=torch.int32),
|
||||
)
|
||||
|
||||
o_ref, state_ref = _run(inp, use_fused=False)
|
||||
o_fused, state_fused = _run(inp, use_fused=True)
|
||||
|
||||
# Looser than the tests above: near-identical keys plus a saturated beta
|
||||
# are past anything real text produces, and the inverse is rounded to bf16
|
||||
# before the w/u GEMMs. The tolerance still catches divergence, which in
|
||||
# this regime is orders of magnitude rather than a few times the bound.
|
||||
torch.testing.assert_close(o_fused.float(), o_ref.float(), rtol=5e-2, atol=5e-2)
|
||||
torch.testing.assert_close(state_fused, state_ref, rtol=5e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens",
|
||||
[
|
||||
[1], # every conv tap comes from the cache
|
||||
[2], # cache and batch both contribute
|
||||
[64],
|
||||
[1024],
|
||||
[513, 64, 1, 1200], # ragged, tail chunk shorter than the conv width
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("has_state", [False, True])
|
||||
def test_fused_conv_matches_reference_conv(seqlens: list[int], has_state: bool) -> None:
|
||||
"""The prologue's optional fused convolution must match a separate conv.
|
||||
|
||||
The layer does not use this path -- recomputing the conv once per load
|
||||
phase costs more than the ``causal_conv1d_fn`` launches it saves -- so it
|
||||
is driven through the ops directly rather than through
|
||||
``chunk_kda_prefill``, which does not expose it.
|
||||
|
||||
The reference is computed in torch rather than with ``causal_conv1d_fn``,
|
||||
which is not self-consistent for these shapes on ROCm (it leaves part of
|
||||
its output unwritten, so repeated identical calls disagree).
|
||||
|
||||
The cases that matter are the ones touching the sequence boundary, where
|
||||
the fused path takes taps from the conv cache instead of the batch:
|
||||
sequences shorter than the kernel width, and a cache that is or is not
|
||||
populated. The rolled-forward cache is checked too, since a wrong shift
|
||||
only shows up on the next step.
|
||||
"""
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
fused_kda_chunk,
|
||||
fused_kda_prologue,
|
||||
)
|
||||
|
||||
_requires_kernel()
|
||||
|
||||
dev, dtype = "cuda", torch.bfloat16
|
||||
lp = NUM_HEADS * HEAD_DIM
|
||||
width, n = 4, len(seqlens)
|
||||
inp = _inputs(seqlens, seed=7 + len(seqlens) + int(has_state))
|
||||
total = int(inp["cu"][-1])
|
||||
|
||||
torch.manual_seed(11 + int(has_state))
|
||||
# The layer hands the prologue a strided view of the QKVGFAB projection.
|
||||
projection = torch.randn(total, 3 * lp + 64, device=dev, dtype=dtype) * 0.5
|
||||
mixed = projection[:, : 3 * lp]
|
||||
# conv1d is built with bias=False and fp32 weights; the fused kernel takes
|
||||
# the width-major [qkv, width, channel] mirror the decode path already
|
||||
# keeps.
|
||||
weight = torch.randn(3, width, lp, device=dev, dtype=torch.float32) * 0.3
|
||||
store = torch.zeros(n, width - 1, 3 * lp, device=dev, dtype=dtype)
|
||||
if has_state:
|
||||
store.normal_(0.0, 0.5)
|
||||
state = store.transpose(-1, -2)
|
||||
idx = torch.arange(n, device=dev, dtype=torch.int32)
|
||||
has_init = torch.full((n,), has_state, device=dev, dtype=torch.bool)
|
||||
|
||||
# Reference conv, per sequence, in fp32 then rounded like a bf16 output.
|
||||
conv_ref = torch.empty(total, 3 * lp, device=dev, dtype=dtype)
|
||||
state_ref = torch.empty_like(store)
|
||||
flat_w = weight.permute(0, 2, 1).reshape(3 * lp, width)
|
||||
for i, (b, e) in enumerate(zip(inp["cu"][:-1].tolist(), inp["cu"][1:].tolist())):
|
||||
pre = (
|
||||
store[i].float()
|
||||
if has_state
|
||||
else torch.zeros(width - 1, 3 * lp, device=dev, dtype=torch.float32)
|
||||
)
|
||||
xs = torch.cat([pre, mixed[b:e].float()], 0)
|
||||
acc = torch.zeros(e - b, 3 * lp, device=dev, dtype=torch.float32)
|
||||
for w in range(width):
|
||||
acc += xs[w : w + (e - b)] * flat_w[:, w].unsqueeze(0)
|
||||
conv_ref[b:e] = F.silu(acc).to(dtype)
|
||||
state_ref[i] = xs[-(width - 1) :].to(dtype)
|
||||
|
||||
def _bands(x, contiguous):
|
||||
out = [
|
||||
b.unsqueeze(0).unflatten(-1, (NUM_HEADS, HEAD_DIM))
|
||||
for b in x.split(lp, dim=-1)
|
||||
]
|
||||
return [b.contiguous() for b in out] if contiguous else out
|
||||
|
||||
# Compare through both kernels rather than the prologue workspaces: `aqk`
|
||||
# is deliberately left unwritten past each chunk's valid rows, so those
|
||||
# elements are whatever the allocator last held.
|
||||
scale = HEAD_DIM**-0.5
|
||||
|
||||
def _through_kernels(q, k, v, conv: dict | None):
|
||||
ws = fused_kda_prologue(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=inp["raw_g"],
|
||||
raw_beta=inp["raw_beta"],
|
||||
A_log=inp["A_log"],
|
||||
dt_bias=inp["dt_bias"],
|
||||
scale=scale,
|
||||
lower_bound=LOWER_BOUND,
|
||||
cu_seqlens=inp["cu"],
|
||||
**(conv or {}),
|
||||
)
|
||||
return fused_kda_chunk(
|
||||
qg=ws["qg"],
|
||||
w=ws["w"],
|
||||
u=ws["u"],
|
||||
kg_t=ws["kg_t"],
|
||||
aqk=ws["aqk"],
|
||||
decay=ws["decay"],
|
||||
out=torch.empty_like(ws["u"]),
|
||||
scale=scale,
|
||||
cu_seqlens=inp["cu"],
|
||||
initial_state=inp["h0"].clone(),
|
||||
output_final_state=True,
|
||||
)
|
||||
|
||||
qr, kr, vr = _bands(conv_ref, True)
|
||||
o_ref, s_ref = _through_kernels(qr, kr, vr, None)
|
||||
|
||||
got_state = state.clone()
|
||||
qf, kf, vf = _bands(mixed, False)
|
||||
o_got, s_got = _through_kernels(
|
||||
qf,
|
||||
kf,
|
||||
vf,
|
||||
dict(
|
||||
conv_weight=weight,
|
||||
conv_state=got_state,
|
||||
conv_state_indices=idx,
|
||||
conv_has_initial_state=has_init,
|
||||
),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(s_got, s_ref, rtol=1e-3, atol=1e-3)
|
||||
torch.testing.assert_close(
|
||||
got_state.transpose(-1, -2).float(), state_ref.float(), rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4, 8])
|
||||
@pytest.mark.parametrize("seqlens", [[64], [64, 64], [100, 64], [513, 64, 1]])
|
||||
def test_chunk_group_scan_matches_serial_walk(
|
||||
groups: int, seqlens: list[int], monkeypatch
|
||||
) -> None:
|
||||
"""Splitting the chunk walk into parallel groups must not change the result.
|
||||
|
||||
Two bugs lived here, both invisible while the path was gated to >= 128
|
||||
chunks per sequence:
|
||||
|
||||
* a group with no chunks returned before writing its ``B_g``, so the scan
|
||||
folded whatever the workspace last held into the final state — these
|
||||
shapes have fewer chunks than groups, so most groups are empty;
|
||||
* the second pass started every group after the first from a zero state
|
||||
instead of the one the scan composed, which only looked right because a
|
||||
strongly decaying gate makes the dropped term small.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
inp = _inputs(seqlens, seed=5 + groups)
|
||||
_force_groups(monkeypatch, 1)
|
||||
o_ref, state_ref = _run(inp, use_fused=True)
|
||||
_force_groups(monkeypatch, groups)
|
||||
o_got, state_got = _run(inp, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seqlens", [[320], [513, 64, 1, 1200], [64, 64, 64]])
|
||||
def test_accepts_chunk_metadata_from_the_attention_builder(
|
||||
seqlens: list[int],
|
||||
) -> None:
|
||||
"""The layer passes precomputed chunk metadata, not None.
|
||||
|
||||
``KimiK3ROCmKDAMetadataBuilder`` builds these on device to avoid a blocking
|
||||
device->host copy. Passing ``None`` instead takes a different path that
|
||||
builds them locally, so drive the op with the builder's own tensors.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import prepare_chunk_metadata_device
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
|
||||
inp = _inputs(seqlens, seed=17)
|
||||
cu = inp["cu"]
|
||||
chunk_indices, chunk_offsets = prepare_chunk_metadata_device(
|
||||
cu, cu.cpu(), FLA_CHUNK_SIZE
|
||||
)
|
||||
|
||||
o_ref, state_ref = _run(inp, use_fused=True)
|
||||
o_got, state_got = _run(
|
||||
inp, use_fused=True, chunk_indices=chunk_indices, chunk_offsets=chunk_offsets
|
||||
)
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=2e-2, atol=2e-2)
|
||||
torch.testing.assert_close(state_got, state_ref, rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def _checkpoint_buffer(rows: int) -> torch.Tensor:
|
||||
"""A destination filled with NaN, so an unwritten row is unmistakable."""
|
||||
return torch.full(
|
||||
(rows, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
float("nan"),
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def _prefix_inputs(inp: dict, seq: int, num_tokens: int) -> dict:
|
||||
"""Sequence ``seq`` truncated to its first ``num_tokens`` tokens."""
|
||||
bos = int(inp["cu"][seq])
|
||||
tokens = slice(bos, bos + num_tokens)
|
||||
prefix = dict(inp)
|
||||
for key in ("q", "k", "v", "raw_g", "raw_beta"):
|
||||
prefix[key] = inp[key][:, tokens].contiguous()
|
||||
prefix["h0"] = inp["h0"][seq : seq + 1].contiguous()
|
||||
prefix["cu"] = torch.tensor([0, num_tokens], device="cuda", dtype=torch.int32)
|
||||
return prefix
|
||||
|
||||
|
||||
def _run_with_checkpoints(
|
||||
inp: dict, offsets: list[int], rows: int | None = None, **kw
|
||||
) -> tuple[torch.Tensor, tuple]:
|
||||
ckpt = _checkpoint_buffer(len(offsets) if rows is None else rows)
|
||||
result = _run(
|
||||
inp,
|
||||
use_fused=True,
|
||||
checkpoint_state=ckpt,
|
||||
checkpoint_offsets=torch.tensor(offsets, device="cuda", dtype=torch.int32),
|
||||
**kw,
|
||||
)
|
||||
return ckpt, result
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seqlens,offsets",
|
||||
[
|
||||
([256], [64]), # the first interior boundary
|
||||
([256], [192]), # the last one
|
||||
([1024], [512]),
|
||||
([320, 320], [64, 256]), # every sequence checkpoints
|
||||
([513, 64, 1, 1200], [448, 0, 0, 640]), # ragged, most opted out
|
||||
],
|
||||
)
|
||||
def test_checkpoint_matches_a_prefill_that_stops_at_the_offset(
|
||||
seqlens: list[int], offsets: list[int], monkeypatch
|
||||
) -> None:
|
||||
"""A checkpoint is only useful if resuming from it is indistinguishable.
|
||||
|
||||
The scheduler serves the exported state on a later prefix-cache hit as if
|
||||
the prompt had been prefilled up to that block and no further, so the
|
||||
kernel has to hand back exactly what a prefill truncated there produces --
|
||||
not merely something close. The walk passes through that state anyway, so
|
||||
equality is bit-exact and the test asserts it as such.
|
||||
|
||||
Sequences with offset 0 opt out, and their rows must stay untouched.
|
||||
"""
|
||||
_requires_kernel()
|
||||
_force_groups(monkeypatch, 1)
|
||||
|
||||
inp = _inputs(seqlens, seed=23 + len(seqlens))
|
||||
ckpt, _ = _run_with_checkpoints(inp, offsets)
|
||||
|
||||
for i, offset in enumerate(offsets):
|
||||
if offset == 0:
|
||||
assert ckpt[i].isnan().all()
|
||||
continue
|
||||
_, truncated = _run(_prefix_inputs(inp, i, offset), use_fused=True)
|
||||
torch.testing.assert_close(ckpt[i], truncated[0], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_checkpoint_leaves_the_output_and_final_state_alone() -> None:
|
||||
"""Exporting a checkpoint must not perturb the walk it is taken from."""
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [768, 320]
|
||||
inp = _inputs(seqlens, seed=29)
|
||||
o_ref, state_ref = _run(inp, use_fused=True)
|
||||
_, (o_got, state_got) = _run_with_checkpoints(inp, [512, 128])
|
||||
|
||||
torch.testing.assert_close(o_got.float(), o_ref.float(), rtol=0, atol=0)
|
||||
torch.testing.assert_close(state_got, state_ref, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("offset", [100, 63, 65, 1024])
|
||||
def test_unusable_checkpoint_offsets_write_nothing(offset: int) -> None:
|
||||
"""Offsets the walk never reaches must leave the destination untouched.
|
||||
|
||||
The recurrent state only exists at chunk boundaries, so an offset that is
|
||||
not a multiple of ``KDA_CHECKPOINT_ALIGNMENT`` -- or that lies past the end
|
||||
of the sequence -- cannot be honoured. Rounding it to a nearby boundary
|
||||
would hand the scheduler a state for the wrong token count, so the kernel
|
||||
stores nothing and leaves the caller to filter these out before the block
|
||||
is hashed.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import KDA_CHECKPOINT_ALIGNMENT
|
||||
|
||||
assert offset % KDA_CHECKPOINT_ALIGNMENT != 0 or offset > 512
|
||||
inp = _inputs([512], seed=31)
|
||||
ckpt, _ = _run_with_checkpoints(inp, [offset])
|
||||
|
||||
assert ckpt.isnan().all()
|
||||
|
||||
|
||||
def test_checkpoint_state_indices_pick_the_destination_row() -> None:
|
||||
"""With indices the export lands straight in the paged state cache.
|
||||
|
||||
FlashKDA writes one row per sequence into a staging buffer that the caller
|
||||
then scatters. The ROCm walk holds the state in registers already, so it
|
||||
can be told which cache block each sequence owns and skip the staging
|
||||
buffer and its scatter launch entirely. A negative row opts a sequence out,
|
||||
the same way a zero offset does.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
seqlens = [256, 256, 256]
|
||||
rows = 8
|
||||
inp = _inputs(seqlens, seed=37)
|
||||
indices = [5, -1, 2]
|
||||
ckpt, _ = _run_with_checkpoints(
|
||||
inp,
|
||||
[64, 64, 128],
|
||||
rows=rows,
|
||||
checkpoint_state_indices=torch.tensor(
|
||||
indices, device="cuda", dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
written = {5: (0, 64), 2: (2, 128)}
|
||||
for row in range(rows):
|
||||
if row not in written:
|
||||
assert ckpt[row].isnan().all(), f"row {row} was written"
|
||||
continue
|
||||
seq, offset = written[row]
|
||||
_, truncated = _run(_prefix_inputs(inp, seq, offset), use_fused=True)
|
||||
torch.testing.assert_close(ckpt[row], truncated[0], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
def test_checkpoint_survives_the_group_split(groups: int, monkeypatch) -> None:
|
||||
"""The split walks each group from a composed state, not from zero.
|
||||
|
||||
Only the second pass holds the true state at an interior boundary, so a
|
||||
checkpoint taken from the first pass would be the group-local state and
|
||||
silently wrong.
|
||||
"""
|
||||
_requires_kernel()
|
||||
|
||||
inp = _inputs([1024], seed=41)
|
||||
_force_groups(monkeypatch, 1)
|
||||
_, serial = _run(_prefix_inputs(inp, 0, 512), use_fused=True)
|
||||
_force_groups(monkeypatch, groups)
|
||||
ckpt, _ = _run_with_checkpoints(inp, [512])
|
||||
|
||||
torch.testing.assert_close(ckpt[0], serial[0], rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
def test_triton_fallback_refuses_a_checkpoint_export() -> None:
|
||||
"""Dropping the export silently would serve an unwritten state.
|
||||
|
||||
The scheduler allocates and hashes the checkpoint block on the strength of
|
||||
the layer opting in, so a backend that cannot write it has to say so rather
|
||||
than return a plausible output.
|
||||
"""
|
||||
inp = _inputs([256], seed=43)
|
||||
with pytest.raises(NotImplementedError):
|
||||
_run(
|
||||
inp,
|
||||
use_fused=False,
|
||||
checkpoint_state=_checkpoint_buffer(1),
|
||||
checkpoint_offsets=torch.tensor([64], device="cuda", dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paged recurrent state
|
||||
#
|
||||
# Each sequence reads its initial state from a row of the paged cache and
|
||||
# writes its final state back to the same row. These check that the result
|
||||
# matches the dense initial_state/final_state pair, that a sequence marked
|
||||
# cold starts from zeros rather than from whatever its row held, and that rows
|
||||
# no sequence owns are left untouched.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _paged_cache(inp: dict, warm: list[bool], rows: list[int], slots: int):
|
||||
"""A cache holding each sequence's h0 at its row, a fill pattern elsewhere."""
|
||||
cache = torch.full(
|
||||
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
-7.0,
|
||||
device="cuda",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
|
||||
for n, row in enumerate(rows):
|
||||
# A cold row keeps its fill pattern, which must not reach the walk.
|
||||
if warm[n]:
|
||||
cache[row] = inp["h0"][n]
|
||||
return cache, idx, torch.tensor(warm, device="cuda", dtype=torch.bool)
|
||||
|
||||
|
||||
def _run_paged(inp: dict, cache, idx, warm, use_fused: bool, **kw):
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
|
||||
return chunk_kda_prefill(
|
||||
q=inp["q"].clone(),
|
||||
k=inp["k"].clone(),
|
||||
v=inp["v"].clone(),
|
||||
raw_g=inp["raw_g"].clone(),
|
||||
raw_beta=inp["raw_beta"],
|
||||
A_log=inp["A_log"],
|
||||
g_bias=inp["dt_bias"],
|
||||
lower_bound=LOWER_BOUND,
|
||||
state_cache=cache,
|
||||
state_indices=idx,
|
||||
has_initial_state=warm,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=inp["cu"],
|
||||
use_fused_chunk=use_fused,
|
||||
**kw,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
@pytest.mark.parametrize("seqlens", [[512], [2048], [320, 64], [513, 64, 1]])
|
||||
def test_paged_state_matches_the_dense_pair(
|
||||
use_fused: bool, seqlens: list[int]
|
||||
) -> None:
|
||||
"""A paged cache must give the same result as the dense pair."""
|
||||
_requires_kernel()
|
||||
inp = _inputs(seqlens, seed=11)
|
||||
n = len(seqlens)
|
||||
rows = [3 * i + 5 for i in range(n)]
|
||||
cache, idx, warm = _paged_cache(inp, [True] * n, rows, slots=3 * n + 12)
|
||||
before = cache.clone()
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=use_fused)
|
||||
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
|
||||
|
||||
assert ht_got is None, "the final state is in the cache, not returned"
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
untouched = torch.ones(cache.shape[0], dtype=torch.bool, device="cuda")
|
||||
untouched[idx.long()] = False
|
||||
assert torch.equal(cache[untouched], before[untouched]), (
|
||||
"a row no sequence owns was written"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_cold_rows_start_from_zero_not_from_cache_junk(use_fused: bool) -> None:
|
||||
"""`has_initial_state=False` must ignore whatever the row already holds."""
|
||||
_requires_kernel()
|
||||
seqlens = [256, 256, 256]
|
||||
inp = _inputs(seqlens, seed=12)
|
||||
warm = [True, False, True]
|
||||
rows = [4, 9, 1]
|
||||
cache, idx, warm_t = _paged_cache(inp, warm, rows, slots=16)
|
||||
|
||||
# The dense reference zeroes exactly the cold sequences' initial states.
|
||||
ref = dict(inp)
|
||||
ref["h0"] = inp["h0"].clone()
|
||||
ref["h0"][1] = 0.0
|
||||
o_ref, ht_ref = _run(ref, use_fused=use_fused)
|
||||
o_got, _ = _run_paged(inp, cache, idx, warm_t, use_fused=use_fused)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("groups", [2, 4])
|
||||
def test_paged_state_survives_the_group_split(groups: int, monkeypatch) -> None:
|
||||
"""At G>1 the read and the write are different blocks of one launch."""
|
||||
_requires_kernel()
|
||||
_force_groups(monkeypatch, groups)
|
||||
inp = _inputs([1024, 512], seed=13)
|
||||
rows = [6, 2]
|
||||
cache, idx, warm = _paged_cache(inp, [True, True], rows, slots=12)
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=True)
|
||||
o_got, _ = _run_paged(inp, cache, idx, warm, use_fused=True)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
def test_paged_state_and_checkpoint_export_together() -> None:
|
||||
"""The checkpoint lands in its own row while the walk owns the state row."""
|
||||
_requires_kernel()
|
||||
inp = _inputs([1024], seed=14)
|
||||
cache, idx, warm = _paged_cache(inp, [True], [3], slots=12)
|
||||
ckidx = torch.tensor([8], device="cuda", dtype=torch.int32)
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=True)
|
||||
prefix = _prefix_inputs(inp, 0, 512)
|
||||
_, ck_ref = _run(prefix, use_fused=True)
|
||||
|
||||
o_got, _ = _run_paged(
|
||||
inp,
|
||||
cache,
|
||||
idx,
|
||||
warm,
|
||||
use_fused=True,
|
||||
checkpoint_state=cache,
|
||||
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
|
||||
checkpoint_state_indices=ckidx,
|
||||
)
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[ckidx.long()], ck_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
def test_paged_state_rejects_a_dense_initial_state() -> None:
|
||||
"""The two ways of passing the state are mutually exclusive."""
|
||||
_requires_kernel()
|
||||
inp = _inputs([256], seed=15)
|
||||
cache, idx, warm = _paged_cache(inp, [True], [1], slots=4)
|
||||
with pytest.raises(ValueError, match="replaces initial_state"):
|
||||
_run_paged(
|
||||
inp,
|
||||
cache,
|
||||
idx,
|
||||
warm,
|
||||
use_fused=True,
|
||||
initial_state=inp["h0"].clone(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_padded_cache_stride_is_honoured(use_fused: bool) -> None:
|
||||
"""A mamba page may pad stride(0); only each state row is dense.
|
||||
|
||||
The real `recurrent_state` is not a contiguous tensor, so a kernel that
|
||||
assumes a packed ``H * V * K`` row stride addresses the wrong rows. The
|
||||
other cases in this file all build a contiguous cache, which cannot
|
||||
exercise that.
|
||||
"""
|
||||
_requires_kernel()
|
||||
seqlens = [1024, 512]
|
||||
inp = _inputs(seqlens, seed=21)
|
||||
n = len(seqlens)
|
||||
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
|
||||
slots, pad = 16, 512
|
||||
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
|
||||
cache = raw.as_strided(
|
||||
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
|
||||
)
|
||||
assert not cache.is_contiguous() and cache[0].is_contiguous()
|
||||
|
||||
rows = [11, 3]
|
||||
idx = torch.tensor(rows, device="cuda", dtype=torch.int32)
|
||||
warm = torch.ones(n, dtype=torch.bool, device="cuda")
|
||||
for i, r in enumerate(rows):
|
||||
cache[r] = inp["h0"][i]
|
||||
before = raw.clone()
|
||||
|
||||
o_ref, ht_ref = _run(inp, use_fused=use_fused)
|
||||
o_got, ht_got = _run_paged(inp, cache, idx, warm, use_fused=use_fused)
|
||||
|
||||
assert ht_got is None
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
# The padding between rows, and every unowned row, must be untouched.
|
||||
untouched = torch.ones(slots, dtype=torch.bool, device="cuda")
|
||||
untouched[idx.long()] = False
|
||||
assert torch.equal(raw[untouched], before[untouched])
|
||||
assert torch.equal(raw[idx.long(), row:], before[idx.long(), row:]), (
|
||||
"the kernel wrote into the inter-row padding"
|
||||
)
|
||||
|
||||
|
||||
def test_padded_checkpoint_stride_is_honoured() -> None:
|
||||
"""The checkpoint destination is the paged cache, which pads stride(0).
|
||||
|
||||
Every other checkpoint case here builds a contiguous buffer, so none of
|
||||
them would catch a kernel that assumes a packed ``H * V * K`` row stride
|
||||
and writes the snapshot into a neighbouring block.
|
||||
"""
|
||||
_requires_kernel()
|
||||
seqlens = [1024]
|
||||
inp = _inputs(seqlens, seed=31)
|
||||
row = NUM_HEADS * HEAD_DIM * HEAD_DIM
|
||||
slots, pad = 16, 512
|
||||
raw = torch.full((slots, row + pad), -7.0, device="cuda", dtype=torch.float32)
|
||||
cache = raw.as_strided(
|
||||
(slots, NUM_HEADS, HEAD_DIM, HEAD_DIM),
|
||||
(row + pad, HEAD_DIM * HEAD_DIM, HEAD_DIM, 1),
|
||||
)
|
||||
assert not cache.is_contiguous()
|
||||
|
||||
state_rows = [11]
|
||||
idx = torch.tensor(state_rows, device="cuda", dtype=torch.int32)
|
||||
warm = torch.ones(1, dtype=torch.bool, device="cuda")
|
||||
cache[state_rows[0]] = inp["h0"][0]
|
||||
ck_rows = [4]
|
||||
before = raw.clone()
|
||||
|
||||
prefix = _prefix_inputs(inp, 0, 512)
|
||||
_, ck_ref = _run(prefix, use_fused=True)
|
||||
o_ref, ht_ref = _run(inp, use_fused=True)
|
||||
|
||||
o_got, _ = _run_paged(
|
||||
inp,
|
||||
cache,
|
||||
idx,
|
||||
warm,
|
||||
use_fused=True,
|
||||
checkpoint_state=cache,
|
||||
checkpoint_offsets=torch.tensor([512], device="cuda", dtype=torch.int32),
|
||||
checkpoint_state_indices=torch.tensor(
|
||||
ck_rows, device="cuda", dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
torch.testing.assert_close(o_got, o_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[idx.long()], ht_ref, rtol=5e-3, atol=5e-3)
|
||||
torch.testing.assert_close(cache[ck_rows[0]], ck_ref[0], rtol=5e-3, atol=5e-3)
|
||||
|
||||
touched = torch.zeros(slots, dtype=torch.bool, device="cuda")
|
||||
touched[state_rows + ck_rows] = True
|
||||
assert torch.equal(raw[~touched], before[~touched])
|
||||
assert torch.equal(raw[touched, row:], before[touched, row:]), (
|
||||
"the kernel wrote into the inter-row padding"
|
||||
)
|
||||
@@ -778,9 +778,9 @@ class EngineArgs:
|
||||
|
||||
fail_on_environ_validation: bool = False
|
||||
gdn_prefill_backend: Literal["flashinfer", "triton", "cutedsl"] | None = None
|
||||
kda_prefill_backend: Literal["auto", "triton", "flashkda", "flashinfer"] | None = (
|
||||
None
|
||||
)
|
||||
kda_prefill_backend: (
|
||||
Literal["auto", "triton", "flashkda", "flashinfer", "fused"] | None
|
||||
) = None
|
||||
kda_decode_backend: Literal["auto", "native", "flashinfer", "triton"] | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
@@ -1761,9 +1761,10 @@ class EngineArgs:
|
||||
parser.add_argument(
|
||||
"--kda-prefill-backend",
|
||||
dest="kda_prefill_backend",
|
||||
choices=["auto", "triton", "flashkda", "flashinfer"],
|
||||
choices=["auto", "triton", "flashkda", "flashinfer", "fused"],
|
||||
default=None,
|
||||
help="Select KDA prefill backend.",
|
||||
help="Select KDA prefill backend. 'flashkda' is CUDA-only and "
|
||||
"'fused' is ROCm-only; 'auto' picks a supported backend.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kda-decode-backend",
|
||||
@@ -2597,6 +2598,17 @@ class EngineArgs:
|
||||
if self.gdn_prefill_backend is not None:
|
||||
self.additional_config["gdn_prefill_backend"] = self.gdn_prefill_backend
|
||||
if self.kda_prefill_backend is not None:
|
||||
if (
|
||||
self.kda_prefill_backend == "flashkda"
|
||||
and not current_platform.is_cuda()
|
||||
):
|
||||
raise ValueError(
|
||||
"--kda-prefill-backend=flashkda is only available on CUDA."
|
||||
)
|
||||
if self.kda_prefill_backend == "fused" and not current_platform.is_rocm():
|
||||
raise ValueError(
|
||||
"--kda-prefill-backend=fused is only available on ROCm."
|
||||
)
|
||||
self.additional_config["kda_prefill_backend"] = self.kda_prefill_backend
|
||||
if self.kda_decode_backend is not None:
|
||||
self.additional_config["kda_decode_backend"] = self.kda_decode_backend
|
||||
|
||||
@@ -35,19 +35,19 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn,
|
||||
causal_conv1d_update,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
is_fused_kda_chunk_supported,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_decode import (
|
||||
is_fused_kda_decode_supported,
|
||||
make_decode_conv1d_weight_loader,
|
||||
make_decode_norm_weight_loader,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
@@ -210,10 +210,21 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The ROCm Kimi-K3 KDA layer only supports the Triton KDA prefill "
|
||||
f"backend, got {backend!r}."
|
||||
assert backend in ("auto", "triton", "fused"), (
|
||||
"The ROCm Kimi-K3 KDA prefill backend must be one of "
|
||||
f"'auto', 'triton' or 'fused', got {backend!r}."
|
||||
)
|
||||
if backend == "fused" and not is_fused_kda_chunk_supported():
|
||||
raise RuntimeError(
|
||||
"The fused KDA chunk kernel requires gfx950 and a build that "
|
||||
"includes it."
|
||||
)
|
||||
self.use_fused_chunk = backend == "fused" or (
|
||||
backend == "auto" and is_fused_kda_chunk_supported()
|
||||
)
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill backend: %s",
|
||||
"fused" if self.use_fused_chunk else "triton",
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
@@ -498,6 +509,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
# prefill tail only.
|
||||
core_attn_out_decode = None
|
||||
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
|
||||
# Without a spec split the non-spec tokens are exactly
|
||||
# core_attn_out[:, :num_actual_tokens] in order, so both kernels
|
||||
# can write their slice straight into it. With one, the tokens
|
||||
# are interleaved and have to be scattered by index afterwards.
|
||||
direct = spec_sequence_masks is None
|
||||
use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None
|
||||
if split_non_spec:
|
||||
assert non_spec_query_start_loc is not None
|
||||
nd_tok = m.num_decode_tokens
|
||||
@@ -515,6 +532,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
ssm_state_indices=non_spec_state_indices_tensor[
|
||||
: m.num_decodes
|
||||
],
|
||||
out=core_attn_out[:, :nd_tok] if direct else None,
|
||||
)
|
||||
q_ns = q_ns[:, nd_tok:]
|
||||
k_ns = k_ns[:, nd_tok:]
|
||||
@@ -531,16 +549,12 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
prefill_query_start_loc = non_spec_query_start_loc
|
||||
prefill_state_indices = non_spec_state_indices_tensor
|
||||
prefill_has_initial_state = has_initial_state
|
||||
nd_tok = 0
|
||||
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
prefill_state_indices,
|
||||
prefill_has_initial_state,
|
||||
)
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_with_fused_gate(
|
||||
_,
|
||||
) = chunk_kda_prefill(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
@@ -549,17 +563,23 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
A_log=self.A_log,
|
||||
g_bias=self.dt_bias,
|
||||
lower_bound=self.gate_lower_bound,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
state_cache=recurrent_state,
|
||||
state_indices=prefill_state_indices,
|
||||
has_initial_state=prefill_has_initial_state,
|
||||
use_qk_l2norm_in_kernel=True,
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
chunk_indices=m.chunk_indices,
|
||||
chunk_offsets=m.chunk_offsets,
|
||||
use_fused_chunk=use_fused_chunk,
|
||||
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[prefill_state_indices] = last_recurrent_state
|
||||
# chunk_kda_prefill updates `recurrent_state` in place, so
|
||||
# there is no final state to write back here.
|
||||
|
||||
if split_non_spec:
|
||||
if direct:
|
||||
# Both slices already landed in core_attn_out.
|
||||
core_attn_out_non_spec = core_attn_out[:, :num_actual_tokens]
|
||||
elif split_non_spec:
|
||||
# Restore decode-first token order for the merge below.
|
||||
core_attn_out_non_spec = torch.cat(
|
||||
[core_attn_out_decode, core_attn_out_non_spec], dim=1
|
||||
@@ -611,9 +631,10 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
|
||||
|
||||
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
|
||||
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
|
||||
path with a single launch that keeps the per-chunk state in registers, so the
|
||||
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
|
||||
"""
|
||||
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.index import (
|
||||
prepare_chunk_indices,
|
||||
prepare_chunk_offsets,
|
||||
)
|
||||
|
||||
CHUNK_SIZE = 64
|
||||
HEAD_DIM = 128
|
||||
KDA_CHECKPOINT_ALIGNMENT = CHUNK_SIZE
|
||||
# Chunk-group split policy constants; see _chunk_groups.
|
||||
_BLOCKS_P1 = 2 # (kV + kK) / kBV
|
||||
_BLOCKS_P2 = 1 # kV / kBV
|
||||
_MIN_LEN = 4
|
||||
_DEEP_LEN = 12
|
||||
_MAX_GROUPS = 32
|
||||
|
||||
|
||||
@cache
|
||||
def is_fused_kda_chunk_supported() -> bool:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not hasattr(torch.ops._C, "fused_kda_chunk"):
|
||||
return False
|
||||
# TODO: Verify on other archs, currently only validated on gfx950
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def can_use_fused_kda_chunk(
|
||||
head_k_dim: int,
|
||||
head_v_dim: int,
|
||||
dtype: torch.dtype,
|
||||
chunk_size: int,
|
||||
) -> bool:
|
||||
return (
|
||||
head_k_dim == HEAD_DIM
|
||||
and head_v_dim == HEAD_DIM
|
||||
and chunk_size == CHUNK_SIZE
|
||||
and dtype == torch.bfloat16
|
||||
and is_fused_kda_chunk_supported()
|
||||
)
|
||||
|
||||
|
||||
def fused_kda_prologue(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
scale: float,
|
||||
lower_bound: float,
|
||||
cu_seqlens: torch.Tensor,
|
||||
conv_weight: torch.Tensor | None = None,
|
||||
conv_state: torch.Tensor | None = None,
|
||||
conv_state_indices: torch.Tensor | None = None,
|
||||
conv_has_initial_state: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Run the whole chunk-path prologue in one launch.
|
||||
|
||||
Replaces the two L2 norms, the gate cumsum, both intra-chunk passes and the
|
||||
w/u recompute. Returns the operands the fused chunk kernel consumes.
|
||||
|
||||
The kernel can also apply the depthwise conv and its silu, in which case
|
||||
``q``/``k``/``v`` are the three raw bands of the QKV projection and
|
||||
``conv_state`` is updated in place. This path is currently not used.
|
||||
"""
|
||||
# `beta` and `g` reach the layer as last-dim slices of the fused QKVGFAB
|
||||
# projection and carry its row stride. The kernel reads both with an
|
||||
# explicit per-token stride, so neither is copied here.
|
||||
_, t_total, num_heads, _ = q.shape
|
||||
if chunk_indices is None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
|
||||
chunk_indices = chunk_indices.to(torch.int32)
|
||||
num_chunks = chunk_indices.shape[0]
|
||||
dev = q.device
|
||||
|
||||
# q may be a strided band view, so the workspaces are sized rather than
|
||||
# cloned from it.
|
||||
def _like(last: int) -> torch.Tensor:
|
||||
return torch.empty(1, t_total, num_heads, last, dtype=q.dtype, device=q.device)
|
||||
|
||||
ws = dict(
|
||||
qg=_like(HEAD_DIM),
|
||||
w=_like(HEAD_DIM),
|
||||
u=_like(HEAD_DIM),
|
||||
kg_t=torch.empty(
|
||||
num_chunks, num_heads, HEAD_DIM, CHUNK_SIZE, dtype=q.dtype, device=dev
|
||||
),
|
||||
aqk=torch.empty(1, t_total, num_heads, CHUNK_SIZE, dtype=q.dtype, device=dev),
|
||||
decay=torch.empty(
|
||||
num_chunks, num_heads, HEAD_DIM, dtype=torch.float32, device=dev
|
||||
),
|
||||
)
|
||||
torch.ops._C.fused_kda_prologue(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log.reshape(-1),
|
||||
dt_bias.reshape(-1),
|
||||
ws["qg"],
|
||||
ws["w"],
|
||||
ws["u"],
|
||||
ws["kg_t"],
|
||||
ws["aqk"],
|
||||
ws["decay"],
|
||||
cu_seqlens.to(torch.int32),
|
||||
chunk_indices,
|
||||
conv_weight,
|
||||
conv_state,
|
||||
conv_state_indices,
|
||||
conv_has_initial_state,
|
||||
scale,
|
||||
lower_bound,
|
||||
)
|
||||
return ws
|
||||
|
||||
|
||||
@cache
|
||||
def _num_cus(device: int) -> int:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.get_device_properties(device).multi_processor_count
|
||||
|
||||
|
||||
@cache
|
||||
def _chunk_groups(chunks_per_seq: int, num_seqs: int, num_heads: int) -> int:
|
||||
"""How many parallel chunk groups to cut each sequence into.
|
||||
|
||||
The scan composes the group operators with the exact transfer
|
||||
``M_g = prod_c (diag(d_c) - w_c^T kg_c)``, so the composition error does
|
||||
not grow with group length and the choice is purely a machine fit:
|
||||
|
||||
* ``fill1``/``fill2``: the largest G whose pass-one / pass-two grid still
|
||||
fits in one scheduling batch. Rows are launched in blocks of at most
|
||||
``kBV = 128``, so pass one (``kV + kK`` rows) needs two blocks per
|
||||
``(n, h, g)`` and pass two one. Past a pass's fill point its block count
|
||||
grows as fast as its depth shrinks and the rung stops paying.
|
||||
* ``_DEEP_LEN``: unless the groups are still long, where pass two's halving
|
||||
is worth letting pass one spill to a second batch.
|
||||
* ``_MIN_LEN``: below this many chunks per group the per-group fixed cost
|
||||
exceeds what the shorter walk saves.
|
||||
|
||||
``G == 2`` is depth-neutral -- two passes of ``nt/2`` -- and only doubles
|
||||
staging, so the split is taken only from ``G >= 4``.
|
||||
"""
|
||||
cus = _num_cus(torch.accelerator.current_device_index())
|
||||
nh = max(num_seqs * num_heads, 1)
|
||||
fill1 = cus // (_BLOCKS_P1 * nh)
|
||||
fill2 = cus // (_BLOCKS_P2 * nh)
|
||||
cand = fill1
|
||||
if fill2 > 0 and chunks_per_seq // fill2 >= _DEEP_LEN:
|
||||
cand = fill2
|
||||
cand = min(cand, chunks_per_seq // _MIN_LEN, _MAX_GROUPS)
|
||||
return cand if cand >= 4 else 1
|
||||
|
||||
|
||||
def _kda_group_workspace(
|
||||
groups: int, nh: int, device: torch.device
|
||||
) -> torch.Tensor | None:
|
||||
"""One fp32 buffer the kernel carves into bg / sin_ / ag / mgT.
|
||||
|
||||
Kept caller-owned so the memory stays inside the caching allocator and
|
||||
vLLM's memory profiling accounts for it; the kernel's carve order must
|
||||
match the layout here.
|
||||
"""
|
||||
if groups <= 1:
|
||||
return None
|
||||
planes = groups * nh
|
||||
plane = HEAD_DIM * HEAD_DIM
|
||||
floats = 2 * planes * plane + planes * HEAD_DIM # bg, sin_, ag
|
||||
floats += (planes * plane + 1) // 2 # mgT, bf16
|
||||
return torch.empty(floats, dtype=torch.float32, device=device)
|
||||
|
||||
|
||||
def fused_kda_chunk(
|
||||
qg: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
u: torch.Tensor,
|
||||
kg_t: torch.Tensor,
|
||||
aqk: torch.Tensor,
|
||||
decay: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
scale: float,
|
||||
cu_seqlens: torch.Tensor,
|
||||
initial_state: torch.Tensor | None,
|
||||
output_final_state: bool,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
checkpoint_state: torch.Tensor | None = None,
|
||||
checkpoint_offsets: torch.Tensor | None = None,
|
||||
checkpoint_state_indices: torch.Tensor | None = None,
|
||||
state_cache: torch.Tensor | None = None,
|
||||
state_indices: torch.Tensor | None = None,
|
||||
has_initial_state: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Run the chunk recurrence and the output projection in one launch.
|
||||
|
||||
Args:
|
||||
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
|
||||
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
|
||||
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
|
||||
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
|
||||
checkpoint_state: destination for the mid-prefill state snapshots,
|
||||
fp32 ``[rows, H, 128, 128]``. Without
|
||||
``checkpoint_state_indices`` it is a staging buffer indexed by
|
||||
sequence; with them it can be the paged state cache itself.
|
||||
checkpoint_offsets: int32 ``[N]`` token offsets, each relative to its
|
||||
sequence's first query token, at which to snapshot the recurrent
|
||||
state. ``0`` disables the export for that sequence. A non-zero
|
||||
offset must be a multiple of :data:`KDA_CHECKPOINT_ALIGNMENT`;
|
||||
the kernel skips any that is not, so the caller has to filter
|
||||
rather than rely on the store happening.
|
||||
checkpoint_state_indices: optional int32 ``[N]`` destination row per
|
||||
sequence. A negative row disables the export for that sequence.
|
||||
state_cache: the paged recurrent state, fp32
|
||||
``[slots, H, 128, 128]``. When given, the walk reads sequence
|
||||
``n``'s initial state from row ``state_indices[n]`` and writes its
|
||||
final state back to the same row, which removes the gather and the
|
||||
scatter that otherwise bracket this kernel. Replaces
|
||||
``initial_state`` / ``output_final_state``, and the returned final
|
||||
state is then ``None`` because it is already in the cache.
|
||||
state_indices: int32 ``[N]`` cache row per sequence.
|
||||
has_initial_state: bool ``[N]``; a false entry starts that sequence
|
||||
from a zero state and its cache row is read only, never before the
|
||||
walk writes it.
|
||||
"""
|
||||
if state_cache is not None:
|
||||
if initial_state is not None or output_final_state:
|
||||
raise ValueError("state_cache replaces initial_state/output_final_state")
|
||||
if state_indices is None or has_initial_state is None:
|
||||
raise ValueError("state_cache needs state_indices and has_initial_state")
|
||||
num_seqs = cu_seqlens.numel() - 1
|
||||
final_state = None
|
||||
if output_final_state:
|
||||
final_state = torch.empty(
|
||||
num_seqs,
|
||||
u.shape[2],
|
||||
u.shape[3],
|
||||
qg.shape[3],
|
||||
dtype=torch.float32,
|
||||
device=u.device,
|
||||
)
|
||||
cu_seqlens = cu_seqlens.to(torch.int32)
|
||||
if chunk_offsets is None:
|
||||
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
|
||||
|
||||
# The chunk walk is serial in chunks. Splitting each sequence into `groups`
|
||||
# stretches that run in parallel costs a second pass and a scan, and only
|
||||
# pays once the chain is long enough to amortise both.
|
||||
chunks_per_seq = kg_t.shape[0] // max(num_seqs, 1)
|
||||
groups = _chunk_groups(chunks_per_seq, num_seqs, u.shape[2])
|
||||
group_state = _kda_group_workspace(groups, num_seqs * u.shape[2], u.device)
|
||||
|
||||
torch.ops._C.fused_kda_chunk(
|
||||
qg,
|
||||
w,
|
||||
u,
|
||||
kg_t,
|
||||
aqk,
|
||||
decay,
|
||||
initial_state,
|
||||
final_state,
|
||||
out,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
scale,
|
||||
group_state,
|
||||
groups,
|
||||
checkpoint_state,
|
||||
None if checkpoint_offsets is None else checkpoint_offsets.to(torch.int32),
|
||||
None
|
||||
if checkpoint_state_indices is None
|
||||
else checkpoint_state_indices.to(torch.int32),
|
||||
state_cache,
|
||||
None if state_indices is None else state_indices.to(torch.int32),
|
||||
has_initial_state,
|
||||
)
|
||||
return out, final_state
|
||||
@@ -0,0 +1,179 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""KDA prefill backend selection for ROCm.
|
||||
|
||||
The Kimi-K3 KDA layer calls :func:`chunk_kda_prefill`, which either runs the
|
||||
fused HIP kernels in ``kda_chunk`` or falls back to the vendored Triton chunk
|
||||
path.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
can_use_fused_kda_chunk,
|
||||
fused_kda_chunk,
|
||||
fused_kda_prologue,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import chunk_kda_with_fused_gate
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def chunk_kda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None = None,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
lower_bound: float | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_fused_chunk: bool = False,
|
||||
out: torch.Tensor | None = None,
|
||||
checkpoint_state: torch.Tensor | None = None,
|
||||
checkpoint_offsets: torch.Tensor | None = None,
|
||||
checkpoint_state_indices: torch.Tensor | None = None,
|
||||
state_cache: torch.Tensor | None = None,
|
||||
state_indices: torch.Tensor | None = None,
|
||||
has_initial_state: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Run chunk KDA from raw gate and beta projections.
|
||||
|
||||
Args:
|
||||
use_fused_chunk: request the two-kernel ROCm path. It is used only when
|
||||
every one of its preconditions holds; otherwise the Triton path
|
||||
runs unchanged.
|
||||
out: buffer the result must land in. Honoured by both backends, so the
|
||||
caller can hand in a slice of its own output and skip a copy.
|
||||
checkpoint_state: destination for mid-prefill recurrent state
|
||||
snapshots, letting a later prefix-cache hit resume from a mamba
|
||||
block boundary. See :func:`fused_kda_chunk`.
|
||||
checkpoint_offsets: per-sequence token offset to snapshot at, ``0``
|
||||
for none.
|
||||
checkpoint_state_indices: optional per-sequence destination row.
|
||||
state_cache: the paged recurrent state. When given, the fused backend
|
||||
reads and writes it in place and neither a gather nor a scatter is
|
||||
needed around this call; the returned final state is ``None``.
|
||||
state_indices: per-sequence cache row.
|
||||
has_initial_state: per-sequence flag; false starts from a zero state.
|
||||
|
||||
Returns:
|
||||
The output and, when requested, the final recurrent state.
|
||||
"""
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
# The fused prologue folds the q/k L2 norm and the gate activation in, so it
|
||||
# needs the raw projections and a bounded gate rather than the pre-normalized
|
||||
# tensors the Triton path takes.
|
||||
fused = (
|
||||
use_fused_chunk
|
||||
and use_qk_l2norm_in_kernel
|
||||
and cu_seqlens is not None
|
||||
and lower_bound is not None
|
||||
and g_bias is not None
|
||||
and can_use_fused_kda_chunk(k.shape[-1], v.shape[-1], k.dtype, FLA_CHUNK_SIZE)
|
||||
)
|
||||
|
||||
if checkpoint_offsets is not None and not fused:
|
||||
raise NotImplementedError(
|
||||
"The KDA prefill checkpoint export needs kda_prefill_backend=fused for ROCm"
|
||||
)
|
||||
# Checked here rather than in the backend so both paths reject the same
|
||||
# arguments.
|
||||
if state_cache is not None:
|
||||
if initial_state is not None or output_final_state:
|
||||
raise ValueError("state_cache replaces initial_state/output_final_state")
|
||||
if state_indices is None or has_initial_state is None:
|
||||
raise ValueError("state_cache needs state_indices and has_initial_state")
|
||||
|
||||
# Only the fused walk addresses the paged rows directly. The Triton path
|
||||
# gathers the rows it needs and scatters the results back here, so callers
|
||||
# pass the cache the same way for either backend.
|
||||
scatter_to: torch.Tensor | None = None
|
||||
if state_cache is not None and not fused:
|
||||
initial_state = gather_initial_states(
|
||||
state_cache, state_indices, has_initial_state
|
||||
)
|
||||
output_final_state = True
|
||||
scatter_to, state_cache = state_cache, None
|
||||
|
||||
if fused:
|
||||
# Restated for the type checker; `fused` already implies all three.
|
||||
assert cu_seqlens is not None and lower_bound is not None
|
||||
assert g_bias is not None
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill: dispatching the fused ROCm chunk kernel."
|
||||
)
|
||||
ws = fused_kda_prologue(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=g_bias,
|
||||
scale=scale,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
)
|
||||
return fused_kda_chunk(
|
||||
qg=ws["qg"],
|
||||
w=ws["w"],
|
||||
u=ws["u"],
|
||||
kg_t=ws["kg_t"],
|
||||
aqk=ws["aqk"],
|
||||
decay=ws["decay"],
|
||||
# v is dead by this point, so it doubles as the output buffer.
|
||||
out=out if out is not None else v,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
chunk_offsets=chunk_offsets,
|
||||
checkpoint_state=checkpoint_state,
|
||||
checkpoint_offsets=checkpoint_offsets,
|
||||
checkpoint_state_indices=checkpoint_state_indices,
|
||||
state_cache=state_cache,
|
||||
state_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
)
|
||||
|
||||
o, final_state = chunk_kda_with_fused_gate(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
lower_bound=lower_bound,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
if out is not None and o.data_ptr() != out.data_ptr():
|
||||
out.copy_(o)
|
||||
o = out
|
||||
if scatter_to is not None:
|
||||
assert state_indices is not None
|
||||
scatter_to[state_indices.long()] = final_state
|
||||
return o, None
|
||||
return o, final_state
|
||||
Reference in New Issue
Block a user