Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
314 lines
12 KiB
Python
314 lines
12 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
|
|
|
|
The kernel in ``csrc/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
|
|
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
|
|
path with a single launch that keeps the per-chunk state in registers, so the
|
|
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
|
|
"""
|
|
|
|
from functools import cache
|
|
|
|
import torch
|
|
|
|
from vllm.third_party.flash_linear_attention.ops.index import (
|
|
prepare_chunk_indices,
|
|
prepare_chunk_offsets,
|
|
)
|
|
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
|
|
|
CHUNK_SIZE = 64
|
|
HEAD_DIM = 128
|
|
KDA_CHECKPOINT_ALIGNMENT = CHUNK_SIZE
|
|
# Chunk-group split policy constants; see _chunk_groups.
|
|
_BLOCKS_P1 = 2 # (kV + kK) / kBV
|
|
_BLOCKS_P2 = 1 # kV / kBV
|
|
_MIN_LEN = 4
|
|
_DEEP_LEN = 12
|
|
_MAX_GROUPS = 32
|
|
|
|
|
|
def _hip_checkpoint_state_indices(state_indices: torch.Tensor) -> torch.Tensor:
|
|
"""NULL_BLOCK_ID marks no checkpoint; the HIP kernel skips negative rows."""
|
|
return (
|
|
torch.where(state_indices != NULL_BLOCK_ID, state_indices, -1)
|
|
.to(torch.int32)
|
|
.contiguous()
|
|
)
|
|
|
|
|
|
@cache
|
|
def is_fused_kda_chunk_supported() -> bool:
|
|
from vllm.platforms.rocm import on_gfx950
|
|
|
|
if not hasattr(torch.ops._C, "fused_kda_chunk"):
|
|
return False
|
|
# TODO: Verify on other archs, currently only validated on gfx950
|
|
return on_gfx950()
|
|
|
|
|
|
def can_use_fused_kda_chunk(
|
|
head_k_dim: int,
|
|
head_v_dim: int,
|
|
dtype: torch.dtype,
|
|
chunk_size: int,
|
|
) -> bool:
|
|
return (
|
|
head_k_dim == HEAD_DIM
|
|
and head_v_dim == HEAD_DIM
|
|
and chunk_size == CHUNK_SIZE
|
|
and dtype == torch.bfloat16
|
|
and is_fused_kda_chunk_supported()
|
|
)
|
|
|
|
|
|
def fused_kda_prologue(
|
|
q: torch.Tensor,
|
|
k: torch.Tensor,
|
|
v: torch.Tensor,
|
|
raw_g: torch.Tensor,
|
|
raw_beta: torch.Tensor,
|
|
A_log: torch.Tensor,
|
|
dt_bias: torch.Tensor,
|
|
scale: float,
|
|
lower_bound: float,
|
|
cu_seqlens: torch.Tensor,
|
|
conv_weight: torch.Tensor | None = None,
|
|
conv_state: torch.Tensor | None = None,
|
|
conv_state_indices: torch.Tensor | None = None,
|
|
conv_has_initial_state: torch.Tensor | None = None,
|
|
chunk_indices: torch.Tensor | None = None,
|
|
) -> dict[str, torch.Tensor]:
|
|
"""Run the whole chunk-path prologue in one launch.
|
|
|
|
Replaces the two L2 norms, the gate cumsum, both intra-chunk passes and the
|
|
w/u recompute. Returns the operands the fused chunk kernel consumes.
|
|
|
|
The kernel can also apply the depthwise conv and its silu, in which case
|
|
``q``/``k``/``v`` are the three raw bands of the QKV projection and
|
|
``conv_state`` is updated in place. This path is currently not used.
|
|
"""
|
|
# `beta` and `g` reach the layer as last-dim slices of the fused QKVGFAB
|
|
# projection and carry its row stride. The kernel reads both with an
|
|
# explicit per-token stride, so neither is copied here.
|
|
_, t_total, num_heads, _ = q.shape
|
|
if chunk_indices is None:
|
|
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
|
|
chunk_indices = chunk_indices.to(torch.int32)
|
|
num_chunks = chunk_indices.shape[0]
|
|
dev = q.device
|
|
|
|
# q may be a strided band view, so the workspaces are sized rather than
|
|
# cloned from it.
|
|
def _like(last: int) -> torch.Tensor:
|
|
return torch.empty(1, t_total, num_heads, last, dtype=q.dtype, device=q.device)
|
|
|
|
ws = dict(
|
|
qg=_like(HEAD_DIM),
|
|
w=_like(HEAD_DIM),
|
|
u=_like(HEAD_DIM),
|
|
kg_t=torch.empty(
|
|
num_chunks, num_heads, HEAD_DIM, CHUNK_SIZE, dtype=q.dtype, device=dev
|
|
),
|
|
aqk=torch.empty(1, t_total, num_heads, CHUNK_SIZE, dtype=q.dtype, device=dev),
|
|
decay=torch.empty(
|
|
num_chunks, num_heads, HEAD_DIM, dtype=torch.float32, device=dev
|
|
),
|
|
)
|
|
torch.ops._C.fused_kda_prologue(
|
|
q,
|
|
k,
|
|
v,
|
|
raw_g,
|
|
raw_beta,
|
|
A_log.reshape(-1),
|
|
dt_bias.reshape(-1),
|
|
ws["qg"],
|
|
ws["w"],
|
|
ws["u"],
|
|
ws["kg_t"],
|
|
ws["aqk"],
|
|
ws["decay"],
|
|
cu_seqlens.to(torch.int32),
|
|
chunk_indices,
|
|
conv_weight,
|
|
conv_state,
|
|
conv_state_indices,
|
|
conv_has_initial_state,
|
|
scale,
|
|
lower_bound,
|
|
)
|
|
return ws
|
|
|
|
|
|
@cache
|
|
def _num_cus(device: int) -> int:
|
|
from vllm.platforms import current_platform
|
|
|
|
return current_platform.get_device_properties(device).multi_processor_count
|
|
|
|
|
|
@cache
|
|
def _chunk_groups(chunks_per_seq: int, num_seqs: int, num_heads: int) -> int:
|
|
"""How many parallel chunk groups to cut each sequence into.
|
|
|
|
The scan composes the group operators with the exact transfer
|
|
``M_g = prod_c (diag(d_c) - w_c^T kg_c)``, so the composition error does
|
|
not grow with group length and the choice is purely a machine fit:
|
|
|
|
* ``fill1``/``fill2``: the largest G whose pass-one / pass-two grid still
|
|
fits in one scheduling batch. Rows are launched in blocks of at most
|
|
``kBV = 128``, so pass one (``kV + kK`` rows) needs two blocks per
|
|
``(n, h, g)`` and pass two one. Past a pass's fill point its block count
|
|
grows as fast as its depth shrinks and the rung stops paying.
|
|
* ``_DEEP_LEN``: unless the groups are still long, where pass two's halving
|
|
is worth letting pass one spill to a second batch.
|
|
* ``_MIN_LEN``: below this many chunks per group the per-group fixed cost
|
|
exceeds what the shorter walk saves.
|
|
|
|
``G == 2`` is depth-neutral -- two passes of ``nt/2`` -- and only doubles
|
|
staging, so the split is taken only from ``G >= 4``.
|
|
"""
|
|
cus = _num_cus(torch.accelerator.current_device_index())
|
|
nh = max(num_seqs * num_heads, 1)
|
|
fill1 = cus // (_BLOCKS_P1 * nh)
|
|
fill2 = cus // (_BLOCKS_P2 * nh)
|
|
cand = fill1
|
|
if fill2 > 0 and chunks_per_seq // fill2 >= _DEEP_LEN:
|
|
cand = fill2
|
|
cand = min(cand, chunks_per_seq // _MIN_LEN, _MAX_GROUPS)
|
|
return cand if cand >= 4 else 1
|
|
|
|
|
|
def _kda_group_workspace(
|
|
groups: int, nh: int, device: torch.device
|
|
) -> torch.Tensor | None:
|
|
"""One fp32 buffer the kernel carves into bg / sin_ / ag / mgT.
|
|
|
|
Kept caller-owned so the memory stays inside the caching allocator and
|
|
vLLM's memory profiling accounts for it; the kernel's carve order must
|
|
match the layout here.
|
|
"""
|
|
if groups <= 1:
|
|
return None
|
|
planes = groups * nh
|
|
plane = HEAD_DIM * HEAD_DIM
|
|
floats = 2 * planes * plane + planes * HEAD_DIM # bg, sin_, ag
|
|
floats += (planes * plane + 1) // 2 # mgT, bf16
|
|
return torch.empty(floats, dtype=torch.float32, device=device)
|
|
|
|
|
|
def fused_kda_chunk(
|
|
qg: torch.Tensor,
|
|
w: torch.Tensor,
|
|
u: torch.Tensor,
|
|
kg_t: torch.Tensor,
|
|
aqk: torch.Tensor,
|
|
decay: torch.Tensor,
|
|
out: torch.Tensor,
|
|
scale: float,
|
|
cu_seqlens: torch.Tensor,
|
|
initial_state: torch.Tensor | None,
|
|
output_final_state: bool,
|
|
chunk_offsets: torch.Tensor | None = None,
|
|
checkpoint_state: torch.Tensor | None = None,
|
|
checkpoint_offsets: torch.Tensor | None = None,
|
|
checkpoint_state_indices: torch.Tensor | None = None,
|
|
state_cache: torch.Tensor | None = None,
|
|
state_indices: torch.Tensor | None = None,
|
|
has_initial_state: torch.Tensor | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
|
"""Run the chunk recurrence and the output projection in one launch.
|
|
|
|
Args:
|
|
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
|
|
w: chunk-local WY representation of the gated keys.
|
|
u: chunk-local WY representation of the values.
|
|
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
|
|
aqk: intra-chunk query-key attention, ``[chunks, H, 64, 64]``.
|
|
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
|
|
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
|
|
scale: scale applied to the query-key products.
|
|
cu_seqlens: int32 cumulative sequence lengths.
|
|
initial_state: fp32 per-sequence initial recurrent state, or ``None``.
|
|
output_final_state: whether to return the final recurrent state.
|
|
chunk_offsets: int32 per-sequence first chunk index.
|
|
checkpoint_state: destination for the mid-prefill state snapshots,
|
|
fp32 ``[rows, H, 128, 128]``. Without
|
|
``checkpoint_state_indices`` it is a staging buffer indexed by
|
|
sequence; with them it can be the paged state cache itself.
|
|
checkpoint_offsets: int32 ``[N]`` token offsets, each relative to its
|
|
sequence's first query token, at which to snapshot the recurrent
|
|
state. ``0`` disables the export for that sequence. A non-zero
|
|
offset must be a multiple of :data:`KDA_CHECKPOINT_ALIGNMENT`;
|
|
the kernel skips any that is not, so the caller has to filter
|
|
rather than rely on the store happening.
|
|
checkpoint_state_indices: optional int32 ``[N]`` destination row per
|
|
sequence. A negative row disables the export for that sequence.
|
|
state_cache: the paged recurrent state, fp32
|
|
``[slots, H, 128, 128]``. When given, the walk reads sequence
|
|
``n``'s initial state from row ``state_indices[n]`` and writes its
|
|
final state back to the same row, which removes the gather and the
|
|
scatter that otherwise bracket this kernel. Replaces
|
|
``initial_state`` / ``output_final_state``, and the returned final
|
|
state is then ``None`` because it is already in the cache.
|
|
state_indices: int32 ``[N]`` cache row per sequence.
|
|
has_initial_state: bool ``[N]``; a false entry starts that sequence
|
|
from a zero state and its cache row is read only, never before the
|
|
walk writes it.
|
|
|
|
"""
|
|
if state_cache is not None:
|
|
if initial_state is not None or output_final_state:
|
|
raise ValueError("state_cache replaces initial_state/output_final_state")
|
|
if state_indices is None or has_initial_state is None:
|
|
raise ValueError("state_cache needs state_indices and has_initial_state")
|
|
num_seqs = cu_seqlens.numel() - 1
|
|
final_state = None
|
|
if output_final_state:
|
|
final_state = torch.empty(
|
|
num_seqs,
|
|
u.shape[2],
|
|
u.shape[3],
|
|
qg.shape[3],
|
|
dtype=torch.float32,
|
|
device=u.device,
|
|
)
|
|
cu_seqlens = cu_seqlens.to(torch.int32)
|
|
if chunk_offsets is None:
|
|
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
|
|
|
|
# The chunk walk is serial in chunks. Splitting each sequence into `groups`
|
|
# stretches that run in parallel costs a second pass and a scan, and only
|
|
# pays once the chain is long enough to amortise both.
|
|
chunks_per_seq = kg_t.shape[0] // max(num_seqs, 1)
|
|
groups = _chunk_groups(chunks_per_seq, num_seqs, u.shape[2])
|
|
group_state = _kda_group_workspace(groups, num_seqs * u.shape[2], u.device)
|
|
|
|
torch.ops._C.fused_kda_chunk(
|
|
qg,
|
|
w,
|
|
u,
|
|
kg_t,
|
|
aqk,
|
|
decay,
|
|
initial_state,
|
|
final_state,
|
|
out,
|
|
cu_seqlens,
|
|
chunk_offsets,
|
|
scale,
|
|
group_state,
|
|
groups,
|
|
checkpoint_state,
|
|
None if checkpoint_offsets is None else checkpoint_offsets.to(torch.int32),
|
|
None
|
|
if checkpoint_state_indices is None
|
|
else _hip_checkpoint_state_indices(checkpoint_state_indices),
|
|
state_cache,
|
|
None if state_indices is None else state_indices.to(torch.int32).contiguous(),
|
|
has_initial_state,
|
|
)
|
|
return out, final_state
|