Revert "[ROCm][Perf] Kimi-K3 Fused kernels for KDA prefill" (#53294)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
0a3a4f7f35
commit
592e06f2ae
@@ -41,16 +41,13 @@ from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.models.kimi_k3.amd.kda_metadata import KimiK3ROCmKDABackend
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
is_fused_kda_chunk_supported,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_decode import (
|
||||
is_fused_kda_decode_supported,
|
||||
make_decode_conv1d_weight_loader,
|
||||
make_decode_norm_weight_loader,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.kda_prefill import chunk_kda_prefill
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import (
|
||||
chunk_kda_with_fused_gate,
|
||||
fused_recurrent_kda,
|
||||
fused_recurrent_kda_packed_decode,
|
||||
)
|
||||
@@ -213,21 +210,10 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
if isinstance(additional_config, dict)
|
||||
else "auto"
|
||||
)
|
||||
assert backend in ("auto", "triton", "fused"), (
|
||||
"The ROCm Kimi-K3 KDA prefill backend must be one of "
|
||||
f"'auto', 'triton' or 'fused', got {backend!r}."
|
||||
)
|
||||
if backend == "fused" and not is_fused_kda_chunk_supported():
|
||||
raise RuntimeError(
|
||||
"The fused KDA chunk kernel requires gfx950 and a build that "
|
||||
"includes it."
|
||||
)
|
||||
self.use_fused_chunk = backend == "fused" or (
|
||||
backend == "auto" and is_fused_kda_chunk_supported()
|
||||
)
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill backend: %s",
|
||||
"fused" if self.use_fused_chunk else "triton",
|
||||
backend = "triton" if backend == "auto" else backend
|
||||
assert backend == "triton", (
|
||||
"The ROCm Kimi-K3 KDA layer only supports the Triton KDA prefill "
|
||||
f"backend, got {backend!r}."
|
||||
)
|
||||
|
||||
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
|
||||
@@ -510,12 +496,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
# prefill tail only.
|
||||
core_attn_out_decode = None
|
||||
split_non_spec = spec_sequence_masks is None and m.num_decodes > 0
|
||||
# Without a spec split the non-spec tokens are exactly
|
||||
# core_attn_out[:, :num_actual_tokens] in order, so both kernels
|
||||
# can write their slice straight into it. With one, the tokens
|
||||
# are interleaved and have to be scattered by index afterwards.
|
||||
direct = spec_sequence_masks is None
|
||||
use_fused_chunk = self.use_fused_chunk and spec_sequence_masks is None
|
||||
if split_non_spec:
|
||||
assert non_spec_query_start_loc is not None
|
||||
nd_tok = m.num_decode_tokens
|
||||
@@ -533,7 +513,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
ssm_state_indices=non_spec_state_indices_tensor[
|
||||
: m.num_decodes
|
||||
],
|
||||
out=core_attn_out[:, :nd_tok] if direct else None,
|
||||
)
|
||||
q_ns = q_ns[:, nd_tok:]
|
||||
k_ns = k_ns[:, nd_tok:]
|
||||
@@ -550,7 +529,6 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
prefill_query_start_loc = non_spec_query_start_loc
|
||||
prefill_state_indices = non_spec_state_indices_tensor
|
||||
prefill_has_initial_state = has_initial_state
|
||||
nd_tok = 0
|
||||
|
||||
initial_state = gather_initial_states(
|
||||
recurrent_state,
|
||||
@@ -560,7 +538,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
(
|
||||
core_attn_out_non_spec,
|
||||
last_recurrent_state,
|
||||
) = chunk_kda_prefill(
|
||||
) = chunk_kda_with_fused_gate(
|
||||
q=q_ns,
|
||||
k=k_ns,
|
||||
v=v_ns,
|
||||
@@ -575,16 +553,11 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
chunk_indices=m.chunk_indices,
|
||||
chunk_offsets=m.chunk_offsets,
|
||||
use_fused_chunk=use_fused_chunk,
|
||||
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
|
||||
)
|
||||
# Init cache
|
||||
recurrent_state[prefill_state_indices] = last_recurrent_state
|
||||
|
||||
if direct:
|
||||
# Both slices already landed in core_attn_out.
|
||||
core_attn_out_non_spec = core_attn_out[:, :num_actual_tokens]
|
||||
elif split_non_spec:
|
||||
if split_non_spec:
|
||||
# Restore decode-first token order for the merge below.
|
||||
core_attn_out_non_spec = torch.cat(
|
||||
[core_attn_out_decode, core_attn_out_non_spec], dim=1
|
||||
@@ -636,10 +609,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
merged.index_copy_(1, non_spec_token_indx, core_attn_out_non_spec)
|
||||
core_attn_out[0, :num_actual_tokens] = merged[0, :num_actual_tokens]
|
||||
elif core_attn_out_non_spec is not None:
|
||||
if core_attn_out_non_spec.data_ptr() != core_attn_out.data_ptr():
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[
|
||||
0, :num_actual_tokens
|
||||
]
|
||||
else:
|
||||
assert core_attn_out_spec is not None
|
||||
core_attn_out.copy_(self.o_norm(core_attn_out, g2))
|
||||
|
||||
@@ -1,251 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm entry point for the fused Kimi-K3 KDA chunk kernel.
|
||||
|
||||
The kernel in ``csrc/libtorch_stable/kimi_k3/fused_kda_chunk_kernel_rocm.cu``
|
||||
replaces the chunk-state recurrence and the output GEMM of the Triton chunk
|
||||
path with a single launch that keeps the per-chunk state in registers, so the
|
||||
``[chunks, H, V, K]`` state tensor and the recomputed values never reach HBM.
|
||||
"""
|
||||
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.third_party.flash_linear_attention.ops.index import (
|
||||
prepare_chunk_indices,
|
||||
prepare_chunk_offsets,
|
||||
)
|
||||
|
||||
CHUNK_SIZE = 64
|
||||
HEAD_DIM = 128
|
||||
# Chunk-group split policy constants; see _chunk_groups.
|
||||
_BLOCKS_P1 = 2 # (kV + kK) / kBV
|
||||
_BLOCKS_P2 = 1 # kV / kBV
|
||||
_MIN_LEN = 4
|
||||
_DEEP_LEN = 12
|
||||
_MAX_GROUPS = 32
|
||||
|
||||
|
||||
@cache
|
||||
def is_fused_kda_chunk_supported() -> bool:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not hasattr(torch.ops._C, "fused_kda_chunk"):
|
||||
return False
|
||||
# TODO: Verify on other archs, currently only validated on gfx950
|
||||
return on_gfx950()
|
||||
|
||||
|
||||
def can_use_fused_kda_chunk(
|
||||
head_k_dim: int,
|
||||
head_v_dim: int,
|
||||
dtype: torch.dtype,
|
||||
chunk_size: int,
|
||||
) -> bool:
|
||||
return (
|
||||
head_k_dim == HEAD_DIM
|
||||
and head_v_dim == HEAD_DIM
|
||||
and chunk_size == CHUNK_SIZE
|
||||
and dtype == torch.bfloat16
|
||||
and is_fused_kda_chunk_supported()
|
||||
)
|
||||
|
||||
|
||||
def fused_kda_prologue(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
dt_bias: torch.Tensor,
|
||||
scale: float,
|
||||
lower_bound: float,
|
||||
cu_seqlens: torch.Tensor,
|
||||
conv_weight: torch.Tensor | None = None,
|
||||
conv_state: torch.Tensor | None = None,
|
||||
conv_state_indices: torch.Tensor | None = None,
|
||||
conv_has_initial_state: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Run the whole chunk-path prologue in one launch.
|
||||
|
||||
Replaces the two L2 norms, the gate cumsum, both intra-chunk passes and the
|
||||
w/u recompute. Returns the operands the fused chunk kernel consumes.
|
||||
|
||||
The kernel can also apply the depthwise conv and its silu, in which case
|
||||
``q``/``k``/``v`` are the three raw bands of the QKV projection and
|
||||
``conv_state`` is updated in place. This path is currently not used.
|
||||
"""
|
||||
# `beta` and `g` reach the layer as last-dim slices of the fused QKVGFAB
|
||||
# projection and carry its row stride. The kernel reads both with an
|
||||
# explicit per-token stride, so neither is copied here.
|
||||
_, t_total, num_heads, _ = q.shape
|
||||
if chunk_indices is None:
|
||||
chunk_indices = prepare_chunk_indices(cu_seqlens, CHUNK_SIZE)
|
||||
chunk_indices = chunk_indices.to(torch.int32)
|
||||
num_chunks = chunk_indices.shape[0]
|
||||
dev = q.device
|
||||
|
||||
# q may be a strided band view, so the workspaces are sized rather than
|
||||
# cloned from it.
|
||||
def _like(last: int) -> torch.Tensor:
|
||||
return torch.empty(1, t_total, num_heads, last, dtype=q.dtype, device=q.device)
|
||||
|
||||
ws = dict(
|
||||
qg=_like(HEAD_DIM),
|
||||
w=_like(HEAD_DIM),
|
||||
u=_like(HEAD_DIM),
|
||||
kg_t=torch.empty(
|
||||
num_chunks, num_heads, HEAD_DIM, CHUNK_SIZE, dtype=q.dtype, device=dev
|
||||
),
|
||||
aqk=torch.empty(1, t_total, num_heads, CHUNK_SIZE, dtype=q.dtype, device=dev),
|
||||
decay=torch.empty(
|
||||
num_chunks, num_heads, HEAD_DIM, dtype=torch.float32, device=dev
|
||||
),
|
||||
)
|
||||
torch.ops._C.fused_kda_prologue(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
raw_g,
|
||||
raw_beta,
|
||||
A_log.reshape(-1),
|
||||
dt_bias.reshape(-1),
|
||||
ws["qg"],
|
||||
ws["w"],
|
||||
ws["u"],
|
||||
ws["kg_t"],
|
||||
ws["aqk"],
|
||||
ws["decay"],
|
||||
cu_seqlens.to(torch.int32),
|
||||
chunk_indices,
|
||||
conv_weight,
|
||||
conv_state,
|
||||
conv_state_indices,
|
||||
conv_has_initial_state,
|
||||
scale,
|
||||
lower_bound,
|
||||
)
|
||||
return ws
|
||||
|
||||
|
||||
@cache
|
||||
def _num_cus(device: int) -> int:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
return current_platform.get_device_properties(device).multi_processor_count
|
||||
|
||||
|
||||
@cache
|
||||
def _chunk_groups(chunks_per_seq: int, num_seqs: int, num_heads: int) -> int:
|
||||
"""How many parallel chunk groups to cut each sequence into.
|
||||
|
||||
The scan composes the group operators with the exact transfer
|
||||
``M_g = prod_c (diag(d_c) - w_c^T kg_c)``, so the composition error does
|
||||
not grow with group length and the choice is purely a machine fit:
|
||||
|
||||
* ``fill1``/``fill2``: the largest G whose pass-one / pass-two grid still
|
||||
fits in one scheduling batch. Rows are launched in blocks of at most
|
||||
``kBV = 128``, so pass one (``kV + kK`` rows) needs two blocks per
|
||||
``(n, h, g)`` and pass two one. Past a pass's fill point its block count
|
||||
grows as fast as its depth shrinks and the rung stops paying.
|
||||
* ``_DEEP_LEN``: unless the groups are still long, where pass two's halving
|
||||
is worth letting pass one spill to a second batch.
|
||||
* ``_MIN_LEN``: below this many chunks per group the per-group fixed cost
|
||||
exceeds what the shorter walk saves.
|
||||
|
||||
``G == 2`` is depth-neutral -- two passes of ``nt/2`` -- and only doubles
|
||||
staging, so the split is taken only from ``G >= 4``.
|
||||
"""
|
||||
cus = _num_cus(torch.accelerator.current_device_index())
|
||||
nh = max(num_seqs * num_heads, 1)
|
||||
fill1 = cus // (_BLOCKS_P1 * nh)
|
||||
fill2 = cus // (_BLOCKS_P2 * nh)
|
||||
cand = fill1
|
||||
if fill2 > 0 and chunks_per_seq // fill2 >= _DEEP_LEN:
|
||||
cand = fill2
|
||||
cand = min(cand, chunks_per_seq // _MIN_LEN, _MAX_GROUPS)
|
||||
return cand if cand >= 4 else 1
|
||||
|
||||
|
||||
def _kda_group_workspace(
|
||||
groups: int, nh: int, device: torch.device
|
||||
) -> torch.Tensor | None:
|
||||
"""One fp32 buffer the kernel carves into bg / sin_ / ag / mgT.
|
||||
|
||||
Kept caller-owned so the memory stays inside the caching allocator and
|
||||
vLLM's memory profiling accounts for it; the kernel's carve order must
|
||||
match the layout here.
|
||||
"""
|
||||
if groups <= 1:
|
||||
return None
|
||||
planes = groups * nh
|
||||
plane = HEAD_DIM * HEAD_DIM
|
||||
floats = 2 * planes * plane + planes * HEAD_DIM # bg, sin_, ag
|
||||
floats += (planes * plane + 1) // 2 # mgT, bf16
|
||||
return torch.empty(floats, dtype=torch.float32, device=device)
|
||||
|
||||
|
||||
def fused_kda_chunk(
|
||||
qg: torch.Tensor,
|
||||
w: torch.Tensor,
|
||||
u: torch.Tensor,
|
||||
kg_t: torch.Tensor,
|
||||
aqk: torch.Tensor,
|
||||
decay: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
scale: float,
|
||||
cu_seqlens: torch.Tensor,
|
||||
initial_state: torch.Tensor | None,
|
||||
output_final_state: bool,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Run the chunk recurrence and the output projection in one launch.
|
||||
|
||||
Args:
|
||||
qg: ``q * exp2(gk_cumsum)``, ``[1, T, H, 128]``.
|
||||
kg_t: chunk-major transposed gated keys, ``[chunks, H, 128, 64]``.
|
||||
decay: ``exp2`` of each chunk's last gate row, ``[chunks, H, 128]``.
|
||||
out: output buffer, ``[1, T, H, 128]``; may alias ``u``'s source.
|
||||
"""
|
||||
num_seqs = cu_seqlens.numel() - 1
|
||||
final_state = None
|
||||
if output_final_state:
|
||||
final_state = torch.empty(
|
||||
num_seqs,
|
||||
u.shape[2],
|
||||
u.shape[3],
|
||||
qg.shape[3],
|
||||
dtype=torch.float32,
|
||||
device=u.device,
|
||||
)
|
||||
cu_seqlens = cu_seqlens.to(torch.int32)
|
||||
if chunk_offsets is None:
|
||||
chunk_offsets = prepare_chunk_offsets(cu_seqlens, CHUNK_SIZE)
|
||||
|
||||
# The chunk walk is serial in chunks. Splitting each sequence into `groups`
|
||||
# stretches that run in parallel costs a second pass and a scan, and only
|
||||
# pays once the chain is long enough to amortise both.
|
||||
chunks_per_seq = kg_t.shape[0] // max(num_seqs, 1)
|
||||
groups = _chunk_groups(chunks_per_seq, num_seqs, u.shape[2])
|
||||
group_state = _kda_group_workspace(groups, num_seqs * u.shape[2], u.device)
|
||||
|
||||
torch.ops._C.fused_kda_chunk(
|
||||
qg,
|
||||
w,
|
||||
u,
|
||||
kg_t,
|
||||
aqk,
|
||||
decay,
|
||||
initial_state,
|
||||
final_state,
|
||||
out,
|
||||
cu_seqlens,
|
||||
chunk_offsets,
|
||||
scale,
|
||||
group_state,
|
||||
groups,
|
||||
)
|
||||
return out, final_state
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""KDA prefill backend selection for ROCm.
|
||||
|
||||
The Kimi-K3 KDA layer calls :func:`chunk_kda_prefill`, which either runs the
|
||||
fused HIP kernels in ``kda_chunk`` or falls back to the vendored Triton chunk
|
||||
path.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.models.kimi_k3.amd.ops.kda_chunk import (
|
||||
can_use_fused_kda_chunk,
|
||||
fused_kda_chunk,
|
||||
fused_kda_prologue,
|
||||
)
|
||||
from vllm.models.kimi_k3.amd.ops.third_party.kda import chunk_kda_with_fused_gate
|
||||
from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def chunk_kda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
v: torch.Tensor,
|
||||
raw_g: torch.Tensor,
|
||||
raw_beta: torch.Tensor,
|
||||
A_log: torch.Tensor,
|
||||
g_bias: torch.Tensor | None = None,
|
||||
scale: float | None = None,
|
||||
initial_state: torch.Tensor | None = None,
|
||||
output_final_state: bool = False,
|
||||
lower_bound: float | None = None,
|
||||
use_qk_l2norm_in_kernel: bool = False,
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_fused_chunk: bool = False,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Run chunk KDA from raw gate and beta projections.
|
||||
|
||||
Args:
|
||||
use_fused_chunk: request the two-kernel ROCm path. It is used only when
|
||||
every one of its preconditions holds; otherwise the Triton path
|
||||
runs unchanged.
|
||||
out: buffer the result must land in. Honoured by both backends, so the
|
||||
caller can hand in a slice of its own output and skip a copy.
|
||||
|
||||
Returns:
|
||||
The output and, when requested, the final recurrent state.
|
||||
"""
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
# The fused prologue folds the q/k L2 norm and the gate activation in, so it
|
||||
# needs the raw projections and a bounded gate rather than the pre-normalized
|
||||
# tensors the Triton path takes.
|
||||
fused = (
|
||||
use_fused_chunk
|
||||
and use_qk_l2norm_in_kernel
|
||||
and cu_seqlens is not None
|
||||
and lower_bound is not None
|
||||
and g_bias is not None
|
||||
and can_use_fused_kda_chunk(k.shape[-1], v.shape[-1], k.dtype, FLA_CHUNK_SIZE)
|
||||
)
|
||||
|
||||
if fused:
|
||||
# Restated for the type checker; `fused` already implies all three.
|
||||
assert cu_seqlens is not None and lower_bound is not None
|
||||
assert g_bias is not None
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill: dispatching the fused ROCm chunk kernel."
|
||||
)
|
||||
ws = fused_kda_prologue(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=g_bias,
|
||||
scale=scale,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
)
|
||||
return fused_kda_chunk(
|
||||
qg=ws["qg"],
|
||||
w=ws["w"],
|
||||
u=ws["u"],
|
||||
kg_t=ws["kg_t"],
|
||||
aqk=ws["aqk"],
|
||||
decay=ws["decay"],
|
||||
# v is dead by this point, so it doubles as the output buffer.
|
||||
out=out if out is not None else v,
|
||||
scale=scale,
|
||||
cu_seqlens=cu_seqlens,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
|
||||
o, final_state = chunk_kda_with_fused_gate(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
raw_g=raw_g,
|
||||
raw_beta=raw_beta,
|
||||
A_log=A_log,
|
||||
g_bias=g_bias,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
lower_bound=lower_bound,
|
||||
use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
)
|
||||
if out is not None and o.data_ptr() != out.data_ptr():
|
||||
out.copy_(o)
|
||||
o = out
|
||||
return o, final_state
|
||||
Reference in New Issue
Block a user