[ROCm][Perf][Kimi-K3] Use AITER Gluon chunk_kda for KDA prefill (#59552)
Signed-off-by: Omar Muhammad <omar.muhammad@amd.com> Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Nicholas Susanto
Douglas Lehr
Kevin H. Luu
parent
b088eba944
commit
28e92eeff2
@@ -95,7 +95,9 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
assert isinstance(spec, MambaSpec)
|
||||
# Only the fused path exports the checkpoint snapshots
|
||||
can_checkpoint = (
|
||||
self.use_prefill_checkpoint and self.use_fused_chunk and self.use_safe_gate
|
||||
self.use_prefill_checkpoint
|
||||
and self.kda_prefill_backend == "fused"
|
||||
and self.use_safe_gate
|
||||
)
|
||||
if can_checkpoint and vllm_config.cache_config.mamba_cache_mode == "align":
|
||||
logger.info_once("Kimi-K3 KDA prefill checkpoint enabled.")
|
||||
@@ -242,13 +244,15 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
"The fused KDA chunk kernel requires gfx950 and a build that "
|
||||
"includes it."
|
||||
)
|
||||
self.use_fused_chunk = backend == "fused" or (
|
||||
backend == "auto" and is_fused_kda_chunk_supported()
|
||||
)
|
||||
logger.info_once(
|
||||
"Kimi-K3 KDA prefill backend: %s",
|
||||
"fused" if self.use_fused_chunk else "triton",
|
||||
)
|
||||
self.kda_prefill_backend = backend
|
||||
if backend == "auto":
|
||||
if self.use_safe_gate and rocm_aiter_ops.is_kda_gluon_prefill_available():
|
||||
self.kda_prefill_backend = "gluon"
|
||||
elif is_fused_kda_chunk_supported():
|
||||
self.kda_prefill_backend = "fused"
|
||||
else:
|
||||
self.kda_prefill_backend = "triton"
|
||||
logger.info_once("Kimi-K3 KDA prefill backend: %s", self.kda_prefill_backend)
|
||||
# TODO: Remove the extra config arg once tested
|
||||
self.use_prefill_checkpoint = (
|
||||
additional_config.get("kda_prefill_checkpoint", True)
|
||||
@@ -694,7 +698,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
|
||||
cu_seqlens=prefill_query_start_loc,
|
||||
chunk_indices=m.chunk_indices,
|
||||
chunk_offsets=m.chunk_offsets,
|
||||
use_fused_chunk=self.use_fused_chunk,
|
||||
backend=self.kda_prefill_backend,
|
||||
out=core_attn_out[:, nd_tok:num_actual_tokens] if direct else None,
|
||||
checkpoint_state=checkpoint_state,
|
||||
checkpoint_offsets=checkpoint_offsets,
|
||||
|
||||
@@ -9,6 +9,7 @@ path.
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
|
||||
gather_initial_states,
|
||||
@@ -24,6 +25,11 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Consumed by kimi_k3_triton_warmup.py during kernel_warmup().
|
||||
def get_gluon_chunk_kda_warmup_profiles() -> tuple[tuple[int, int], ...]:
|
||||
return ((1, 64), (2, 64), (4, 64), (8, 64), (16, 64), (1, 4096), (2, 2048))
|
||||
|
||||
|
||||
def chunk_kda_prefill(
|
||||
q: torch.Tensor,
|
||||
k: torch.Tensor,
|
||||
@@ -40,7 +46,7 @@ def chunk_kda_prefill(
|
||||
cu_seqlens: torch.Tensor | None = None,
|
||||
chunk_indices: torch.Tensor | None = None,
|
||||
chunk_offsets: torch.Tensor | None = None,
|
||||
use_fused_chunk: bool = False,
|
||||
backend: str = "triton",
|
||||
out: torch.Tensor | None = None,
|
||||
checkpoint_state: torch.Tensor | None = None,
|
||||
checkpoint_offsets: torch.Tensor | None = None,
|
||||
@@ -68,9 +74,10 @@ def chunk_kda_prefill(
|
||||
cu_seqlens: int32 cumulative sequence lengths.
|
||||
chunk_indices: precomputed chunk indices for ``cu_seqlens``.
|
||||
chunk_offsets: int32 per-sequence first chunk index.
|
||||
use_fused_chunk: request the two-kernel ROCm path. It is used only when
|
||||
every one of its preconditions holds; otherwise the Triton path
|
||||
runs unchanged.
|
||||
backend: ``"triton"`` (the vendored chunk path), ``"fused"`` (the
|
||||
two-kernel ROCm path; used only when every one of its
|
||||
preconditions holds, otherwise the Triton path runs unchanged) or
|
||||
``"gluon"`` (AITER's ``chunk_kda``).
|
||||
out: buffer the result must land in. Honoured by both backends, so the
|
||||
caller can hand in a slice of its own output and skip a copy.
|
||||
checkpoint_state: destination for mid-prefill recurrent state
|
||||
@@ -89,6 +96,11 @@ def chunk_kda_prefill(
|
||||
The output and, when requested, the final recurrent state.
|
||||
|
||||
"""
|
||||
if backend not in ("triton", "fused", "gluon"):
|
||||
raise ValueError(
|
||||
"chunk_kda_prefill backend must be 'triton', 'fused' or 'gluon', "
|
||||
f"got {backend!r}."
|
||||
)
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
@@ -96,7 +108,7 @@ def chunk_kda_prefill(
|
||||
# needs the raw projections and a bounded gate rather than the pre-normalized
|
||||
# tensors the Triton path takes.
|
||||
fused = (
|
||||
use_fused_chunk
|
||||
backend == "fused"
|
||||
and use_qk_l2norm_in_kernel
|
||||
and cu_seqlens is not None
|
||||
and lower_bound is not None
|
||||
@@ -116,6 +128,28 @@ def chunk_kda_prefill(
|
||||
if state_indices is None or has_initial_state is None:
|
||||
raise ValueError("state_cache needs state_indices and has_initial_state")
|
||||
|
||||
if backend == "gluon":
|
||||
return rocm_aiter_ops.kda_chunk_prefill(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=raw_g,
|
||||
beta=raw_beta,
|
||||
A_log=A_log,
|
||||
dt_bias=g_bias,
|
||||
lower_bound=lower_bound,
|
||||
cu_seqlens=cu_seqlens,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
scale=scale,
|
||||
out=out,
|
||||
initial_state=initial_state,
|
||||
output_final_state=output_final_state,
|
||||
state_cache=state_cache,
|
||||
state_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
)
|
||||
|
||||
# Only the fused walk addresses the paged rows directly. The Triton path
|
||||
# gathers the rows it needs and scatters the results back here, so callers
|
||||
# pass the cache the same way for either backend.
|
||||
|
||||
Reference in New Issue
Block a user