[ROCm][Perf][Kimi-K3] Use AITER Gluon KDA kernels for decode and spec decode (#57392)

Signed-off-by: omuhamma <omar.muhammad@amd.com>
Signed-off-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Nicholas Susanto <nicholas.susanto@amd.com>
Co-authored-by: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com>
This commit is contained in:
Omar Muhammad
2026-10-15 17:54:24 -05:00
committed by GitHub
co-authored by Nicholas Susanto Simon Danielsson
parent de7d70676d
commit 3f1e57225a
3 changed files with 503 additions and 17 deletions
+68 -4
View File
@@ -8,6 +8,7 @@ from einops import rearrange
from torch import nn
from vllm import _custom_ops as ops
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.config import VllmConfig
from vllm.distributed import divide
@@ -182,16 +183,19 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
# into one kernel, which wants a width-major fp32 conv weight staged at
# load time. Everything else keeps the [channel, width] layout.
conv_state_dtype, _ = self.get_state_dtype()
decode_conv1d_weight = None
if is_fused_kda_decode_supported(
self.use_aiter_decode = bool(rocm_aiter_ops.is_kda_gluon_decode_available())
use_hip_decode = not self.use_aiter_decode and is_fused_kda_decode_supported(
self.local_num_heads,
self.head_dim,
self.conv_size,
self.num_spec,
vllm_config.model_config.dtype,
conv_state_dtype,
):
)
if use_hip_decode:
logger.info_once("Fused KDA decode kernel (conv+KDA+norm) is enabled.")
decode_conv1d_weight = None
if use_hip_decode or self.use_aiter_decode:
decode_conv1d_weight = torch.empty(
3,
self.conv_size,
@@ -257,7 +261,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
self.o_norm = FusedRMSNormGated(self.head_dim, activation="sigmoid")
decode_norm_weight = None
if decode_conv1d_weight is not None:
if use_hip_decode:
# Upcast once at load time; a BF16 norm weight slows the fused
# decode kernel's epilogue.
decode_norm_weight = torch.empty(
@@ -388,6 +392,66 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if not is_conv_state_dim_first():
conv_state = conv_state.transpose(-1, -2)
if (
self.use_aiter_decode
and spec_sequence_masks is None
and m.num_prefills == 0
and m.num_decodes > 0
):
assert non_spec_state_indices_tensor is not None
rocm_aiter_ops.kda_fused_recurrent_packed_decode(
mixed_qkv=mixed_qkv,
g=g1[0],
beta=beta[0],
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
ssm_state_indices=non_spec_state_indices_tensor[:num_actual_tokens],
conv_state=conv_state,
conv_weight=self.decode_conv1d_weight,
out_gate=g2[:num_actual_tokens],
norm_weight=self.o_norm.weight,
norm_eps=self.o_norm.eps,
out=core_attn_out[0, :num_actual_tokens],
)
return
if (
self.use_aiter_decode
and spec_sequence_masks is not None
and m.num_prefills == 0
):
# Spec-only batch (gdn_attn never mixes spec and plain decodes).
assert spec_state_indices_tensor is not None
assert spec_query_start_loc is not None
q, k, v = (
rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim)
for x in mixed_qkv.split(self.local_projection_size, dim=-1)
)
n_spec = q.shape[1]
rocm_aiter_ops.kda_fused_recurrent_spec_decode(
q=q,
k=k,
v=v,
g=g1,
beta=beta,
A_log=self.A_log,
dt_bias=self.dt_bias,
lower_bound=self.gate_lower_bound,
initial_state=recurrent_state,
cu_seqlens=spec_query_start_loc[: m.num_spec_decodes + 1],
ssm_state_indices=spec_state_indices_tensor,
num_accepted_tokens=num_accepted_tokens,
conv_state=conv_state,
conv_weight=self.decode_conv1d_weight,
out_gate=g2[:n_spec].unsqueeze(0),
norm_weight=self.o_norm.weight,
norm_eps=self.o_norm.eps,
out=core_attn_out[:, :n_spec],
)
return
if (
self.decode_conv1d_weight is not None
and self.decode_norm_weight is not None