[ROCm][Perf][GLM-5.3-Flash] Stride-aware decode KDA (#57979)

Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
Simon Danielsson
2026-09-30 08:16:29 +00:00
committed by GitHub
parent b81984879a
commit 3627a6a124
3 changed files with 54 additions and 19 deletions
+3 -4
View File
@@ -5,8 +5,7 @@
The decode path hands the kernel column slices of the merged ``q|k|v`` conv
output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are
token-strided rather than contiguous. Both backends must match a pure-PyTorch
recurrence. CUDA reads these slices in place and rejects unsupported layouts;
the ROCm wrapper makes contiguous copies before launching its kernel.
recurrence, read these slices in place and reject unsupported layouts.
"""
import pytest
@@ -166,8 +165,8 @@ def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous(
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Only the CUDA implementation requires zero-copy token-strided layouts",
not current_platform.is_cuda_alike(),
reason="Both implementations require zero-copy token-strided layouts",
)
@torch.inference_mode()
def test_fused_recurrent_kda_rejects_unaddressable_layouts():
@@ -15,6 +15,23 @@ from vllm.third_party.flash_linear_attention.ops.op import exp
from vllm.triton_utils import tl, triton
def token_stride(x: torch.Tensor) -> int:
"""Token stride (elements) of a ``[B, T, H, D]`` or ``[B, T, H]`` tensor.
The recurrent kernel walks tokens with this stride and addresses heads
densely inside a token, so each token's ``[H, D]`` (or ``[H]``) block must
be contiguous, tokens must not overlap, and with ``B > 1`` sequence ``n``
must start at token ``n * T`` (dense batch). Column slices of a wider
per-token projection buffer satisfy this and are consumed in place.
"""
st = x.stride()
assert x.dim() in (3, 4) and st[-1] == 1, (x.shape, st)
assert x.dim() == 3 or st[2] == x.shape[3], (x.shape, st)
assert st[1] >= x.shape[2] * (x.shape[3] if x.dim() == 4 else 1), (x.shape, st)
assert x.shape[0] == 1 or st[0] == x.shape[1] * st[1], (x.shape, st)
return st[1]
@triton.heuristics(
{
"USE_INITIAL_STATE": lambda args: args["h0"] is not None,
@@ -52,6 +69,11 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
stride_final_state_token: tl.constexpr,
stride_indices_seq: tl.constexpr,
stride_indices_tok: tl.constexpr,
# Token strides of q/k/v/beta (elements), see `token_stride`.
stride_q_t,
stride_k_t,
stride_v_t,
stride_beta_t,
USE_INITIAL_STATE: tl.constexpr, # whether to use initial state
INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace
IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar,
@@ -86,13 +108,13 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
o_k = i_k * BK + tl.arange(0, BK)
o_v = i_v * BV + tl.arange(0, BV)
p_q = q + (bos * H + i_h) * K + o_k
p_k = k + (bos * H + i_h) * K + o_k
p_v = v + (bos * HV + i_hv) * V + o_v
p_q = q + bos * stride_q_t + i_h * K + o_k
p_k = k + bos * stride_k_t + i_h * K + o_k
p_v = v + bos * stride_v_t + i_hv * V + o_v
if IS_BETA_HEADWISE:
p_beta = beta + (bos * HV + i_hv) * V + o_v
p_beta = beta + bos * stride_beta_t + i_hv * V + o_v
else:
p_beta = beta + bos * HV + i_hv
p_beta = beta + bos * stride_beta_t + i_hv
if not IS_KDA:
p_g = g + bos * HV + i_hv
@@ -188,15 +210,15 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
p_q += H * K
p_k += H * K
p_q += stride_q_t
p_k += stride_k_t
p_o += HV * V
p_v += HV * V
p_v += stride_v_t
if not IS_KDA:
p_g += HV
else:
p_gk += HV * K
p_beta += HV * (V if IS_BETA_HEADWISE else 1)
p_beta += stride_beta_t
def fused_recurrent_gated_delta_rule_fwd(
@@ -265,6 +287,10 @@ def fused_recurrent_gated_delta_rule_fwd(
stride_final_state_token=stride_final_state_token,
stride_indices_seq=stride_indices_seq,
stride_indices_tok=stride_indices_tok,
stride_q_t=token_stride(q),
stride_k_t=token_stride(k),
stride_v_t=token_stride(v),
stride_beta_t=token_stride(beta),
IS_BETA_HEADWISE=beta.ndim == v.ndim,
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
INPLACE_FINAL_STATE=inplace_final_state,
+16 -6
View File
@@ -24,7 +24,10 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel
from .fused_recurrent import (
fused_recurrent_gated_delta_rule_fwd_kernel,
token_stride,
)
BT_LIST_AUTOTUNE = [32, 64, 128]
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
@@ -70,7 +73,7 @@ def fused_recurrent_kda_fwd(
g_bias = g_bias.reshape(-1).contiguous()
if out is None:
o = torch.empty_like(k)
o = torch.empty(k.shape, dtype=k.dtype, device=k.device)
else:
# Caller-provided output buffer; must be layout-compatible with the
# tensor the kernel indexes (contiguous, same shape/dtype as k).
@@ -119,6 +122,10 @@ def fused_recurrent_kda_fwd(
stride_final_state_token=stride_final_state_token,
stride_indices_seq=stride_indices_seq,
stride_indices_tok=stride_indices_tok,
stride_q_t=token_stride(q),
stride_k_t=token_stride(k),
stride_v_t=token_stride(v),
stride_beta_t=token_stride(beta),
IS_BETA_HEADWISE=beta.ndim == v.ndim,
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
INPLACE_FINAL_STATE=inplace_final_state,
@@ -165,12 +172,15 @@ def fused_recurrent_kda(
if scale is None:
scale = k.shape[-1] ** -0.5
# q/k/v/beta are consumed in place with an explicit token stride, so
# column slices of the fused projection buffer need no copy; layouts the
# kernel cannot address fail loudly in `token_stride`.
o, final_state = fused_recurrent_kda_fwd(
q=q.contiguous(),
k=k.contiguous(),
v=v.contiguous(),
q=q,
k=k,
v=v,
g=g.contiguous(),
beta=beta.contiguous(),
beta=beta,
scale=scale,
initial_state=initial_state,
inplace_final_state=inplace_final_state,