[ROCm][Perf][GLM-5.3-Flash] Stride-aware decode KDA (#57979)
Signed-off-by: simondanielsson <simon.danielsson99@hotmail.com>
This commit is contained in:
@@ -5,8 +5,7 @@
|
||||
The decode path hands the kernel column slices of the merged ``q|k|v`` conv
|
||||
output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are
|
||||
token-strided rather than contiguous. Both backends must match a pure-PyTorch
|
||||
recurrence. CUDA reads these slices in place and rejects unsupported layouts;
|
||||
the ROCm wrapper makes contiguous copies before launching its kernel.
|
||||
recurrence, read these slices in place and reject unsupported layouts.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
@@ -166,8 +165,8 @@ def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Only the CUDA implementation requires zero-copy token-strided layouts",
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Both implementations require zero-copy token-strided layouts",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_recurrent_kda_rejects_unaddressable_layouts():
|
||||
|
||||
@@ -15,6 +15,23 @@ from vllm.third_party.flash_linear_attention.ops.op import exp
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
def token_stride(x: torch.Tensor) -> int:
|
||||
"""Token stride (elements) of a ``[B, T, H, D]`` or ``[B, T, H]`` tensor.
|
||||
|
||||
The recurrent kernel walks tokens with this stride and addresses heads
|
||||
densely inside a token, so each token's ``[H, D]`` (or ``[H]``) block must
|
||||
be contiguous, tokens must not overlap, and with ``B > 1`` sequence ``n``
|
||||
must start at token ``n * T`` (dense batch). Column slices of a wider
|
||||
per-token projection buffer satisfy this and are consumed in place.
|
||||
"""
|
||||
st = x.stride()
|
||||
assert x.dim() in (3, 4) and st[-1] == 1, (x.shape, st)
|
||||
assert x.dim() == 3 or st[2] == x.shape[3], (x.shape, st)
|
||||
assert st[1] >= x.shape[2] * (x.shape[3] if x.dim() == 4 else 1), (x.shape, st)
|
||||
assert x.shape[0] == 1 or st[0] == x.shape[1] * st[1], (x.shape, st)
|
||||
return st[1]
|
||||
|
||||
|
||||
@triton.heuristics(
|
||||
{
|
||||
"USE_INITIAL_STATE": lambda args: args["h0"] is not None,
|
||||
@@ -52,6 +69,11 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
stride_final_state_token: tl.constexpr,
|
||||
stride_indices_seq: tl.constexpr,
|
||||
stride_indices_tok: tl.constexpr,
|
||||
# Token strides of q/k/v/beta (elements), see `token_stride`.
|
||||
stride_q_t,
|
||||
stride_k_t,
|
||||
stride_v_t,
|
||||
stride_beta_t,
|
||||
USE_INITIAL_STATE: tl.constexpr, # whether to use initial state
|
||||
INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace
|
||||
IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar,
|
||||
@@ -86,13 +108,13 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
o_k = i_k * BK + tl.arange(0, BK)
|
||||
o_v = i_v * BV + tl.arange(0, BV)
|
||||
|
||||
p_q = q + (bos * H + i_h) * K + o_k
|
||||
p_k = k + (bos * H + i_h) * K + o_k
|
||||
p_v = v + (bos * HV + i_hv) * V + o_v
|
||||
p_q = q + bos * stride_q_t + i_h * K + o_k
|
||||
p_k = k + bos * stride_k_t + i_h * K + o_k
|
||||
p_v = v + bos * stride_v_t + i_hv * V + o_v
|
||||
if IS_BETA_HEADWISE:
|
||||
p_beta = beta + (bos * HV + i_hv) * V + o_v
|
||||
p_beta = beta + bos * stride_beta_t + i_hv * V + o_v
|
||||
else:
|
||||
p_beta = beta + bos * HV + i_hv
|
||||
p_beta = beta + bos * stride_beta_t + i_hv
|
||||
|
||||
if not IS_KDA:
|
||||
p_g = g + bos * HV + i_hv
|
||||
@@ -188,15 +210,15 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
|
||||
p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :]
|
||||
tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h)
|
||||
|
||||
p_q += H * K
|
||||
p_k += H * K
|
||||
p_q += stride_q_t
|
||||
p_k += stride_k_t
|
||||
p_o += HV * V
|
||||
p_v += HV * V
|
||||
p_v += stride_v_t
|
||||
if not IS_KDA:
|
||||
p_g += HV
|
||||
else:
|
||||
p_gk += HV * K
|
||||
p_beta += HV * (V if IS_BETA_HEADWISE else 1)
|
||||
p_beta += stride_beta_t
|
||||
|
||||
|
||||
def fused_recurrent_gated_delta_rule_fwd(
|
||||
@@ -265,6 +287,10 @@ def fused_recurrent_gated_delta_rule_fwd(
|
||||
stride_final_state_token=stride_final_state_token,
|
||||
stride_indices_seq=stride_indices_seq,
|
||||
stride_indices_tok=stride_indices_tok,
|
||||
stride_q_t=token_stride(q),
|
||||
stride_k_t=token_stride(k),
|
||||
stride_v_t=token_stride(v),
|
||||
stride_beta_t=token_stride(beta),
|
||||
IS_BETA_HEADWISE=beta.ndim == v.ndim,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
INPLACE_FINAL_STATE=inplace_final_state,
|
||||
|
||||
+16
-6
@@ -24,7 +24,10 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2
|
||||
|
||||
from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel
|
||||
from .fused_recurrent import (
|
||||
fused_recurrent_gated_delta_rule_fwd_kernel,
|
||||
token_stride,
|
||||
)
|
||||
|
||||
BT_LIST_AUTOTUNE = [32, 64, 128]
|
||||
NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32]
|
||||
@@ -70,7 +73,7 @@ def fused_recurrent_kda_fwd(
|
||||
g_bias = g_bias.reshape(-1).contiguous()
|
||||
|
||||
if out is None:
|
||||
o = torch.empty_like(k)
|
||||
o = torch.empty(k.shape, dtype=k.dtype, device=k.device)
|
||||
else:
|
||||
# Caller-provided output buffer; must be layout-compatible with the
|
||||
# tensor the kernel indexes (contiguous, same shape/dtype as k).
|
||||
@@ -119,6 +122,10 @@ def fused_recurrent_kda_fwd(
|
||||
stride_final_state_token=stride_final_state_token,
|
||||
stride_indices_seq=stride_indices_seq,
|
||||
stride_indices_tok=stride_indices_tok,
|
||||
stride_q_t=token_stride(q),
|
||||
stride_k_t=token_stride(k),
|
||||
stride_v_t=token_stride(v),
|
||||
stride_beta_t=token_stride(beta),
|
||||
IS_BETA_HEADWISE=beta.ndim == v.ndim,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
INPLACE_FINAL_STATE=inplace_final_state,
|
||||
@@ -165,12 +172,15 @@ def fused_recurrent_kda(
|
||||
if scale is None:
|
||||
scale = k.shape[-1] ** -0.5
|
||||
|
||||
# q/k/v/beta are consumed in place with an explicit token stride, so
|
||||
# column slices of the fused projection buffer need no copy; layouts the
|
||||
# kernel cannot address fail loudly in `token_stride`.
|
||||
o, final_state = fused_recurrent_kda_fwd(
|
||||
q=q.contiguous(),
|
||||
k=k.contiguous(),
|
||||
v=v.contiguous(),
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g.contiguous(),
|
||||
beta=beta.contiguous(),
|
||||
beta=beta,
|
||||
scale=scale,
|
||||
initial_state=initial_state,
|
||||
inplace_final_state=inplace_final_state,
|
||||
|
||||
Reference in New Issue
Block a user