diff --git a/tests/models/glm5next/test_kda_recurrent.py b/tests/models/glm5next/test_kda_recurrent.py index ddda0e0c34..fe017e4131 100644 --- a/tests/models/glm5next/test_kda_recurrent.py +++ b/tests/models/glm5next/test_kda_recurrent.py @@ -5,8 +5,7 @@ The decode path hands the kernel column slices of the merged ``q|k|v`` conv output and of the fused ``qkvbfg_a`` projection (beta), so q/k/v/beta are token-strided rather than contiguous. Both backends must match a pure-PyTorch -recurrence. CUDA reads these slices in place and rejects unsupported layouts; -the ROCm wrapper makes contiguous copies before launching its kernel. +recurrence, read these slices in place and reject unsupported layouts. """ import pytest @@ -166,8 +165,8 @@ def test_fused_recurrent_kda_strided_inputs_bit_identical_to_contiguous( @pytest.mark.skipif( - not current_platform.is_cuda(), - reason="Only the CUDA implementation requires zero-copy token-strided layouts", + not current_platform.is_cuda_alike(), + reason="Both implementations require zero-copy token-strided layouts", ) @torch.inference_mode() def test_fused_recurrent_kda_rejects_unaddressable_layouts(): diff --git a/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py b/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py index d8b25211e0..a9d0fba476 100644 --- a/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py +++ b/vllm/models/glm5next/amd/ops/third_party/kda/fused_recurrent.py @@ -15,6 +15,23 @@ from vllm.third_party.flash_linear_attention.ops.op import exp from vllm.triton_utils import tl, triton +def token_stride(x: torch.Tensor) -> int: + """Token stride (elements) of a ``[B, T, H, D]`` or ``[B, T, H]`` tensor. + + The recurrent kernel walks tokens with this stride and addresses heads + densely inside a token, so each token's ``[H, D]`` (or ``[H]``) block must + be contiguous, tokens must not overlap, and with ``B > 1`` sequence ``n`` + must start at token ``n * T`` (dense batch). Column slices of a wider + per-token projection buffer satisfy this and are consumed in place. + """ + st = x.stride() + assert x.dim() in (3, 4) and st[-1] == 1, (x.shape, st) + assert x.dim() == 3 or st[2] == x.shape[3], (x.shape, st) + assert st[1] >= x.shape[2] * (x.shape[3] if x.dim() == 4 else 1), (x.shape, st) + assert x.shape[0] == 1 or st[0] == x.shape[1] * st[1], (x.shape, st) + return st[1] + + @triton.heuristics( { "USE_INITIAL_STATE": lambda args: args["h0"] is not None, @@ -52,6 +69,11 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( stride_final_state_token: tl.constexpr, stride_indices_seq: tl.constexpr, stride_indices_tok: tl.constexpr, + # Token strides of q/k/v/beta (elements), see `token_stride`. + stride_q_t, + stride_k_t, + stride_v_t, + stride_beta_t, USE_INITIAL_STATE: tl.constexpr, # whether to use initial state INPLACE_FINAL_STATE: tl.constexpr, # whether to store final state inplace IS_BETA_HEADWISE: tl.constexpr, # whether beta is headwise vector or scalar, @@ -86,13 +108,13 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( o_k = i_k * BK + tl.arange(0, BK) o_v = i_v * BV + tl.arange(0, BV) - p_q = q + (bos * H + i_h) * K + o_k - p_k = k + (bos * H + i_h) * K + o_k - p_v = v + (bos * HV + i_hv) * V + o_v + p_q = q + bos * stride_q_t + i_h * K + o_k + p_k = k + bos * stride_k_t + i_h * K + o_k + p_v = v + bos * stride_v_t + i_hv * V + o_v if IS_BETA_HEADWISE: - p_beta = beta + (bos * HV + i_hv) * V + o_v + p_beta = beta + bos * stride_beta_t + i_hv * V + o_v else: - p_beta = beta + bos * HV + i_hv + p_beta = beta + bos * stride_beta_t + i_hv if not IS_KDA: p_g = g + bos * HV + i_hv @@ -188,15 +210,15 @@ def fused_recurrent_gated_delta_rule_fwd_kernel( p_ht = p_ht + i_hv * V * K + o_v[:, None] * K + o_k[None, :] tl.store(p_ht, b_h.to(p_ht.dtype.element_ty), mask=mask_h) - p_q += H * K - p_k += H * K + p_q += stride_q_t + p_k += stride_k_t p_o += HV * V - p_v += HV * V + p_v += stride_v_t if not IS_KDA: p_g += HV else: p_gk += HV * K - p_beta += HV * (V if IS_BETA_HEADWISE else 1) + p_beta += stride_beta_t def fused_recurrent_gated_delta_rule_fwd( @@ -265,6 +287,10 @@ def fused_recurrent_gated_delta_rule_fwd( stride_final_state_token=stride_final_state_token, stride_indices_seq=stride_indices_seq, stride_indices_tok=stride_indices_tok, + stride_q_t=token_stride(q), + stride_k_t=token_stride(k), + stride_v_t=token_stride(v), + stride_beta_t=token_stride(beta), IS_BETA_HEADWISE=beta.ndim == v.ndim, USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, INPLACE_FINAL_STATE=inplace_final_state, diff --git a/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py b/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py index ea86750e7d..9805b37471 100644 --- a/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py +++ b/vllm/models/glm5next/amd/ops/third_party/kda/kernels.py @@ -24,7 +24,10 @@ from vllm.third_party.flash_linear_attention.ops.utils import FLA_CHUNK_SIZE, is from vllm.triton_utils import tl, triton from vllm.utils.math_utils import RCP_LN2, cdiv, next_power_of_2 -from .fused_recurrent import fused_recurrent_gated_delta_rule_fwd_kernel +from .fused_recurrent import ( + fused_recurrent_gated_delta_rule_fwd_kernel, + token_stride, +) BT_LIST_AUTOTUNE = [32, 64, 128] NUM_WARPS_AUTOTUNE = [2, 4, 8, 16] if is_amd else [4, 8, 16, 32] @@ -70,7 +73,7 @@ def fused_recurrent_kda_fwd( g_bias = g_bias.reshape(-1).contiguous() if out is None: - o = torch.empty_like(k) + o = torch.empty(k.shape, dtype=k.dtype, device=k.device) else: # Caller-provided output buffer; must be layout-compatible with the # tensor the kernel indexes (contiguous, same shape/dtype as k). @@ -119,6 +122,10 @@ def fused_recurrent_kda_fwd( stride_final_state_token=stride_final_state_token, stride_indices_seq=stride_indices_seq, stride_indices_tok=stride_indices_tok, + stride_q_t=token_stride(q), + stride_k_t=token_stride(k), + stride_v_t=token_stride(v), + stride_beta_t=token_stride(beta), IS_BETA_HEADWISE=beta.ndim == v.ndim, USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, INPLACE_FINAL_STATE=inplace_final_state, @@ -165,12 +172,15 @@ def fused_recurrent_kda( if scale is None: scale = k.shape[-1] ** -0.5 + # q/k/v/beta are consumed in place with an explicit token stride, so + # column slices of the fused projection buffer need no copy; layouts the + # kernel cannot address fail loudly in `token_stride`. o, final_state = fused_recurrent_kda_fwd( - q=q.contiguous(), - k=k.contiguous(), - v=v.contiguous(), + q=q, + k=k, + v=v, g=g.contiguous(), - beta=beta.contiguous(), + beta=beta, scale=scale, initial_state=initial_state, inplace_final_state=inplace_final_state,