From 2d24355eb87b716fc1169e66731dc0386ed1a3a2 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 13 Aug 2026 10:19:08 -0400 Subject: [PATCH] [Bugfix] Fix packed GDN decode launch for large batch-head grids (#52030) Signed-off-by: mgoin --- .../test_fused_recurrent_packed_decode.py | 23 +++++++++++++++++++ .../ops/fused_recurrent.py | 13 ++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/kernels/test_fused_recurrent_packed_decode.py b/tests/kernels/test_fused_recurrent_packed_decode.py index 928c3e5bd7..33a66034b4 100644 --- a/tests/kernels/test_fused_recurrent_packed_decode.py +++ b/tests/kernels/test_fused_recurrent_packed_decode.py @@ -100,3 +100,26 @@ def test_fused_recurrent_packed_decode_matches_reference( valid = ssm_state_indices > 0 torch.testing.assert_close(out_packed[valid], out_ref[valid], rtol=rtol, atol=atol) torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device") +def test_packed_decode_supports_large_batch_head_grid(): + B, H, HV, K, V = 1024, 8, 64, 1, 1 + device = torch.device("cuda") + gates = torch.empty((B, HV), device=device) + params = torch.empty((HV,), device=device) + out = torch.empty((B, 1, HV, V), device=device) + + fused_recurrent_gated_delta_rule_packed_decode( + mixed_qkv=torch.empty((B, 2 * H * K + HV * V), device=device), + a=gates, + b=gates, + A_log=params, + dt_bias=params, + scale=1.0, + initial_state=torch.empty((1, HV, V, K), device=device), + out=out, + ssm_state_indices=torch.zeros((B,), device=device, dtype=torch.int32), + ) + + assert torch.count_nonzero(out).item() == 0 diff --git a/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py b/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py index c8004cb571..e1d0d965f0 100644 --- a/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py +++ b/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py @@ -278,9 +278,13 @@ def fused_recurrent_gated_delta_rule_packed_decode_kernel( BV: tl.constexpr, SOFTPLUS_THRESHOLD: tl.constexpr, USE_QK_L2NORM_IN_KERNEL: tl.constexpr, + SPLIT_BATCH_HEAD_GRID: tl.constexpr, ): - i_v, i_nh = tl.program_id(0), tl.program_id(1) - i_n, i_hv = i_nh // HV, i_nh % HV + if SPLIT_BATCH_HEAD_GRID: + i_v, i_hv, i_n = tl.program_id(0), tl.program_id(1), tl.program_id(2) + else: + i_v, i_nh = tl.program_id(0), tl.program_id(1) + i_n, i_hv = i_nh // HV, i_nh % HV i_h = i_hv // (HV // H) o_k = tl.arange(0, BK) @@ -446,7 +450,9 @@ def fused_recurrent_gated_delta_rule_packed_decode( stride_indices_seq = ssm_state_indices.stride(0) NV = triton.cdiv(V, BV) - grid = (NV, B * HV) + # CUDA limits grid Y/Z dimensions to 65535. + split_batch_head_grid = B * HV > 65535 + grid = (NV, HV, B) if split_batch_head_grid else (NV, B * HV) fused_recurrent_gated_delta_rule_packed_decode_kernel[grid]( mixed_qkv=mixed_qkv, a=a, @@ -472,6 +478,7 @@ def fused_recurrent_gated_delta_rule_packed_decode( BV=BV, SOFTPLUS_THRESHOLD=20.0, USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel, + SPLIT_BATCH_HEAD_GRID=split_batch_head_grid, num_warps=num_warps, num_stages=num_stages, )