[Bugfix] Fix packed GDN decode launch for large batch-head grids (#52030)

Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Michael Goin
2026-08-13 07:19:08 -07:00
committed by GitHub
parent 170592a931
commit 2d24355eb8
2 changed files with 33 additions and 3 deletions
@@ -278,9 +278,13 @@ def fused_recurrent_gated_delta_rule_packed_decode_kernel(
BV: tl.constexpr,
SOFTPLUS_THRESHOLD: tl.constexpr,
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
SPLIT_BATCH_HEAD_GRID: tl.constexpr,
):
i_v, i_nh = tl.program_id(0), tl.program_id(1)
i_n, i_hv = i_nh // HV, i_nh % HV
if SPLIT_BATCH_HEAD_GRID:
i_v, i_hv, i_n = tl.program_id(0), tl.program_id(1), tl.program_id(2)
else:
i_v, i_nh = tl.program_id(0), tl.program_id(1)
i_n, i_hv = i_nh // HV, i_nh % HV
i_h = i_hv // (HV // H)
o_k = tl.arange(0, BK)
@@ -446,7 +450,9 @@ def fused_recurrent_gated_delta_rule_packed_decode(
stride_indices_seq = ssm_state_indices.stride(0)
NV = triton.cdiv(V, BV)
grid = (NV, B * HV)
# CUDA limits grid Y/Z dimensions to 65535.
split_batch_head_grid = B * HV > 65535
grid = (NV, HV, B) if split_batch_head_grid else (NV, B * HV)
fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
mixed_qkv=mixed_qkv,
a=a,
@@ -472,6 +478,7 @@ def fused_recurrent_gated_delta_rule_packed_decode(
BV=BV,
SOFTPLUS_THRESHOLD=20.0,
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
SPLIT_BATCH_HEAD_GRID=split_batch_head_grid,
num_warps=num_warps,
num_stages=num_stages,
)