[Bugfix][GLM-5.3-Flash] Keep batch x heads out of gridDim.z in the GLM-5.3-Flash fused recurrent KDA kernel (#56974)

Signed-off-by: NolenLiang <nliang@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
bcsdhjew
2026-10-05 09:31:45 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Codex
parent b1f229fb75
commit ae53b06898
3 changed files with 78 additions and 3 deletions
@@ -87,7 +87,7 @@ def fused_recurrent_gated_delta_rule_fwd_kernel(
SAFE_GATE: tl.constexpr, # bounded gate variant (only branch implemented)
LOWER_BOUND: tl.constexpr,
):
i_k, i_v, i_nh = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_nh, i_v, i_k = tl.program_id(0), tl.program_id(1), tl.program_id(2)
i_n, i_hv = i_nh // HV, i_nh % HV
i_h = i_hv // (HV // H)
if IS_VARLEN:
@@ -260,7 +260,9 @@ def fused_recurrent_gated_delta_rule_fwd(
else:
stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()
grid = (NK, NV, N * HV)
# N * HV goes in gridDim.x: gridDim.z is capped at 65535 and batch x heads exceeds it
# (e.g. GLM-5.3-Flash at TP=1: 1024 x 64 = 65536 -> "invalid argument" at CUDA-graph capture).
grid = (N * HV, NV, NK)
fused_recurrent_gated_delta_rule_fwd_kernel[grid](
q=q,
k=k,
+3 -1
View File
@@ -95,7 +95,9 @@ def fused_recurrent_kda_fwd(
else:
stride_indices_seq, stride_indices_tok = ssm_state_indices.stride()
grid = (NK, NV, N * HV)
# N * HV goes in gridDim.x: gridDim.z is capped at 65535 and batch x heads exceeds it
# (e.g. GLM-5.3-Flash at TP=1: 1024 x 64 = 65536 -> "invalid argument" at CUDA-graph capture).
grid = (N * HV, NV, NK)
fused_recurrent_gated_delta_rule_fwd_kernel[grid](
q=q,
k=k,