[Bugfix] Fix packed GDN decode launch for large batch-head grids (#52030)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -100,3 +100,26 @@ def test_fused_recurrent_packed_decode_matches_reference(
|
||||
valid = ssm_state_indices > 0
|
||||
torch.testing.assert_close(out_packed[valid], out_ref[valid], rtol=rtol, atol=atol)
|
||||
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Need CUDA device")
|
||||
def test_packed_decode_supports_large_batch_head_grid():
|
||||
B, H, HV, K, V = 1024, 8, 64, 1, 1
|
||||
device = torch.device("cuda")
|
||||
gates = torch.empty((B, HV), device=device)
|
||||
params = torch.empty((HV,), device=device)
|
||||
out = torch.empty((B, 1, HV, V), device=device)
|
||||
|
||||
fused_recurrent_gated_delta_rule_packed_decode(
|
||||
mixed_qkv=torch.empty((B, 2 * H * K + HV * V), device=device),
|
||||
a=gates,
|
||||
b=gates,
|
||||
A_log=params,
|
||||
dt_bias=params,
|
||||
scale=1.0,
|
||||
initial_state=torch.empty((1, HV, V, K), device=device),
|
||||
out=out,
|
||||
ssm_state_indices=torch.zeros((B,), device=device, dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert torch.count_nonzero(out).item() == 0
|
||||
|
||||
@@ -278,9 +278,13 @@ def fused_recurrent_gated_delta_rule_packed_decode_kernel(
|
||||
BV: tl.constexpr,
|
||||
SOFTPLUS_THRESHOLD: tl.constexpr,
|
||||
USE_QK_L2NORM_IN_KERNEL: tl.constexpr,
|
||||
SPLIT_BATCH_HEAD_GRID: tl.constexpr,
|
||||
):
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_hv = i_nh // HV, i_nh % HV
|
||||
if SPLIT_BATCH_HEAD_GRID:
|
||||
i_v, i_hv, i_n = tl.program_id(0), tl.program_id(1), tl.program_id(2)
|
||||
else:
|
||||
i_v, i_nh = tl.program_id(0), tl.program_id(1)
|
||||
i_n, i_hv = i_nh // HV, i_nh % HV
|
||||
i_h = i_hv // (HV // H)
|
||||
|
||||
o_k = tl.arange(0, BK)
|
||||
@@ -446,7 +450,9 @@ def fused_recurrent_gated_delta_rule_packed_decode(
|
||||
stride_indices_seq = ssm_state_indices.stride(0)
|
||||
|
||||
NV = triton.cdiv(V, BV)
|
||||
grid = (NV, B * HV)
|
||||
# CUDA limits grid Y/Z dimensions to 65535.
|
||||
split_batch_head_grid = B * HV > 65535
|
||||
grid = (NV, HV, B) if split_batch_head_grid else (NV, B * HV)
|
||||
fused_recurrent_gated_delta_rule_packed_decode_kernel[grid](
|
||||
mixed_qkv=mixed_qkv,
|
||||
a=a,
|
||||
@@ -472,6 +478,7 @@ def fused_recurrent_gated_delta_rule_packed_decode(
|
||||
BV=BV,
|
||||
SOFTPLUS_THRESHOLD=20.0,
|
||||
USE_QK_L2NORM_IN_KERNEL=use_qk_l2norm_in_kernel,
|
||||
SPLIT_BATCH_HEAD_GRID=split_batch_head_grid,
|
||||
num_warps=num_warps,
|
||||
num_stages=num_stages,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user