[Bugfix][Kernel] Keep packed GDN decode beta in FP32 (#53877)

Signed-off-by: CherryLemon <lemon_cn@alumni.sjtu.edu.cn>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Yongji Wu
2026-08-30 22:15:36 +08:00
committed by GitHub
co-authored by Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Jiangyun Zhu
parent 87b9b5b8d9
commit 56058fd572
2 changed files with 33 additions and 2 deletions
@@ -70,7 +70,7 @@ def test_fused_recurrent_packed_decode_matches_reference(
x <= 20.0, torch.log1p(torch.exp(torch.clamp(x, max=20.0))), x
)
g = (-torch.exp(A_log.float()) * softplus_x).unsqueeze(1)
beta = torch.sigmoid(b.float()).to(dtype).unsqueeze(1)
beta = torch.sigmoid(b.float()).unsqueeze(1)
out_ref, state_ref = fused_recurrent_gated_delta_rule(
q=q,
@@ -109,6 +109,37 @@ def test_fused_recurrent_packed_decode_matches_reference(
torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol)
def test_packed_decode_keeps_beta_in_fp32():
device = torch.device(DEVICE)
dtype = torch.bfloat16
mixed_qkv = torch.ones((1, 3), device=device, dtype=dtype)
a = torch.zeros((1, 1), device=device, dtype=dtype)
b = torch.full((1, 1), 0.5, device=device, dtype=dtype)
params = torch.zeros((1,), device=device, dtype=dtype)
ssm_state_indices = torch.ones((1,), device=device, dtype=torch.int32)
state_packed = torch.zeros((2, 1, 1, 1), device=device, dtype=torch.float32)
out_packed = torch.empty((1, 1, 1, 1), device=device, dtype=dtype)
fused_recurrent_gated_delta_rule_packed_decode(
mixed_qkv=mixed_qkv,
a=a,
b=b,
A_log=params,
dt_bias=params,
scale=1.0,
initial_state=state_packed,
out=out_packed,
ssm_state_indices=ssm_state_indices,
)
expected_beta = torch.sigmoid(b.float()).squeeze()
torch.testing.assert_close(
state_packed[1, 0, 0, 0], expected_beta, rtol=1e-6, atol=1e-6
)
def test_packed_decode_supports_large_batch_head_grid():
B, H, HV, K, V = 1024, 8, 64, 1, 1
device = torch.device(DEVICE)
@@ -326,7 +326,7 @@ def fused_recurrent_gated_delta_rule_packed_decode_kernel(
x = a_val + dt_bias_val
softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x)
g_val = -tl.exp(A_log_val) * softplus_x
beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32)
beta_val = tl.sigmoid(b_val)
b_h *= exp(g_val)
b_v -= tl.sum(b_h * b_k[None, :], 1)