From 56058fd572f6a7fec6899385f4a4ed7f4b964477 Mon Sep 17 00:00:00 2001 From: Yongji Wu Date: Sun, 30 Aug 2026 22:15:36 +0800 Subject: [PATCH] [Bugfix][Kernel] Keep packed GDN decode beta in FP32 (#53877) Signed-off-by: CherryLemon Co-authored-by: Codex Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jiangyun Zhu --- .../test_fused_recurrent_packed_decode.py | 33 ++++++++++++++++++- .../ops/fused_recurrent.py | 2 +- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/tests/kernels/test_fused_recurrent_packed_decode.py b/tests/kernels/test_fused_recurrent_packed_decode.py index 5e83537d94..be6597cc86 100644 --- a/tests/kernels/test_fused_recurrent_packed_decode.py +++ b/tests/kernels/test_fused_recurrent_packed_decode.py @@ -70,7 +70,7 @@ def test_fused_recurrent_packed_decode_matches_reference( x <= 20.0, torch.log1p(torch.exp(torch.clamp(x, max=20.0))), x ) g = (-torch.exp(A_log.float()) * softplus_x).unsqueeze(1) - beta = torch.sigmoid(b.float()).to(dtype).unsqueeze(1) + beta = torch.sigmoid(b.float()).unsqueeze(1) out_ref, state_ref = fused_recurrent_gated_delta_rule( q=q, @@ -109,6 +109,37 @@ def test_fused_recurrent_packed_decode_matches_reference( torch.testing.assert_close(state_packed, state_ref, rtol=rtol, atol=atol) +def test_packed_decode_keeps_beta_in_fp32(): + device = torch.device(DEVICE) + dtype = torch.bfloat16 + + mixed_qkv = torch.ones((1, 3), device=device, dtype=dtype) + a = torch.zeros((1, 1), device=device, dtype=dtype) + b = torch.full((1, 1), 0.5, device=device, dtype=dtype) + params = torch.zeros((1,), device=device, dtype=dtype) + ssm_state_indices = torch.ones((1,), device=device, dtype=torch.int32) + + state_packed = torch.zeros((2, 1, 1, 1), device=device, dtype=torch.float32) + out_packed = torch.empty((1, 1, 1, 1), device=device, dtype=dtype) + + fused_recurrent_gated_delta_rule_packed_decode( + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=params, + dt_bias=params, + scale=1.0, + initial_state=state_packed, + out=out_packed, + ssm_state_indices=ssm_state_indices, + ) + + expected_beta = torch.sigmoid(b.float()).squeeze() + torch.testing.assert_close( + state_packed[1, 0, 0, 0], expected_beta, rtol=1e-6, atol=1e-6 + ) + + def test_packed_decode_supports_large_batch_head_grid(): B, H, HV, K, V = 1024, 8, 64, 1, 1 device = torch.device(DEVICE) diff --git a/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py b/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py index e1d0d965f0..eb08b938c2 100644 --- a/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py +++ b/vllm/third_party/flash_linear_attention/ops/fused_recurrent.py @@ -326,7 +326,7 @@ def fused_recurrent_gated_delta_rule_packed_decode_kernel( x = a_val + dt_bias_val softplus_x = tl.where(x <= SOFTPLUS_THRESHOLD, tl.log(1.0 + tl.exp(x)), x) g_val = -tl.exp(A_log_val) * softplus_x - beta_val = tl.sigmoid(b_val).to(b.dtype.element_ty).to(tl.float32) + beta_val = tl.sigmoid(b_val) b_h *= exp(g_val) b_v -= tl.sum(b_h * b_k[None, :], 1)