[Qwen3.8-Flash-Next] Fuse PLE residual and QSA output gate (#55309)
Signed-off-by: Canlin <canlinguosdu@gmail.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: Kimi Code <noreply@moonshot.cn>
This commit is contained in:
co-authored by
zjy0516
Kimi Code
parent
c612e2bff0
commit
3f55ad2f07
@@ -944,6 +944,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
q = torch.randn(
|
||||
num_rows, num_query_heads, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
output_gate = torch.randn_like(q)
|
||||
kv_cache = torch.randn(
|
||||
num_cache_blocks,
|
||||
page_size,
|
||||
@@ -1023,6 +1024,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
block_table,
|
||||
token_to_req,
|
||||
use_prefill_config=use_prefill_config,
|
||||
output_gate=output_gate,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
@@ -1033,6 +1035,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
token_to_req,
|
||||
scale,
|
||||
)
|
||||
expected = expected * torch.sigmoid(output_gate)
|
||||
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user