[Qwen3.8-Flash-Next] Improve QSA sparse GQA for prefill and short-ctx decode (#54873)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
Thien Tran
2026-09-04 06:08:02 -07:00
committed by GitHub
co-authored by Kimi
parent 8340fe1bb9
commit 31a8a26662
6 changed files with 365 additions and 106 deletions
+65 -26
View File
@@ -754,7 +754,10 @@ def test_qsa_block_expansion_correctness() -> None:
sequence_lengths.index_select(0, token_to_req.long()) // 4,
).to(torch.int32)
actual = torch.empty((2, 11), device="cuda", dtype=torch.int32)
# Packed layout: one trailing column per row holds the valid-entry count
# (never a token index). Row 0: 1 visible block + 2 tail; row 1: 2 blocks
# + 3 tail.
actual = torch.empty((2, 12), device="cuda", dtype=torch.int32)
qsa_indexer_ops.expand_qsa_block_indices(
blocks,
query_positions,
@@ -771,19 +774,35 @@ def test_qsa_block_expansion_correctness() -> None:
token_topk=8,
)
torch.testing.assert_close(actual, expected)
torch.testing.assert_close(actual[:, :11], expected)
assert actual[:, 11].tolist() == [6, 11]
@requires_qsa_kernels
@pytest.mark.parametrize(
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
(
"num_rows",
"num_query_heads",
"num_kv_heads",
"page_size",
"use_prefill_config",
"num_requests",
),
[
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
# Production page sizes from hybrid-cache block alignment: 784/800
# at TP4 and 1568/1600 at TP1/TP2 (no-MTP / MTP num_spec=3). Head
# splits are per-rank TP1/TP2/TP4; the largest batch runs both
# use_prefill_config variants.
pytest.param(1, 24, 2, 1600, True, 2, id="tp1_r1"),
pytest.param(16, 12, 1, 1600, True, 3, id="tp2_r16"),
pytest.param(32, 6, 1, 800, True, 5, id="tp4_r32"),
pytest.param(128, 24, 2, 1568, True, 7, id="tp1_r128"),
pytest.param(257, 6, 1, 800, True, 13, id="tp4_r257"),
pytest.param(513, 6, 1, 784, True, 17, id="tp4_r513"),
pytest.param(700, 6, 1, 800, True, 23, id="tp4_r700"),
pytest.param(1024, 24, 2, 1600, True, 33, id="tp1_r1024"),
pytest.param(2048, 24, 2, 1600, True, 63, id="tp1_r2048_prefill"),
pytest.param(2048, 24, 2, 1600, False, 63, id="tp1_r2048_uniform"),
],
)
def test_qsa_sparse_paged_attention_correctness(
@@ -791,10 +810,11 @@ def test_qsa_sparse_paged_attention_correctness(
num_query_heads: int,
num_kv_heads: int,
page_size: int,
use_prefill_config: bool,
num_requests: int,
) -> None:
torch.manual_seed(2)
head_dim = 256
num_requests = 2
num_selected_pages = 64
# Keep the newest page outside the synthetic top-k as causal headroom.
num_pages_per_request = num_selected_pages + 1
@@ -822,13 +842,22 @@ def test_qsa_sparse_paged_attention_correctness(
rows_per_request = math.ceil(num_rows / num_requests)
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
token_to_req = row_indices // rows_per_request
request_row_counts = torch.tensor(
[rows_per_request, num_rows - rows_per_request],
# Uniform row split; the last request takes the remainder (possibly 0).
request_row_counts = torch.full(
(num_requests,), rows_per_request, device="cuda", dtype=torch.int32
)
request_row_counts[-1] = num_rows - rows_per_request * (num_requests - 1)
# Mix context lengths: every third request is short-context, attending
# to only its first few pages; the rest fill their cache.
context_lengths = torch.full(
(num_requests,),
num_pages_per_request * page_size - 1,
device="cuda",
dtype=torch.int32,
)
context_length = num_pages_per_request * page_size - 1
short_requests = torch.arange(num_requests, device="cuda") % 3 == 1
context_lengths[short_requests] = request_row_counts[short_requests] + 8
block_topk = indexer_budget // indexer_compress_ratio
compressed_blocks_per_page = page_size // indexer_compress_ratio
selection = torch.arange(block_topk, device="cuda")
@@ -842,17 +871,19 @@ def test_qsa_sparse_paged_attention_correctness(
)
rows_within_request = row_indices % rows_per_request
query_positions = (
context_length - request_row_counts[token_to_req.long()] + rows_within_request
context_lengths[token_to_req.long()]
- request_row_counts[token_to_req.long()]
+ rows_within_request
).to(torch.int64)
sequence_lengths = torch.full(
(num_requests,), context_length, device="cuda", dtype=torch.int32
)
sequence_lengths = context_lengths
visible_blocks = torch.minimum(
(query_positions + 1) // indexer_compress_ratio,
sequence_lengths.index_select(0, token_to_req.long()) // indexer_compress_ratio,
).to(torch.int32)
# +1: the packed trailing column holds each row's valid-entry count
# (never a token index); the reference reads only the selection region.
logical_indices = torch.empty(
(num_rows, selection_width), device="cuda", dtype=torch.int32
(num_rows, selection_width + 1), device="cuda", dtype=torch.int32
)
qsa_indexer_ops.expand_qsa_block_indices(
block_indices,
@@ -862,7 +893,7 @@ def test_qsa_sparse_paged_attention_correctness(
indexer_budget,
logical_indices,
)
assert logical_indices.shape == (num_rows, selection_width)
assert logical_indices.shape == (num_rows, selection_width + 1)
scale = q.shape[-1] ** -0.5
actual = qsa_ops.qsa_sparse_paged_attention(
@@ -872,12 +903,13 @@ def test_qsa_sparse_paged_attention_correctness(
logical_indices,
block_table,
token_to_req,
use_prefill_config=use_prefill_config,
)
expected = _qsa_sparse_paged_attention_reference(
q,
k_cache,
v_cache,
logical_indices,
logical_indices[:, :selection_width],
block_table,
token_to_req,
scale,
@@ -948,8 +980,10 @@ def test_qsa_split_selection_correctness(workspace_init, decode_query_len: int)
block_indices[prefill_slice],
max_seq_len=sequence_lengths.max().item(),
)
# +1: the packed trailing count column (never a token index; excluded
# from the comparison).
actual = torch.empty(
(rows, token_topk + compress_ratio - 1), device="cuda", dtype=torch.int32
(rows, token_topk + compress_ratio), device="cuda", dtype=torch.int32
)
qsa_indexer_ops.expand_qsa_block_indices(
block_indices,
@@ -977,7 +1011,10 @@ def test_qsa_split_selection_correctness(workspace_init, decode_query_len: int)
token_topk,
)
torch.testing.assert_close(actual.sort().values, expected.sort().values)
torch.testing.assert_close(
actual[:, : token_topk + compress_ratio - 1].sort().values,
expected.sort().values,
)
@requires_qsa_kernels
@@ -1001,7 +1038,7 @@ def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) ->
block_indices=block_indices,
max_seq_len=64, # clamps to the page-table capacity
)
selected = torch.empty((2, 2051), device="cuda", dtype=torch.int32)
selected = torch.empty((2, 2052), device="cuda", dtype=torch.int32)
qsa_indexer_ops.expand_qsa_block_indices(
block_indices,
query_positions,
@@ -1013,8 +1050,10 @@ def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) ->
assert selected[0, :2].tolist() == [0, 1]
assert selected[1, :3].tolist() == [0, 1, 2]
assert torch.all(selected[0, 2:] == -1)
assert torch.all(selected[1, 3:] == -1)
assert torch.all(selected[0, 2:2051] == -1)
assert torch.all(selected[1, 3:2051] == -1)
# The packed trailing column holds each row's valid-entry count.
assert selected[:, 2051].tolist() == [2, 3]
@requires_qsa_kernels