[Qwen3.8-Flash-Next] Improve QSA sparse GQA for prefill and short-ctx decode (#54873)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
@@ -754,7 +754,10 @@ def test_qsa_block_expansion_correctness() -> None:
|
||||
sequence_lengths.index_select(0, token_to_req.long()) // 4,
|
||||
).to(torch.int32)
|
||||
|
||||
actual = torch.empty((2, 11), device="cuda", dtype=torch.int32)
|
||||
# Packed layout: one trailing column per row holds the valid-entry count
|
||||
# (never a token index). Row 0: 1 visible block + 2 tail; row 1: 2 blocks
|
||||
# + 3 tail.
|
||||
actual = torch.empty((2, 12), device="cuda", dtype=torch.int32)
|
||||
qsa_indexer_ops.expand_qsa_block_indices(
|
||||
blocks,
|
||||
query_positions,
|
||||
@@ -771,19 +774,35 @@ def test_qsa_block_expansion_correctness() -> None:
|
||||
token_topk=8,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected)
|
||||
torch.testing.assert_close(actual[:, :11], expected)
|
||||
assert actual[:, 11].tolist() == [6, 11]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_query_heads", "num_kv_heads", "page_size"),
|
||||
(
|
||||
"num_rows",
|
||||
"num_query_heads",
|
||||
"num_kv_heads",
|
||||
"page_size",
|
||||
"use_prefill_config",
|
||||
"num_requests",
|
||||
),
|
||||
[
|
||||
# Kernel-visible pages with --block-size 256 and hybrid-cache alignment.
|
||||
pytest.param(1, 24, 2, 1792, id="tp1_split64"),
|
||||
pytest.param(16, 12, 1, 1792, id="tp2_split32"),
|
||||
pytest.param(32, 6, 1, 1024, id="tp4_split8"),
|
||||
pytest.param(257, 6, 1, 1024, id="tp4_split4"),
|
||||
pytest.param(513, 6, 1, 1024, id="tp4_split1"),
|
||||
# Production page sizes from hybrid-cache block alignment: 784/800
|
||||
# at TP4 and 1568/1600 at TP1/TP2 (no-MTP / MTP num_spec=3). Head
|
||||
# splits are per-rank TP1/TP2/TP4; the largest batch runs both
|
||||
# use_prefill_config variants.
|
||||
pytest.param(1, 24, 2, 1600, True, 2, id="tp1_r1"),
|
||||
pytest.param(16, 12, 1, 1600, True, 3, id="tp2_r16"),
|
||||
pytest.param(32, 6, 1, 800, True, 5, id="tp4_r32"),
|
||||
pytest.param(128, 24, 2, 1568, True, 7, id="tp1_r128"),
|
||||
pytest.param(257, 6, 1, 800, True, 13, id="tp4_r257"),
|
||||
pytest.param(513, 6, 1, 784, True, 17, id="tp4_r513"),
|
||||
pytest.param(700, 6, 1, 800, True, 23, id="tp4_r700"),
|
||||
pytest.param(1024, 24, 2, 1600, True, 33, id="tp1_r1024"),
|
||||
pytest.param(2048, 24, 2, 1600, True, 63, id="tp1_r2048_prefill"),
|
||||
pytest.param(2048, 24, 2, 1600, False, 63, id="tp1_r2048_uniform"),
|
||||
],
|
||||
)
|
||||
def test_qsa_sparse_paged_attention_correctness(
|
||||
@@ -791,10 +810,11 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
num_query_heads: int,
|
||||
num_kv_heads: int,
|
||||
page_size: int,
|
||||
use_prefill_config: bool,
|
||||
num_requests: int,
|
||||
) -> None:
|
||||
torch.manual_seed(2)
|
||||
head_dim = 256
|
||||
num_requests = 2
|
||||
num_selected_pages = 64
|
||||
# Keep the newest page outside the synthetic top-k as causal headroom.
|
||||
num_pages_per_request = num_selected_pages + 1
|
||||
@@ -822,13 +842,22 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
rows_per_request = math.ceil(num_rows / num_requests)
|
||||
row_indices = torch.arange(num_rows, device="cuda", dtype=torch.int32)
|
||||
token_to_req = row_indices // rows_per_request
|
||||
request_row_counts = torch.tensor(
|
||||
[rows_per_request, num_rows - rows_per_request],
|
||||
# Uniform row split; the last request takes the remainder (possibly 0).
|
||||
request_row_counts = torch.full(
|
||||
(num_requests,), rows_per_request, device="cuda", dtype=torch.int32
|
||||
)
|
||||
request_row_counts[-1] = num_rows - rows_per_request * (num_requests - 1)
|
||||
|
||||
# Mix context lengths: every third request is short-context, attending
|
||||
# to only its first few pages; the rest fill their cache.
|
||||
context_lengths = torch.full(
|
||||
(num_requests,),
|
||||
num_pages_per_request * page_size - 1,
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
context_length = num_pages_per_request * page_size - 1
|
||||
short_requests = torch.arange(num_requests, device="cuda") % 3 == 1
|
||||
context_lengths[short_requests] = request_row_counts[short_requests] + 8
|
||||
block_topk = indexer_budget // indexer_compress_ratio
|
||||
compressed_blocks_per_page = page_size // indexer_compress_ratio
|
||||
selection = torch.arange(block_topk, device="cuda")
|
||||
@@ -842,17 +871,19 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
)
|
||||
rows_within_request = row_indices % rows_per_request
|
||||
query_positions = (
|
||||
context_length - request_row_counts[token_to_req.long()] + rows_within_request
|
||||
context_lengths[token_to_req.long()]
|
||||
- request_row_counts[token_to_req.long()]
|
||||
+ rows_within_request
|
||||
).to(torch.int64)
|
||||
sequence_lengths = torch.full(
|
||||
(num_requests,), context_length, device="cuda", dtype=torch.int32
|
||||
)
|
||||
sequence_lengths = context_lengths
|
||||
visible_blocks = torch.minimum(
|
||||
(query_positions + 1) // indexer_compress_ratio,
|
||||
sequence_lengths.index_select(0, token_to_req.long()) // indexer_compress_ratio,
|
||||
).to(torch.int32)
|
||||
# +1: the packed trailing column holds each row's valid-entry count
|
||||
# (never a token index); the reference reads only the selection region.
|
||||
logical_indices = torch.empty(
|
||||
(num_rows, selection_width), device="cuda", dtype=torch.int32
|
||||
(num_rows, selection_width + 1), device="cuda", dtype=torch.int32
|
||||
)
|
||||
qsa_indexer_ops.expand_qsa_block_indices(
|
||||
block_indices,
|
||||
@@ -862,7 +893,7 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
indexer_budget,
|
||||
logical_indices,
|
||||
)
|
||||
assert logical_indices.shape == (num_rows, selection_width)
|
||||
assert logical_indices.shape == (num_rows, selection_width + 1)
|
||||
scale = q.shape[-1] ** -0.5
|
||||
|
||||
actual = qsa_ops.qsa_sparse_paged_attention(
|
||||
@@ -872,12 +903,13 @@ def test_qsa_sparse_paged_attention_correctness(
|
||||
logical_indices,
|
||||
block_table,
|
||||
token_to_req,
|
||||
use_prefill_config=use_prefill_config,
|
||||
)
|
||||
expected = _qsa_sparse_paged_attention_reference(
|
||||
q,
|
||||
k_cache,
|
||||
v_cache,
|
||||
logical_indices,
|
||||
logical_indices[:, :selection_width],
|
||||
block_table,
|
||||
token_to_req,
|
||||
scale,
|
||||
@@ -948,8 +980,10 @@ def test_qsa_split_selection_correctness(workspace_init, decode_query_len: int)
|
||||
block_indices[prefill_slice],
|
||||
max_seq_len=sequence_lengths.max().item(),
|
||||
)
|
||||
# +1: the packed trailing count column (never a token index; excluded
|
||||
# from the comparison).
|
||||
actual = torch.empty(
|
||||
(rows, token_topk + compress_ratio - 1), device="cuda", dtype=torch.int32
|
||||
(rows, token_topk + compress_ratio), device="cuda", dtype=torch.int32
|
||||
)
|
||||
qsa_indexer_ops.expand_qsa_block_indices(
|
||||
block_indices,
|
||||
@@ -977,7 +1011,10 @@ def test_qsa_split_selection_correctness(workspace_init, decode_query_len: int)
|
||||
token_topk,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual.sort().values, expected.sort().values)
|
||||
torch.testing.assert_close(
|
||||
actual[:, : token_topk + compress_ratio - 1].sort().values,
|
||||
expected.sort().values,
|
||||
)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@@ -1001,7 +1038,7 @@ def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) ->
|
||||
block_indices=block_indices,
|
||||
max_seq_len=64, # clamps to the page-table capacity
|
||||
)
|
||||
selected = torch.empty((2, 2051), device="cuda", dtype=torch.int32)
|
||||
selected = torch.empty((2, 2052), device="cuda", dtype=torch.int32)
|
||||
qsa_indexer_ops.expand_qsa_block_indices(
|
||||
block_indices,
|
||||
query_positions,
|
||||
@@ -1013,8 +1050,10 @@ def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) ->
|
||||
|
||||
assert selected[0, :2].tolist() == [0, 1]
|
||||
assert selected[1, :3].tolist() == [0, 1, 2]
|
||||
assert torch.all(selected[0, 2:] == -1)
|
||||
assert torch.all(selected[1, 3:] == -1)
|
||||
assert torch.all(selected[0, 2:2051] == -1)
|
||||
assert torch.all(selected[1, 3:2051] == -1)
|
||||
# The packed trailing column holds each row's valid-entry count.
|
||||
assert selected[:, 2051].tolist() == [2, 3]
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
|
||||
Reference in New Issue
Block a user