[Qwen3.8-Flash-Next] Compact indexer logits workspace to improve prefill efficiency (#54915)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
@@ -244,6 +244,7 @@ def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None:
|
||||
num_actual_tokens=16,
|
||||
num_reqs=4,
|
||||
max_query_len=4,
|
||||
max_seq_len=68,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device),
|
||||
@@ -297,6 +298,7 @@ def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None:
|
||||
num_actual_tokens=16,
|
||||
num_reqs=3,
|
||||
max_query_len=7,
|
||||
max_seq_len=11,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device),
|
||||
@@ -428,6 +430,7 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None:
|
||||
num_actual_tokens=8,
|
||||
num_reqs=3,
|
||||
max_query_len=5,
|
||||
max_seq_len=12,
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device),
|
||||
@@ -673,13 +676,24 @@ def test_qsa_decode_selection_correctness(
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
def test_qsa_prefill_selection_correctness() -> None:
|
||||
@pytest.mark.parametrize("seq_len_slack", [0, 1792])
|
||||
@pytest.mark.parametrize("force_chunk", [False, True])
|
||||
def test_qsa_prefill_selection_correctness(
|
||||
monkeypatch: pytest.MonkeyPatch, seq_len_slack: int, force_chunk: bool
|
||||
) -> None:
|
||||
# page_size=24 (does not divide the 64-aligned clipped width) and an
|
||||
# oversized page table, so the clipped logits width comes from
|
||||
# max_seq_len, not page geometry. seq_len_slack > 0 simulates the
|
||||
# spec-decode case where the bound is an over-estimate. force_chunk
|
||||
# drives the logits budget to one row per chunk.
|
||||
if force_chunk:
|
||||
monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "0")
|
||||
torch.manual_seed(2)
|
||||
query_lens = [3, 33]
|
||||
rows, heads, head_dim = sum(query_lens), 4, 128
|
||||
q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
cache = torch.randn(160, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(160, device="cuda", dtype=torch.int32).reshape(2, 80)
|
||||
cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
page_table = torch.randperm(128, device="cuda", dtype=torch.int32).reshape(2, 64)
|
||||
token_to_req = torch.repeat_interleave(
|
||||
torch.arange(2, device="cuda", dtype=torch.int32),
|
||||
torch.tensor(query_lens, device="cuda"),
|
||||
@@ -713,6 +727,7 @@ def test_qsa_prefill_selection_correctness() -> None:
|
||||
compress_ratio,
|
||||
max(query_lens),
|
||||
actual,
|
||||
max_seq_len=sequence_lengths.max().item() + seq_len_slack,
|
||||
)
|
||||
expected = _qsa_select_paged_reference(
|
||||
q,
|
||||
@@ -931,6 +946,7 @@ def test_qsa_split_selection_correctness(workspace_init, decode_query_len: int)
|
||||
compress_ratio,
|
||||
query_lens[-1],
|
||||
block_indices[prefill_slice],
|
||||
max_seq_len=sequence_lengths.max().item(),
|
||||
)
|
||||
actual = torch.empty(
|
||||
(rows, token_topk + compress_ratio - 1), device="cuda", dtype=torch.int32
|
||||
@@ -983,6 +999,7 @@ def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) ->
|
||||
compress_ratio=4,
|
||||
max_query_len=2,
|
||||
block_indices=block_indices,
|
||||
max_seq_len=64, # clamps to the page-table capacity
|
||||
)
|
||||
selected = torch.empty((2, 2051), device="cuda", dtype=torch.int32)
|
||||
qsa_indexer_ops.expand_qsa_block_indices(
|
||||
|
||||
@@ -591,6 +591,7 @@ class QSAForwardMetadata(AttentionMetadata):
|
||||
num_prefill_tokens: int
|
||||
max_query_len: int
|
||||
decode_query_len: int
|
||||
max_seq_len: int
|
||||
storage_block_size: int
|
||||
compress_ratio: int
|
||||
|
||||
@@ -717,6 +718,7 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]):
|
||||
num_prefill_tokens=num_prefill_tokens,
|
||||
max_query_len=common_attn_metadata.max_query_len,
|
||||
decode_query_len=decode_query_len,
|
||||
max_seq_len=common_attn_metadata.max_seq_len,
|
||||
storage_block_size=self.storage_block_size,
|
||||
compress_ratio=self.compress_ratio,
|
||||
)
|
||||
|
||||
@@ -405,6 +405,7 @@ class QSAIndexer(nn.Module):
|
||||
self.compress_ratio,
|
||||
compressed_metadata.max_query_len,
|
||||
block_indices[prefill_slice],
|
||||
compressed_metadata.max_seq_len,
|
||||
)
|
||||
expand_qsa_block_indices(
|
||||
block_indices,
|
||||
|
||||
@@ -97,7 +97,7 @@ def _qsa_mqa_paged_uniform_kernel(
|
||||
scores,
|
||||
(BLOCK_N, DECODE_QUERY_LEN_PADDED, NUM_HEADS_PADDED),
|
||||
)
|
||||
score = tl.sum(scores, axis=2) / HEAD_DIM**0.5
|
||||
score = tl.sum(scores, axis=2)
|
||||
tl.store(
|
||||
logits_ptr + rows[None, :] * stride_logits_row + columns[:, None],
|
||||
score,
|
||||
@@ -123,8 +123,8 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
stride_logits_row,
|
||||
num_rows,
|
||||
query_offset,
|
||||
page_table_width,
|
||||
PAGE_SIZE: tl.constexpr,
|
||||
PAGE_TABLE_WIDTH: tl.constexpr,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
TILE_R: tl.constexpr,
|
||||
@@ -132,7 +132,7 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
K_TILES: tl.constexpr,
|
||||
STAGES: tl.constexpr,
|
||||
) -> None:
|
||||
NUM_COLUMNS: tl.constexpr = PAGE_TABLE_WIDTH * PAGE_SIZE
|
||||
num_columns = page_table_width * PAGE_SIZE
|
||||
NUM_HEADS_PADDED: tl.constexpr = triton.next_power_of_2(NUM_HEADS)
|
||||
# tl.dot requires a reduction dimension of at least 16.
|
||||
BLOCK_D: tl.constexpr = max(16, triton.next_power_of_2(HEAD_DIM))
|
||||
@@ -163,7 +163,7 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
if k_tile_start * BLOCK_N >= max_visible:
|
||||
return
|
||||
k_tile_end = tl.minimum(k_tile_start + K_TILES, tl.cdiv(max_visible, BLOCK_N))
|
||||
k_tile_end = tl.minimum(k_tile_end, tl.cdiv(NUM_COLUMNS, BLOCK_N))
|
||||
k_tile_end = tl.minimum(k_tile_end, tl.cdiv(num_columns, BLOCK_N))
|
||||
|
||||
dims = tl.arange(0, BLOCK_D)
|
||||
m = tl.arange(0, TILE_R * NUM_HEADS_PADDED)
|
||||
@@ -184,7 +184,7 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
for tile in tl.range(k_tile_start, k_tile_end, num_stages=STAGES):
|
||||
columns = tile * BLOCK_N + column_offsets
|
||||
live = columns < max_visible
|
||||
logical_page = tl.minimum(columns // PAGE_SIZE, PAGE_TABLE_WIDTH - 1)
|
||||
logical_page = tl.minimum(columns // PAGE_SIZE, page_table_width - 1)
|
||||
page_offset = columns % PAGE_SIZE
|
||||
physical_page = tl.load(
|
||||
page_table_ptr + request * stride_table_req + logical_page,
|
||||
@@ -202,11 +202,11 @@ def _qsa_mqa_paged_prefill_kernel(
|
||||
)
|
||||
scores = tl.dot(keys, query, out_dtype=tl.float32)
|
||||
scores = tl.reshape(scores, (BLOCK_N, TILE_R, NUM_HEADS_PADDED))
|
||||
score = tl.sum(tl.maximum(scores, 0.0), axis=2) / HEAD_DIM**0.5
|
||||
score = tl.sum(tl.maximum(scores, 0.0), axis=2)
|
||||
store_mask = (
|
||||
valid_rows[None, :]
|
||||
& (columns[:, None] < visible[None, :])
|
||||
& (columns[:, None] < NUM_COLUMNS)
|
||||
& (columns[:, None] < num_columns)
|
||||
)
|
||||
tl.store(
|
||||
logits_ptr + rows[None, :] * stride_logits_row + columns[:, None],
|
||||
@@ -372,22 +372,25 @@ def _prefill_logits(
|
||||
query_start_loc: torch.Tensor,
|
||||
visible_blocks: torch.Tensor,
|
||||
max_query_len: int,
|
||||
logits_width: int,
|
||||
query_offset: int,
|
||||
num_queries: int,
|
||||
) -> torch.Tensor:
|
||||
assert query_start_loc.shape == (page_table.shape[0] + 1,)
|
||||
assert visible_blocks.shape == (q.shape[0],)
|
||||
assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0]
|
||||
assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1]
|
||||
|
||||
columns = page_table.shape[1] * k_cache.shape[1]
|
||||
logits = torch.empty((num_queries, columns), dtype=torch.float32, device=q.device)
|
||||
logits = torch.empty(
|
||||
(num_queries, logits_width), dtype=torch.float32, device=q.device
|
||||
)
|
||||
TILE_R = 64
|
||||
BLOCK_N = 64
|
||||
K_TILES = 16
|
||||
grid = (
|
||||
page_table.shape[0],
|
||||
triton.cdiv(min(num_queries, max_query_len), TILE_R),
|
||||
triton.cdiv(columns, BLOCK_N * K_TILES),
|
||||
triton.cdiv(logits_width, BLOCK_N * K_TILES),
|
||||
)
|
||||
_qsa_mqa_paged_prefill_kernel[grid](
|
||||
q,
|
||||
@@ -402,8 +405,8 @@ def _prefill_logits(
|
||||
*logits.stride()[:-1],
|
||||
num_queries,
|
||||
query_offset,
|
||||
page_table.shape[1],
|
||||
PAGE_SIZE=k_cache.shape[1],
|
||||
PAGE_TABLE_WIDTH=page_table.shape[1],
|
||||
NUM_HEADS=q.shape[1],
|
||||
HEAD_DIM=q.shape[2],
|
||||
TILE_R=TILE_R,
|
||||
@@ -557,6 +560,7 @@ def qsa_select_paged_prefill(
|
||||
compress_ratio: int,
|
||||
max_query_len: int,
|
||||
block_indices: torch.Tensor,
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Score and select compressed prefill blocks in bounded chunks.
|
||||
|
||||
@@ -572,16 +576,21 @@ def qsa_select_paged_prefill(
|
||||
compress_ratio: Number of logical tokens represented by a cache row.
|
||||
max_query_len: Maximum number of query tokens in one request.
|
||||
block_indices: Compressed-index output buffer.
|
||||
max_seq_len: Longest context length in the batch this step.
|
||||
"""
|
||||
|
||||
assert token_topk % compress_ratio == 0
|
||||
assert block_indices.shape == (q.shape[0], token_topk // compress_ratio)
|
||||
rows = q.shape[0]
|
||||
columns = page_table.shape[1] * k_cache.shape[1]
|
||||
# No row scores beyond cdiv(max_seq_len, compress_ratio) compressed
|
||||
# columns. Round up to 64 to keep the logits row stride
|
||||
# cooperative_topk-compatible.
|
||||
logits_width = triton.cdiv(triton.cdiv(max_seq_len, compress_ratio), 64) * 64
|
||||
logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1])
|
||||
|
||||
# chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB
|
||||
max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||||
rows_per_chunk = max(1, max_logits_bytes // (columns * 4))
|
||||
rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4))
|
||||
topk_workspace = torch.empty(
|
||||
(_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device
|
||||
)
|
||||
@@ -596,6 +605,7 @@ def qsa_select_paged_prefill(
|
||||
query_start_loc,
|
||||
visible_blocks,
|
||||
max_query_len,
|
||||
logits_width,
|
||||
query_offset=query_start,
|
||||
num_queries=query_end - query_start,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user