diff --git a/tests/models/qwen4_exp/test_qsa_reference.py b/tests/models/qwen4_exp/test_qsa_reference.py index 163f3a4f9c..01dcb20f40 100644 --- a/tests/models/qwen4_exp/test_qsa_reference.py +++ b/tests/models/qwen4_exp/test_qsa_reference.py @@ -244,6 +244,7 @@ def test_qsa_side_metadata_marks_cudagraph_padding_inert() -> None: num_actual_tokens=16, num_reqs=4, max_query_len=4, + max_seq_len=68, query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc.cpu(), seq_lens=torch.tensor([68, 68, 68, 0], dtype=torch.int32, device=device), @@ -297,6 +298,7 @@ def test_qsa_circular_buffer_metadata_keeps_only_each_requests_suffix() -> None: num_actual_tokens=16, num_reqs=3, max_query_len=7, + max_seq_len=11, query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc.cpu(), seq_lens=torch.tensor([9, 11, 0], dtype=torch.int32, device=device), @@ -428,6 +430,7 @@ def test_qsa_compressed_metadata_keeps_dummy_slots_inert() -> None: num_actual_tokens=8, num_reqs=3, max_query_len=5, + max_seq_len=12, query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc.cpu(), seq_lens=torch.tensor([7, 0, 12], dtype=torch.int32, device=device), @@ -673,13 +676,24 @@ def test_qsa_decode_selection_correctness( @requires_qsa_kernels -def test_qsa_prefill_selection_correctness() -> None: +@pytest.mark.parametrize("seq_len_slack", [0, 1792]) +@pytest.mark.parametrize("force_chunk", [False, True]) +def test_qsa_prefill_selection_correctness( + monkeypatch: pytest.MonkeyPatch, seq_len_slack: int, force_chunk: bool +) -> None: + # page_size=24 (does not divide the 64-aligned clipped width) and an + # oversized page table, so the clipped logits width comes from + # max_seq_len, not page geometry. seq_len_slack > 0 simulates the + # spec-decode case where the bound is an over-estimate. force_chunk + # drives the logits budget to one row per chunk. + if force_chunk: + monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "0") torch.manual_seed(2) query_lens = [3, 33] rows, heads, head_dim = sum(query_lens), 4, 128 q = torch.randn(rows, heads, head_dim, device="cuda", dtype=torch.bfloat16) - cache = torch.randn(160, 16, 1, head_dim, device="cuda", dtype=torch.bfloat16) - page_table = torch.randperm(160, device="cuda", dtype=torch.int32).reshape(2, 80) + cache = torch.randn(128, 24, 1, head_dim, device="cuda", dtype=torch.bfloat16) + page_table = torch.randperm(128, device="cuda", dtype=torch.int32).reshape(2, 64) token_to_req = torch.repeat_interleave( torch.arange(2, device="cuda", dtype=torch.int32), torch.tensor(query_lens, device="cuda"), @@ -713,6 +727,7 @@ def test_qsa_prefill_selection_correctness() -> None: compress_ratio, max(query_lens), actual, + max_seq_len=sequence_lengths.max().item() + seq_len_slack, ) expected = _qsa_select_paged_reference( q, @@ -931,6 +946,7 @@ def test_qsa_split_selection_correctness(workspace_init, decode_query_len: int) compress_ratio, query_lens[-1], block_indices[prefill_slice], + max_seq_len=sequence_lengths.max().item(), ) actual = torch.empty( (rows, token_topk + compress_ratio - 1), device="cuda", dtype=torch.int32 @@ -983,6 +999,7 @@ def test_qsa_selection_handles_no_complete_compressed_blocks(workspace_init) -> compress_ratio=4, max_query_len=2, block_indices=block_indices, + max_seq_len=64, # clamps to the page-table capacity ) selected = torch.empty((2, 2051), device="cuda", dtype=torch.int32) qsa_indexer_ops.expand_qsa_block_indices( diff --git a/vllm/models/qwen4_exp/common/qsa_cache.py b/vllm/models/qwen4_exp/common/qsa_cache.py index 539b7c1121..acddd4f9e6 100644 --- a/vllm/models/qwen4_exp/common/qsa_cache.py +++ b/vllm/models/qwen4_exp/common/qsa_cache.py @@ -591,6 +591,7 @@ class QSAForwardMetadata(AttentionMetadata): num_prefill_tokens: int max_query_len: int decode_query_len: int + max_seq_len: int storage_block_size: int compress_ratio: int @@ -717,6 +718,7 @@ class QSAMetadataBuilder(AttentionMetadataBuilder[QSAForwardMetadata]): num_prefill_tokens=num_prefill_tokens, max_query_len=common_attn_metadata.max_query_len, decode_query_len=decode_query_len, + max_seq_len=common_attn_metadata.max_seq_len, storage_block_size=self.storage_block_size, compress_ratio=self.compress_ratio, ) diff --git a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py index ffaac6434c..11c787b4e5 100644 --- a/vllm/models/qwen4_exp/nvidia/indexer_qsa.py +++ b/vllm/models/qwen4_exp/nvidia/indexer_qsa.py @@ -405,6 +405,7 @@ class QSAIndexer(nn.Module): self.compress_ratio, compressed_metadata.max_query_len, block_indices[prefill_slice], + compressed_metadata.max_seq_len, ) expand_qsa_block_indices( block_indices, diff --git a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py index d58f28078b..1f0abae468 100644 --- a/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py +++ b/vllm/models/qwen4_exp/nvidia/ops/qsa_indexer.py @@ -97,7 +97,7 @@ def _qsa_mqa_paged_uniform_kernel( scores, (BLOCK_N, DECODE_QUERY_LEN_PADDED, NUM_HEADS_PADDED), ) - score = tl.sum(scores, axis=2) / HEAD_DIM**0.5 + score = tl.sum(scores, axis=2) tl.store( logits_ptr + rows[None, :] * stride_logits_row + columns[:, None], score, @@ -123,8 +123,8 @@ def _qsa_mqa_paged_prefill_kernel( stride_logits_row, num_rows, query_offset, + page_table_width, PAGE_SIZE: tl.constexpr, - PAGE_TABLE_WIDTH: tl.constexpr, NUM_HEADS: tl.constexpr, HEAD_DIM: tl.constexpr, TILE_R: tl.constexpr, @@ -132,7 +132,7 @@ def _qsa_mqa_paged_prefill_kernel( K_TILES: tl.constexpr, STAGES: tl.constexpr, ) -> None: - NUM_COLUMNS: tl.constexpr = PAGE_TABLE_WIDTH * PAGE_SIZE + num_columns = page_table_width * PAGE_SIZE NUM_HEADS_PADDED: tl.constexpr = triton.next_power_of_2(NUM_HEADS) # tl.dot requires a reduction dimension of at least 16. BLOCK_D: tl.constexpr = max(16, triton.next_power_of_2(HEAD_DIM)) @@ -163,7 +163,7 @@ def _qsa_mqa_paged_prefill_kernel( if k_tile_start * BLOCK_N >= max_visible: return k_tile_end = tl.minimum(k_tile_start + K_TILES, tl.cdiv(max_visible, BLOCK_N)) - k_tile_end = tl.minimum(k_tile_end, tl.cdiv(NUM_COLUMNS, BLOCK_N)) + k_tile_end = tl.minimum(k_tile_end, tl.cdiv(num_columns, BLOCK_N)) dims = tl.arange(0, BLOCK_D) m = tl.arange(0, TILE_R * NUM_HEADS_PADDED) @@ -184,7 +184,7 @@ def _qsa_mqa_paged_prefill_kernel( for tile in tl.range(k_tile_start, k_tile_end, num_stages=STAGES): columns = tile * BLOCK_N + column_offsets live = columns < max_visible - logical_page = tl.minimum(columns // PAGE_SIZE, PAGE_TABLE_WIDTH - 1) + logical_page = tl.minimum(columns // PAGE_SIZE, page_table_width - 1) page_offset = columns % PAGE_SIZE physical_page = tl.load( page_table_ptr + request * stride_table_req + logical_page, @@ -202,11 +202,11 @@ def _qsa_mqa_paged_prefill_kernel( ) scores = tl.dot(keys, query, out_dtype=tl.float32) scores = tl.reshape(scores, (BLOCK_N, TILE_R, NUM_HEADS_PADDED)) - score = tl.sum(tl.maximum(scores, 0.0), axis=2) / HEAD_DIM**0.5 + score = tl.sum(tl.maximum(scores, 0.0), axis=2) store_mask = ( valid_rows[None, :] & (columns[:, None] < visible[None, :]) - & (columns[:, None] < NUM_COLUMNS) + & (columns[:, None] < num_columns) ) tl.store( logits_ptr + rows[None, :] * stride_logits_row + columns[:, None], @@ -372,22 +372,25 @@ def _prefill_logits( query_start_loc: torch.Tensor, visible_blocks: torch.Tensor, max_query_len: int, + logits_width: int, query_offset: int, num_queries: int, ) -> torch.Tensor: assert query_start_loc.shape == (page_table.shape[0] + 1,) assert visible_blocks.shape == (q.shape[0],) assert 0 <= query_offset <= query_offset + num_queries <= q.shape[0] + assert 0 < logits_width <= page_table.shape[1] * k_cache.shape[1] - columns = page_table.shape[1] * k_cache.shape[1] - logits = torch.empty((num_queries, columns), dtype=torch.float32, device=q.device) + logits = torch.empty( + (num_queries, logits_width), dtype=torch.float32, device=q.device + ) TILE_R = 64 BLOCK_N = 64 K_TILES = 16 grid = ( page_table.shape[0], triton.cdiv(min(num_queries, max_query_len), TILE_R), - triton.cdiv(columns, BLOCK_N * K_TILES), + triton.cdiv(logits_width, BLOCK_N * K_TILES), ) _qsa_mqa_paged_prefill_kernel[grid]( q, @@ -402,8 +405,8 @@ def _prefill_logits( *logits.stride()[:-1], num_queries, query_offset, + page_table.shape[1], PAGE_SIZE=k_cache.shape[1], - PAGE_TABLE_WIDTH=page_table.shape[1], NUM_HEADS=q.shape[1], HEAD_DIM=q.shape[2], TILE_R=TILE_R, @@ -557,6 +560,7 @@ def qsa_select_paged_prefill( compress_ratio: int, max_query_len: int, block_indices: torch.Tensor, + max_seq_len: int, ) -> None: """Score and select compressed prefill blocks in bounded chunks. @@ -572,16 +576,21 @@ def qsa_select_paged_prefill( compress_ratio: Number of logical tokens represented by a cache row. max_query_len: Maximum number of query tokens in one request. block_indices: Compressed-index output buffer. + max_seq_len: Longest context length in the batch this step. """ assert token_topk % compress_ratio == 0 assert block_indices.shape == (q.shape[0], token_topk // compress_ratio) rows = q.shape[0] - columns = page_table.shape[1] * k_cache.shape[1] + # No row scores beyond cdiv(max_seq_len, compress_ratio) compressed + # columns. Round up to 64 to keep the logits row stride + # cooperative_topk-compatible. + logits_width = triton.cdiv(triton.cdiv(max_seq_len, compress_ratio), 64) * 64 + logits_width = min(max(64, logits_width), page_table.shape[1] * k_cache.shape[1]) # chunk the inputs to keep temp logits below VLLM_SPARSE_INDEXER_MAX_LOGITS_MB max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 - rows_per_chunk = max(1, max_logits_bytes // (columns * 4)) + rows_per_chunk = max(1, max_logits_bytes // (logits_width * 4)) topk_workspace = torch.empty( (_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device ) @@ -596,6 +605,7 @@ def qsa_select_paged_prefill( query_start_loc, visible_blocks, max_query_len, + logits_width, query_offset=query_start, num_queries=query_end - query_start, )