diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index a2e2169943..f025d67b08 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -1366,6 +1366,47 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens(): torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0) +def test_flashmla_hisparse_resident_mixed_batch_skips_host_staging(): + """A resident prefill batched with decodes reads resident KV in one kernel + instead of staging its whole history from host.""" + num_tokens = 6 + q = torch.zeros(num_tokens, 4, 576, dtype=torch.bfloat16, device=DEVICE_TYPE) + topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE) + counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE) + kernel_rows: list[int] = [] + + def kernel(q, kv, indices, lengths, actual_num_heads): # noqa: ARG001 + kernel_rows.append(q.shape[0]) + return q[..., :512], None + + index_group = object.__new__(HiSparseMLAIndexGroup) + index_group.caches = [SimpleNamespace(all_context_pages_resident=True)] + index_group.convert_logical_to_physical_topk = MagicMock( + return_value=(topk, counts) + ) + index_group.physical_kv_cache = MagicMock( + return_value=torch.empty(1, device=DEVICE_TYPE) + ) + index_group.stage_prefill_rows = MagicMock(side_effect=AssertionError) + impl = SimpleNamespace( + _bf16_flash_mla_kernel=kernel, + index_group=index_group, + index_group_index=0, + ) + metadata = SimpleNamespace( + req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32), + block_table=torch.zeros((2, 1), dtype=torch.int32), + num_decode_tokens=2, + ) + + out, _ = FlashMLASparseImpl._forward_bf16_kv( + impl, q, torch.empty(1, device=DEVICE_TYPE), topk, metadata, q.shape[1] + ) + + assert kernel_rows == [num_tokens] + assert out.shape == (num_tokens, 4, 512) + + @pytest.mark.parametrize( "seq_lens,max_buf,expected", [ @@ -3268,6 +3309,7 @@ def test_hisparse_mixed_batch_bf16_row_split( kv_pool = kv_cache.squeeze(1).cpu().pin_memory() cache_handle.runtime.bind_source_cache(kv_pool) cache_handle.source_block_table = metadata.block_table + cache_handle.all_context_pages_resident = False staging_calls = [] original_gather = cache_handle.runtime.gather_prefill_cache @@ -3519,6 +3561,9 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention(): num_decodes, 1, dtype=torch.int32, device=device ), swap_in=MagicMock(side_effect=swap_in), + all_context_pages_resident=True, + num_decode_tokens=num_tokens, + num_actual_tokens=num_tokens, ) index_group = object.__new__(HiSparseMLAIndexGroup) index_group.caches = [cache] @@ -3780,14 +3825,16 @@ def test_sparse_mqa_plain_lane_wiring(monkeypatch): [ pytest.param(0, True, False, id="resident"), pytest.param(0, False, False, id="staged"), - pytest.param(2, True, True, id="mixed"), + pytest.param(2, True, True, id="mixed_resident"), + pytest.param(2, False, True, id="mixed_staged"), pytest.param(5, True, False, id="decode"), ], ) def test_sparse_mqa_hisparse_lanes( monkeypatch, num_decode_tokens, all_resident, split_q ): - """HiSparse splits a batch into at most one decode and one prefill call.""" + """HiSparse runs all-resident and decode-only batches in one call against the + hot cache, and otherwise splits into one decode and one staged-prefill call.""" import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla num_tokens, topk_tokens, block_size, head_dim = 5, 8, 4, 6 @@ -3814,9 +3861,9 @@ def test_sparse_mqa_hisparse_lanes( ) # The fill value names the converter that produced a call's indices. index_group.convert_logical_to_physical_topk = MagicMock( - return_value=converted(num_decode_tokens, 1) - if num_decode_tokens - else converted(num_tokens, 2) + side_effect=lambda layer, topk, *args, **kwargs: converted( + topk.shape[0], 1 if topk.shape[0] == num_decode_tokens else 2 + ) ) index_group.stage_prefill_rows = MagicMock(return_value=(staged, None, None)) calls: list[tuple] = [] @@ -3850,36 +3897,41 @@ def test_sparse_mqa_hisparse_lanes( q_rows = q.flatten().tolist() expected = [] - if num_decode_tokens: - # Decode reads the hot cache through the group's conversion. - expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1)) - if num_decode_tokens == 0 and all_resident: + if not num_prefill: + expected.append((q_rows, hot.shape[0], 1)) + elif all_resident: + # Decode and prefill rows read the hot cache in one call. expected.append((q_rows, hot.shape[0], 2)) - elif num_prefill: + else: + if num_decode_tokens: + expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1)) expected.append((q_rows[num_decode_tokens:], staged.shape[0], 3)) assert calls == expected if not num_prefill: # A decode-only batch returns before it looks up the prefill cache. index_group.cache.assert_not_called() assert lse is None - # A lane handed the whole batch would concatenate to more rows than this. - assert out.flatten().tolist() == [1.0] * num_decode_tokens + [2.0] * num_prefill + num_decode_rows = num_tokens if not num_prefill else 0 + if num_prefill and not all_resident: + num_decode_rows = num_decode_tokens + assert out.flatten().tolist() == [1.0] * num_decode_rows + [2.0] * ( + num_tokens - num_decode_rows + ) -def test_hisparse_resident_prefill_uses_attention_block_stride(): - """A batch beyond the swap-row capacity must bypass swap_in. - - The index-group workspace has one more row than the HiSparse swap rows, so - routing on its size sent a 2-token batch with max_num_seqs=1 (a piecewise - CUDA-graph capture) into swap_in, which overflowed its capacity of 1. - """ +@pytest.mark.parametrize("max_swap_rows", [1, 4], ids=["over_capacity", "mixed"]) +def test_hisparse_resident_prefill_uses_attention_block_stride(max_swap_rows): + """Resident batches with prefill rows read resident pages directly, whether + they exceed the swap-row capacity or fit it.""" expected = torch.tensor([[19]], dtype=torch.int32) cache_handle = SimpleNamespace( all_context_pages_resident=True, + num_decode_tokens=1, + num_actual_tokens=2, view=SimpleNamespace(block_size=64, attention_block_stride=832), block_table=torch.tensor([[3]], dtype=torch.int32), batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32), - runtime=SimpleNamespace(max_swap_rows=1), + runtime=SimpleNamespace(max_swap_rows=max_swap_rows), ) index_group = object.__new__(HiSparseMLAIndexGroup) index_group.caches = [cache_handle] diff --git a/vllm/model_executor/layers/attention/sparse_mla_attention.py b/vllm/model_executor/layers/attention/sparse_mla_attention.py index a8fa74f91f..f38d6fa431 100644 --- a/vllm/model_executor/layers/attention/sparse_mla_attention.py +++ b/vllm/model_executor/layers/attention/sparse_mla_attention.py @@ -802,6 +802,27 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic kv_c_and_k_pe_cache.dtype ) assert hot_cache.is_contiguous(), "HiSparse hot buffer must be contiguous" + if ( + num_decode_tokens >= num_tokens + or index_group.cache(layer_index).all_context_pages_resident + ): + physical_topk, valid_counts = index_group.convert_logical_to_physical_topk( + layer_index, + topk, + attn_metadata, + block_stride_rows=None, + return_valid_counts=True, + ) + return self._forward_mqa_kernel( + q, + hot_cache, + physical_topk, + valid_counts, + layer=layer, + block_size=block_size, + is_decode=num_decode_tokens >= num_tokens, + ) + decode_out = decode_lse = None if num_decode_tokens: physical_topk, valid_counts = index_group.convert_logical_to_physical_topk( @@ -825,31 +846,18 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic block_size=block_size, is_decode=True, ) - if num_decode_tokens >= num_tokens: - return decode_out, decode_lse - cache = index_group.cache(layer_index) - if num_decode_tokens == 0 and cache.all_context_pages_resident: - physical_topk, valid_counts = index_group.convert_logical_to_physical_topk( - layer_index, - topk, - attn_metadata, - block_stride_rows=None, - return_valid_counts=True, - ) - prefill_cache = hot_cache - else: - prefill_cache, block_table, req_ids = index_group.stage_prefill_rows( - layer_index, kv_c_and_k_pe_cache, attn_metadata - ) - physical_topk, valid_counts = triton_convert_req_index_to_global_index( - req_ids, - block_table, - topk[num_decode_tokens:], - BLOCK_SIZE=block_size, - NUM_TOPK_TOKENS=topk.shape[1], - return_valid_counts=True, - ) + prefill_cache, block_table, req_ids = index_group.stage_prefill_rows( + layer_index, kv_c_and_k_pe_cache, attn_metadata + ) + physical_topk, valid_counts = triton_convert_req_index_to_global_index( + req_ids, + block_table, + topk[num_decode_tokens:], + BLOCK_SIZE=block_size, + NUM_TOPK_TOKENS=topk.shape[1], + return_valid_counts=True, + ) q_prefill = ( (q[0][num_decode_tokens:], q[1][num_decode_tokens:]) if isinstance(q, tuple) diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py index 2bfcee05c4..04b6edf125 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla_sparse_sm120.py @@ -122,6 +122,26 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada index_group = self.index_group if isinstance(index_group, HiSparseMLAIndexGroup): num_decode_tokens = attn_metadata.num_decode_tokens + physical_kv_cache = index_group.physical_kv_cache(self.index_group_index) + if ( + num_decode_tokens == num_actual_toks + or index_group.cache(self.index_group_index).all_context_pages_resident + ): + topk_indices_physical = cast( + torch.Tensor, + index_group.convert_logical_to_physical_topk( + self.index_group_index, + topk_indices, + attn_metadata, + block_stride_rows=None, + return_valid_counts=False, + ), + ) + return ( + self._run_mqa_kernel(q, physical_kv_cache, topk_indices_physical), + None, + ) + outputs = [] if num_decode_tokens: topk_indices_physical = cast( @@ -137,51 +157,32 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada outputs.append( self._run_mqa_kernel( q[:num_decode_tokens], - index_group.physical_kv_cache(self.index_group_index), + physical_kv_cache, topk_indices_physical, ) ) - if num_decode_tokens < num_actual_toks: - cache = index_group.cache(self.index_group_index) - if num_decode_tokens == 0 and cache.all_context_pages_resident: - topk_indices_physical = cast( - torch.Tensor, - index_group.convert_logical_to_physical_topk( - self.index_group_index, - topk_indices, - attn_metadata, - block_stride_rows=None, - return_valid_counts=False, - ), - ) - prefill_cache = index_group.physical_kv_cache( - self.index_group_index - ) - else: - prefill_cache, block_table, req_ids = ( - index_group.stage_prefill_rows( - self.index_group_index, - kv_c_and_k_pe_cache, - attn_metadata, - ) - ) - topk_indices_physical = cast( - torch.Tensor, - triton_convert_req_index_to_global_index( - req_ids, - block_table, - topk_indices[num_decode_tokens:], - BLOCK_SIZE=attn_metadata.block_size, - NUM_TOPK_TOKENS=topk_indices.shape[1], - ), - ) - outputs.append( - self._run_mqa_kernel( - q[num_decode_tokens:], - prefill_cache, - topk_indices_physical, - ) + prefill_cache, block_table, req_ids = index_group.stage_prefill_rows( + self.index_group_index, + kv_c_and_k_pe_cache, + attn_metadata, + ) + topk_indices_physical = cast( + torch.Tensor, + triton_convert_req_index_to_global_index( + req_ids, + block_table, + topk_indices[num_decode_tokens:], + BLOCK_SIZE=attn_metadata.block_size, + NUM_TOPK_TOKENS=topk_indices.shape[1], + ), + ) + outputs.append( + self._run_mqa_kernel( + q[num_decode_tokens:], + prefill_cache, + topk_indices_physical, ) + ) output = torch.cat(outputs) if len(outputs) > 1 else outputs[0] return output, None diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 80b8da1797..fd4b562f53 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -852,6 +852,21 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]): if cache is not None: assert isinstance(index_group, HiSparseMLAIndexGroup) num_decode_tokens = attn_metadata.num_decode_tokens + if num_decode_tokens < q.shape[0] and cache.all_context_pages_resident: + physical_topk, lengths = index_group.convert_logical_to_physical_topk( + self.index_group_index, + topk_indices, + attn_metadata, + block_stride_rows=None, + return_valid_counts=True, + ) + return self._bf16_flash_mla_kernel( + q, + index_group.physical_kv_cache(self.index_group_index), + physical_topk, + lengths, + actual_num_heads, + ) if num_decode_tokens > 0: decode_topk, decode_lengths = ( index_group.convert_logical_to_physical_topk( diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index 0dfda0ded9..ba787b38da 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -233,7 +233,10 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup): cache = self.cache(layer_index) num_tokens = logical_topk_indices.shape[0] req_id_per_token = attn_metadata.req_id_per_token[:num_tokens] - if num_tokens > cache.runtime.max_swap_rows: + if num_tokens > cache.runtime.max_swap_rows or ( + cache.all_context_pages_resident + and cache.num_decode_tokens < cache.num_actual_tokens + ): # Prefill-sized batches do not fit the decode residency workspace. # Non-resident prefills are staged before reaching this path. assert cache.all_context_pages_resident