From 1afb029839a7c84f14dae30bb08f2fea79184431 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 8 Oct 2026 15:45:44 -0400 Subject: [PATCH] [Bugfix][HiSparse] Route by swap-row capacity, not the index-group workspace (#60682) Signed-off-by: Matthew Bonanni Co-authored-by: Claude Opus 5.5 --- tests/v1/attention/test_sparse_mla_backends.py | 12 ++++++++++-- vllm/v1/attention/backends/mla/index_group.py | 2 +- vllm/v1/hisparse/runtime.py | 9 ++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/v1/attention/test_sparse_mla_backends.py b/tests/v1/attention/test_sparse_mla_backends.py index 4d33075c93..973a9e03e8 100644 --- a/tests/v1/attention/test_sparse_mla_backends.py +++ b/tests/v1/attention/test_sparse_mla_backends.py @@ -3296,7 +3296,8 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention(): cache = SimpleNamespace( runtime=SimpleNamespace( - hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)) + hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)), + max_swap_rows=num_tokens, ), source_block_table=torch.empty( num_decodes, 1, dtype=torch.int32, device=device @@ -3604,15 +3605,22 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group(): def test_hisparse_resident_prefill_uses_attention_block_stride(): + """A batch beyond the swap-row capacity must bypass swap_in. + + The index-group workspace has one more row than the HiSparse swap rows, so + routing on its size sent a 2-token batch with max_num_seqs=1 (a piecewise + CUDA-graph capture) into swap_in, which overflowed its capacity of 1. + """ expected = torch.tensor([[19]], dtype=torch.int32) cache_handle = SimpleNamespace( all_context_pages_resident=True, view=SimpleNamespace(block_size=64, attention_block_stride=832), block_table=torch.tensor([[3]], dtype=torch.int32), + runtime=SimpleNamespace(max_swap_rows=1), ) index_group = object.__new__(HiSparseMLAIndexGroup) index_group.caches = [cache_handle] - index_group.physical_topk_indices = torch.empty((1, 1), dtype=torch.int32) + index_group.physical_topk_indices = torch.empty((2, 1), dtype=torch.int32) index_group._convert_once = MagicMock(return_value=expected) topk = torch.zeros((2, 1), dtype=torch.int32) metadata = SimpleNamespace( diff --git a/vllm/v1/attention/backends/mla/index_group.py b/vllm/v1/attention/backends/mla/index_group.py index 5a18332e41..39f205b91d 100644 --- a/vllm/v1/attention/backends/mla/index_group.py +++ b/vllm/v1/attention/backends/mla/index_group.py @@ -233,7 +233,7 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup): cache = self.cache(layer_index) num_tokens = logical_topk_indices.shape[0] req_id_per_token = attn_metadata.req_id_per_token[:num_tokens] - if num_tokens > self.physical_topk_indices.shape[0]: + if num_tokens > cache.runtime.max_swap_rows: # Prefill-sized batches do not fit the decode residency workspace. # Non-resident prefills are staged before reaching this path. assert cache.all_context_pages_resident diff --git a/vllm/v1/hisparse/runtime.py b/vllm/v1/hisparse/runtime.py index a6378ca914..1f7a5ad5ca 100644 --- a/vllm/v1/hisparse/runtime.py +++ b/vllm/v1/hisparse/runtime.py @@ -894,14 +894,17 @@ class HiSparseRuntime: def begin_forward(self) -> None: self._swap_step = 0 + @property + def max_swap_rows(self) -> int: + return self.index_group.shared_topk.physical_topk_indices.shape[0] + def _step_rows(self, num_tokens: int) -> slice: start = self._swap_step * num_tokens stop = start + num_tokens - if stop > self.index_group.shared_topk.physical_topk_indices.shape[0]: + if stop > self.max_swap_rows: raise ValueError( "HiSparse swap rows exceed the configured speculative decode " - f"capacity: stop={stop}, capacity=" - f"{self.index_group.shared_topk.physical_topk_indices.shape[0]}." + f"capacity: stop={stop}, capacity={self.max_swap_rows}." ) self._swap_step += 1 return slice(start, stop)