[Bugfix][HiSparse] Route by swap-row capacity, not the index-group workspace (#60682)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-08 19:45:44 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c23ca06b6d
commit 1afb029839
3 changed files with 17 additions and 6 deletions
+10 -2
View File
@@ -3296,7 +3296,8 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
cache = SimpleNamespace(
runtime=SimpleNamespace(
hot=SimpleNamespace(attention_cache=torch.empty(1, device=device))
hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)),
max_swap_rows=num_tokens,
),
source_block_table=torch.empty(
num_decodes, 1, dtype=torch.int32, device=device
@@ -3604,15 +3605,22 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group():
def test_hisparse_resident_prefill_uses_attention_block_stride():
"""A batch beyond the swap-row capacity must bypass swap_in.
The index-group workspace has one more row than the HiSparse swap rows, so
routing on its size sent a 2-token batch with max_num_seqs=1 (a piecewise
CUDA-graph capture) into swap_in, which overflowed its capacity of 1.
"""
expected = torch.tensor([[19]], dtype=torch.int32)
cache_handle = SimpleNamespace(
all_context_pages_resident=True,
view=SimpleNamespace(block_size=64, attention_block_stride=832),
block_table=torch.tensor([[3]], dtype=torch.int32),
runtime=SimpleNamespace(max_swap_rows=1),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache_handle]
index_group.physical_topk_indices = torch.empty((1, 1), dtype=torch.int32)
index_group.physical_topk_indices = torch.empty((2, 1), dtype=torch.int32)
index_group._convert_once = MagicMock(return_value=expected)
topk = torch.zeros((2, 1), dtype=torch.int32)
metadata = SimpleNamespace(
@@ -233,7 +233,7 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
cache = self.cache(layer_index)
num_tokens = logical_topk_indices.shape[0]
req_id_per_token = attn_metadata.req_id_per_token[:num_tokens]
if num_tokens > self.physical_topk_indices.shape[0]:
if num_tokens > cache.runtime.max_swap_rows:
# Prefill-sized batches do not fit the decode residency workspace.
# Non-resident prefills are staged before reaching this path.
assert cache.all_context_pages_resident
+6 -3
View File
@@ -894,14 +894,17 @@ class HiSparseRuntime:
def begin_forward(self) -> None:
self._swap_step = 0
@property
def max_swap_rows(self) -> int:
return self.index_group.shared_topk.physical_topk_indices.shape[0]
def _step_rows(self, num_tokens: int) -> slice:
start = self._swap_step * num_tokens
stop = start + num_tokens
if stop > self.index_group.shared_topk.physical_topk_indices.shape[0]:
if stop > self.max_swap_rows:
raise ValueError(
"HiSparse swap rows exceed the configured speculative decode "
f"capacity: stop={stop}, capacity="
f"{self.index_group.shared_topk.physical_topk_indices.shape[0]}."
f"capacity: stop={stop}, capacity={self.max_swap_rows}."
)
self._swap_step += 1
return slice(start, stop)