[Bugfix][HiSparse] Route by swap-row capacity, not the index-group workspace (#60682)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c23ca06b6d
commit
1afb029839
@@ -3296,7 +3296,8 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
|
||||
|
||||
cache = SimpleNamespace(
|
||||
runtime=SimpleNamespace(
|
||||
hot=SimpleNamespace(attention_cache=torch.empty(1, device=device))
|
||||
hot=SimpleNamespace(attention_cache=torch.empty(1, device=device)),
|
||||
max_swap_rows=num_tokens,
|
||||
),
|
||||
source_block_table=torch.empty(
|
||||
num_decodes, 1, dtype=torch.int32, device=device
|
||||
@@ -3604,15 +3605,22 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group():
|
||||
|
||||
|
||||
def test_hisparse_resident_prefill_uses_attention_block_stride():
|
||||
"""A batch beyond the swap-row capacity must bypass swap_in.
|
||||
|
||||
The index-group workspace has one more row than the HiSparse swap rows, so
|
||||
routing on its size sent a 2-token batch with max_num_seqs=1 (a piecewise
|
||||
CUDA-graph capture) into swap_in, which overflowed its capacity of 1.
|
||||
"""
|
||||
expected = torch.tensor([[19]], dtype=torch.int32)
|
||||
cache_handle = SimpleNamespace(
|
||||
all_context_pages_resident=True,
|
||||
view=SimpleNamespace(block_size=64, attention_block_stride=832),
|
||||
block_table=torch.tensor([[3]], dtype=torch.int32),
|
||||
runtime=SimpleNamespace(max_swap_rows=1),
|
||||
)
|
||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||
index_group.caches = [cache_handle]
|
||||
index_group.physical_topk_indices = torch.empty((1, 1), dtype=torch.int32)
|
||||
index_group.physical_topk_indices = torch.empty((2, 1), dtype=torch.int32)
|
||||
index_group._convert_once = MagicMock(return_value=expected)
|
||||
topk = torch.zeros((2, 1), dtype=torch.int32)
|
||||
metadata = SimpleNamespace(
|
||||
|
||||
@@ -233,7 +233,7 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
|
||||
cache = self.cache(layer_index)
|
||||
num_tokens = logical_topk_indices.shape[0]
|
||||
req_id_per_token = attn_metadata.req_id_per_token[:num_tokens]
|
||||
if num_tokens > self.physical_topk_indices.shape[0]:
|
||||
if num_tokens > cache.runtime.max_swap_rows:
|
||||
# Prefill-sized batches do not fit the decode residency workspace.
|
||||
# Non-resident prefills are staged before reaching this path.
|
||||
assert cache.all_context_pages_resident
|
||||
|
||||
@@ -894,14 +894,17 @@ class HiSparseRuntime:
|
||||
def begin_forward(self) -> None:
|
||||
self._swap_step = 0
|
||||
|
||||
@property
|
||||
def max_swap_rows(self) -> int:
|
||||
return self.index_group.shared_topk.physical_topk_indices.shape[0]
|
||||
|
||||
def _step_rows(self, num_tokens: int) -> slice:
|
||||
start = self._swap_step * num_tokens
|
||||
stop = start + num_tokens
|
||||
if stop > self.index_group.shared_topk.physical_topk_indices.shape[0]:
|
||||
if stop > self.max_swap_rows:
|
||||
raise ValueError(
|
||||
"HiSparse swap rows exceed the configured speculative decode "
|
||||
f"capacity: stop={stop}, capacity="
|
||||
f"{self.index_group.shared_topk.physical_topk_indices.shape[0]}."
|
||||
f"capacity: stop={stop}, capacity={self.max_swap_rows}."
|
||||
)
|
||||
self._swap_step += 1
|
||||
return slice(start, stop)
|
||||
|
||||
Reference in New Issue
Block a user