[HiSparse] Read GPU-resident KV directly for mixed and BF16 prefills (#60417)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-13 16:40:20 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c19c263ac8
commit 292da21f9d
5 changed files with 165 additions and 86 deletions
+72 -20
View File
@@ -1366,6 +1366,47 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
def test_flashmla_hisparse_resident_mixed_batch_skips_host_staging():
"""A resident prefill batched with decodes reads resident KV in one kernel
instead of staging its whole history from host."""
num_tokens = 6
q = torch.zeros(num_tokens, 4, 576, dtype=torch.bfloat16, device=DEVICE_TYPE)
topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE)
kernel_rows: list[int] = []
def kernel(q, kv, indices, lengths, actual_num_heads): # noqa: ARG001
kernel_rows.append(q.shape[0])
return q[..., :512], None
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [SimpleNamespace(all_context_pages_resident=True)]
index_group.convert_logical_to_physical_topk = MagicMock(
return_value=(topk, counts)
)
index_group.physical_kv_cache = MagicMock(
return_value=torch.empty(1, device=DEVICE_TYPE)
)
index_group.stage_prefill_rows = MagicMock(side_effect=AssertionError)
impl = SimpleNamespace(
_bf16_flash_mla_kernel=kernel,
index_group=index_group,
index_group_index=0,
)
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32),
block_table=torch.zeros((2, 1), dtype=torch.int32),
num_decode_tokens=2,
)
out, _ = FlashMLASparseImpl._forward_bf16_kv(
impl, q, torch.empty(1, device=DEVICE_TYPE), topk, metadata, q.shape[1]
)
assert kernel_rows == [num_tokens]
assert out.shape == (num_tokens, 4, 512)
@pytest.mark.parametrize(
"seq_lens,max_buf,expected",
[
@@ -3268,6 +3309,7 @@ def test_hisparse_mixed_batch_bf16_row_split(
kv_pool = kv_cache.squeeze(1).cpu().pin_memory()
cache_handle.runtime.bind_source_cache(kv_pool)
cache_handle.source_block_table = metadata.block_table
cache_handle.all_context_pages_resident = False
staging_calls = []
original_gather = cache_handle.runtime.gather_prefill_cache
@@ -3519,6 +3561,9 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
num_decodes, 1, dtype=torch.int32, device=device
),
swap_in=MagicMock(side_effect=swap_in),
all_context_pages_resident=True,
num_decode_tokens=num_tokens,
num_actual_tokens=num_tokens,
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache]
@@ -3780,14 +3825,16 @@ def test_sparse_mqa_plain_lane_wiring(monkeypatch):
[
pytest.param(0, True, False, id="resident"),
pytest.param(0, False, False, id="staged"),
pytest.param(2, True, True, id="mixed"),
pytest.param(2, True, True, id="mixed_resident"),
pytest.param(2, False, True, id="mixed_staged"),
pytest.param(5, True, False, id="decode"),
],
)
def test_sparse_mqa_hisparse_lanes(
monkeypatch, num_decode_tokens, all_resident, split_q
):
"""HiSparse splits a batch into at most one decode and one prefill call."""
"""HiSparse runs all-resident and decode-only batches in one call against the
hot cache, and otherwise splits into one decode and one staged-prefill call."""
import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla
num_tokens, topk_tokens, block_size, head_dim = 5, 8, 4, 6
@@ -3814,9 +3861,9 @@ def test_sparse_mqa_hisparse_lanes(
)
# The fill value names the converter that produced a call's indices.
index_group.convert_logical_to_physical_topk = MagicMock(
return_value=converted(num_decode_tokens, 1)
if num_decode_tokens
else converted(num_tokens, 2)
side_effect=lambda layer, topk, *args, **kwargs: converted(
topk.shape[0], 1 if topk.shape[0] == num_decode_tokens else 2
)
)
index_group.stage_prefill_rows = MagicMock(return_value=(staged, None, None))
calls: list[tuple] = []
@@ -3850,36 +3897,41 @@ def test_sparse_mqa_hisparse_lanes(
q_rows = q.flatten().tolist()
expected = []
if num_decode_tokens:
# Decode reads the hot cache through the group's conversion.
expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1))
if num_decode_tokens == 0 and all_resident:
if not num_prefill:
expected.append((q_rows, hot.shape[0], 1))
elif all_resident:
# Decode and prefill rows read the hot cache in one call.
expected.append((q_rows, hot.shape[0], 2))
elif num_prefill:
else:
if num_decode_tokens:
expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1))
expected.append((q_rows[num_decode_tokens:], staged.shape[0], 3))
assert calls == expected
if not num_prefill:
# A decode-only batch returns before it looks up the prefill cache.
index_group.cache.assert_not_called()
assert lse is None
# A lane handed the whole batch would concatenate to more rows than this.
assert out.flatten().tolist() == [1.0] * num_decode_tokens + [2.0] * num_prefill
num_decode_rows = num_tokens if not num_prefill else 0
if num_prefill and not all_resident:
num_decode_rows = num_decode_tokens
assert out.flatten().tolist() == [1.0] * num_decode_rows + [2.0] * (
num_tokens - num_decode_rows
)
def test_hisparse_resident_prefill_uses_attention_block_stride():
"""A batch beyond the swap-row capacity must bypass swap_in.
The index-group workspace has one more row than the HiSparse swap rows, so
routing on its size sent a 2-token batch with max_num_seqs=1 (a piecewise
CUDA-graph capture) into swap_in, which overflowed its capacity of 1.
"""
@pytest.mark.parametrize("max_swap_rows", [1, 4], ids=["over_capacity", "mixed"])
def test_hisparse_resident_prefill_uses_attention_block_stride(max_swap_rows):
"""Resident batches with prefill rows read resident pages directly, whether
they exceed the swap-row capacity or fit it."""
expected = torch.tensor([[19]], dtype=torch.int32)
cache_handle = SimpleNamespace(
all_context_pages_resident=True,
num_decode_tokens=1,
num_actual_tokens=2,
view=SimpleNamespace(block_size=64, attention_block_stride=832),
block_table=torch.tensor([[3]], dtype=torch.int32),
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
runtime=SimpleNamespace(max_swap_rows=1),
runtime=SimpleNamespace(max_swap_rows=max_swap_rows),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache_handle]