[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
+52 -11
View File
@@ -3241,7 +3241,8 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
[[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]], dtype=torch.int32
)
plan.ensure_gpu_sources(resident_table, resident_block_size)
state_indices = torch.arange(2, dtype=torch.int32)
plan.ensure_gpu_sources(resident_table, state_indices, resident_block_size)
assert plan.gpu_row_ids is not None
unique_hosts = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
@@ -3271,12 +3272,51 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
check_dtype=False,
)
plan.ensure_gpu_sources(torch.zeros_like(resident_table), resident_block_size)
plan.ensure_gpu_sources(
torch.zeros_like(resident_table), state_indices, resident_block_size
)
assert plan.gpu_row_ids is not None
assert (plan.gpu_row_ids == -1).all()
def test_hisparse_prefill_resident_sources_follow_each_resident_group():
"""Layers of different resident groups share one batch staging plan.
Each group's layers must stage from that group's resident blocks. Keying
the plan's cached resident sources on a per-call batch gather let a later
group's gather reuse the freed address of an earlier one, so its layers
read the earlier group's block ids.
"""
block_size = 4
plan = build_hisparse_prefill_staging_plan(
torch.tensor([[5, 2]], dtype=torch.int32),
torch.tensor([8], dtype=torch.int32),
block_size,
staging_block_capacity=2,
)
# The prefill runs in request state row 1; one table holds both groups.
# Rows are wide so that a cache keyed on a freed per-call gather of a row
# would see the next gather reuse its address.
state_rows = torch.zeros((4, 2, 1024), dtype=torch.int32)
state_rows[1, 0, :2] = torch.tensor([11, 12])
state_rows[1, 1, :2] = torch.tensor([21, 22])
state_indices = torch.tensor([1], dtype=torch.int32)
host_ids = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
for group in (0, 1, 0, 1):
plan.ensure_gpu_sources(state_rows[:, group], state_indices, block_size)
assert plan.gpu_row_ids is not None
gpu_blocks = plan.gpu_row_ids[0].view(-1, block_size)[:, 0] // block_size
staged = {
host_id: gpu_block
for host_id, gpu_block in zip(host_ids, gpu_blocks.tolist())
if host_id > 0
}
first_page, second_page = state_rows[1, group, :2].tolist()
assert staged == {5: first_page, 2: second_page}
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_hisparse_gather_prefill_cache_prefers_resident_rows():
"""Staged rows come from the resident cache when a shadow page exists."""
@@ -3294,7 +3334,11 @@ def test_hisparse_gather_prefill_cache_prefers_resident_rows():
dtype=torch.int32,
device=device,
)
plan.ensure_gpu_sources(resident_table, resident_block_size)
plan.ensure_gpu_sources(
resident_table,
torch.arange(2, dtype=torch.int32, device=device),
resident_block_size,
)
num_host_blocks, num_res_blocks = 10, 24
host_cache = (
@@ -3677,6 +3721,7 @@ def test_hisparse_resident_prefill_uses_attention_block_stride():
all_context_pages_resident=True,
view=SimpleNamespace(block_size=64, attention_block_stride=832),
block_table=torch.tensor([[3]], dtype=torch.int32),
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
runtime=SimpleNamespace(max_swap_rows=1),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
@@ -4091,6 +4136,7 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
cache = SimpleNamespace(
runtime=SimpleNamespace(
gather_prefill_cache=gather,
request_state_indices=torch.tensor([0], dtype=torch.int32),
),
view=SimpleNamespace(cache=resident_cache, block_size=1),
block_table=resident_block_table,
@@ -4113,10 +4159,6 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
assert result is staged
assert block_table is plan.block_table
torch.testing.assert_close(request_ids, metadata.req_id_per_token)
plan.ensure_gpu_sources.assert_called_once()
args = plan.ensure_gpu_sources.call_args.args
torch.testing.assert_close(args[0], resident_block_table)
assert args[1] == 1
assert calls == [(source, plan, resident_cache)]
@@ -4134,6 +4176,9 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
cache = SimpleNamespace(
view=SimpleNamespace(cache=resident_cache, block_size=4),
block_table=torch.tensor([[2, 3]], dtype=torch.int32),
runtime=SimpleNamespace(
request_state_indices=torch.tensor([0], dtype=torch.int32)
),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache]
@@ -4178,10 +4223,6 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
assert args.kwargs["host_cache"].data_ptr() == source.data_ptr()
assert args.kwargs["host_row_ids"] is plan.row_ids
assert args.kwargs["device_row_ids"] is plan.gpu_row_ids
plan.ensure_gpu_sources.assert_called_once()
ensure_args = plan.ensure_gpu_sources.call_args.args
torch.testing.assert_close(ensure_args[0], cache.block_table)
assert ensure_args[1] == 4
def test_sparse_impl_observes_repointed_indexer_buffer():