[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -3241,7 +3241,8 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
|
||||
[[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]], dtype=torch.int32
|
||||
)
|
||||
|
||||
plan.ensure_gpu_sources(resident_table, resident_block_size)
|
||||
state_indices = torch.arange(2, dtype=torch.int32)
|
||||
plan.ensure_gpu_sources(resident_table, state_indices, resident_block_size)
|
||||
|
||||
assert plan.gpu_row_ids is not None
|
||||
unique_hosts = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
|
||||
@@ -3271,12 +3272,51 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
|
||||
check_dtype=False,
|
||||
)
|
||||
|
||||
plan.ensure_gpu_sources(torch.zeros_like(resident_table), resident_block_size)
|
||||
plan.ensure_gpu_sources(
|
||||
torch.zeros_like(resident_table), state_indices, resident_block_size
|
||||
)
|
||||
|
||||
assert plan.gpu_row_ids is not None
|
||||
assert (plan.gpu_row_ids == -1).all()
|
||||
|
||||
|
||||
def test_hisparse_prefill_resident_sources_follow_each_resident_group():
|
||||
"""Layers of different resident groups share one batch staging plan.
|
||||
|
||||
Each group's layers must stage from that group's resident blocks. Keying
|
||||
the plan's cached resident sources on a per-call batch gather let a later
|
||||
group's gather reuse the freed address of an earlier one, so its layers
|
||||
read the earlier group's block ids.
|
||||
"""
|
||||
block_size = 4
|
||||
plan = build_hisparse_prefill_staging_plan(
|
||||
torch.tensor([[5, 2]], dtype=torch.int32),
|
||||
torch.tensor([8], dtype=torch.int32),
|
||||
block_size,
|
||||
staging_block_capacity=2,
|
||||
)
|
||||
# The prefill runs in request state row 1; one table holds both groups.
|
||||
# Rows are wide so that a cache keyed on a freed per-call gather of a row
|
||||
# would see the next gather reuse its address.
|
||||
state_rows = torch.zeros((4, 2, 1024), dtype=torch.int32)
|
||||
state_rows[1, 0, :2] = torch.tensor([11, 12])
|
||||
state_rows[1, 1, :2] = torch.tensor([21, 22])
|
||||
state_indices = torch.tensor([1], dtype=torch.int32)
|
||||
host_ids = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
|
||||
|
||||
for group in (0, 1, 0, 1):
|
||||
plan.ensure_gpu_sources(state_rows[:, group], state_indices, block_size)
|
||||
assert plan.gpu_row_ids is not None
|
||||
gpu_blocks = plan.gpu_row_ids[0].view(-1, block_size)[:, 0] // block_size
|
||||
staged = {
|
||||
host_id: gpu_block
|
||||
for host_id, gpu_block in zip(host_ids, gpu_blocks.tolist())
|
||||
if host_id > 0
|
||||
}
|
||||
first_page, second_page = state_rows[1, group, :2].tolist()
|
||||
assert staged == {5: first_page, 2: second_page}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_hisparse_gather_prefill_cache_prefers_resident_rows():
|
||||
"""Staged rows come from the resident cache when a shadow page exists."""
|
||||
@@ -3294,7 +3334,11 @@ def test_hisparse_gather_prefill_cache_prefers_resident_rows():
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
plan.ensure_gpu_sources(resident_table, resident_block_size)
|
||||
plan.ensure_gpu_sources(
|
||||
resident_table,
|
||||
torch.arange(2, dtype=torch.int32, device=device),
|
||||
resident_block_size,
|
||||
)
|
||||
|
||||
num_host_blocks, num_res_blocks = 10, 24
|
||||
host_cache = (
|
||||
@@ -3677,6 +3721,7 @@ def test_hisparse_resident_prefill_uses_attention_block_stride():
|
||||
all_context_pages_resident=True,
|
||||
view=SimpleNamespace(block_size=64, attention_block_stride=832),
|
||||
block_table=torch.tensor([[3]], dtype=torch.int32),
|
||||
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
|
||||
runtime=SimpleNamespace(max_swap_rows=1),
|
||||
)
|
||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||
@@ -4091,6 +4136,7 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
|
||||
cache = SimpleNamespace(
|
||||
runtime=SimpleNamespace(
|
||||
gather_prefill_cache=gather,
|
||||
request_state_indices=torch.tensor([0], dtype=torch.int32),
|
||||
),
|
||||
view=SimpleNamespace(cache=resident_cache, block_size=1),
|
||||
block_table=resident_block_table,
|
||||
@@ -4113,10 +4159,6 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
|
||||
assert result is staged
|
||||
assert block_table is plan.block_table
|
||||
torch.testing.assert_close(request_ids, metadata.req_id_per_token)
|
||||
plan.ensure_gpu_sources.assert_called_once()
|
||||
args = plan.ensure_gpu_sources.call_args.args
|
||||
torch.testing.assert_close(args[0], resident_block_table)
|
||||
assert args[1] == 1
|
||||
assert calls == [(source, plan, resident_cache)]
|
||||
|
||||
|
||||
@@ -4134,6 +4176,9 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
|
||||
cache = SimpleNamespace(
|
||||
view=SimpleNamespace(cache=resident_cache, block_size=4),
|
||||
block_table=torch.tensor([[2, 3]], dtype=torch.int32),
|
||||
runtime=SimpleNamespace(
|
||||
request_state_indices=torch.tensor([0], dtype=torch.int32)
|
||||
),
|
||||
)
|
||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||
index_group.caches = [cache]
|
||||
@@ -4178,10 +4223,6 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
|
||||
assert args.kwargs["host_cache"].data_ptr() == source.data_ptr()
|
||||
assert args.kwargs["host_row_ids"] is plan.row_ids
|
||||
assert args.kwargs["device_row_ids"] is plan.gpu_row_ids
|
||||
plan.ensure_gpu_sources.assert_called_once()
|
||||
ensure_args = plan.ensure_gpu_sources.call_args.args
|
||||
torch.testing.assert_close(ensure_args[0], cache.block_table)
|
||||
assert ensure_args[1] == 4
|
||||
|
||||
|
||||
def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
|
||||
Reference in New Issue
Block a user