[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
+52 -11
View File
@@ -3241,7 +3241,8 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
[[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]], dtype=torch.int32
)
plan.ensure_gpu_sources(resident_table, resident_block_size)
state_indices = torch.arange(2, dtype=torch.int32)
plan.ensure_gpu_sources(resident_table, state_indices, resident_block_size)
assert plan.gpu_row_ids is not None
unique_hosts = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
@@ -3271,12 +3272,51 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
check_dtype=False,
)
plan.ensure_gpu_sources(torch.zeros_like(resident_table), resident_block_size)
plan.ensure_gpu_sources(
torch.zeros_like(resident_table), state_indices, resident_block_size
)
assert plan.gpu_row_ids is not None
assert (plan.gpu_row_ids == -1).all()
def test_hisparse_prefill_resident_sources_follow_each_resident_group():
"""Layers of different resident groups share one batch staging plan.
Each group's layers must stage from that group's resident blocks. Keying
the plan's cached resident sources on a per-call batch gather let a later
group's gather reuse the freed address of an earlier one, so its layers
read the earlier group's block ids.
"""
block_size = 4
plan = build_hisparse_prefill_staging_plan(
torch.tensor([[5, 2]], dtype=torch.int32),
torch.tensor([8], dtype=torch.int32),
block_size,
staging_block_capacity=2,
)
# The prefill runs in request state row 1; one table holds both groups.
# Rows are wide so that a cache keyed on a freed per-call gather of a row
# would see the next gather reuse its address.
state_rows = torch.zeros((4, 2, 1024), dtype=torch.int32)
state_rows[1, 0, :2] = torch.tensor([11, 12])
state_rows[1, 1, :2] = torch.tensor([21, 22])
state_indices = torch.tensor([1], dtype=torch.int32)
host_ids = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
for group in (0, 1, 0, 1):
plan.ensure_gpu_sources(state_rows[:, group], state_indices, block_size)
assert plan.gpu_row_ids is not None
gpu_blocks = plan.gpu_row_ids[0].view(-1, block_size)[:, 0] // block_size
staged = {
host_id: gpu_block
for host_id, gpu_block in zip(host_ids, gpu_blocks.tolist())
if host_id > 0
}
first_page, second_page = state_rows[1, group, :2].tolist()
assert staged == {5: first_page, 2: second_page}
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_hisparse_gather_prefill_cache_prefers_resident_rows():
"""Staged rows come from the resident cache when a shadow page exists."""
@@ -3294,7 +3334,11 @@ def test_hisparse_gather_prefill_cache_prefers_resident_rows():
dtype=torch.int32,
device=device,
)
plan.ensure_gpu_sources(resident_table, resident_block_size)
plan.ensure_gpu_sources(
resident_table,
torch.arange(2, dtype=torch.int32, device=device),
resident_block_size,
)
num_host_blocks, num_res_blocks = 10, 24
host_cache = (
@@ -3677,6 +3721,7 @@ def test_hisparse_resident_prefill_uses_attention_block_stride():
all_context_pages_resident=True,
view=SimpleNamespace(block_size=64, attention_block_stride=832),
block_table=torch.tensor([[3]], dtype=torch.int32),
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
runtime=SimpleNamespace(max_swap_rows=1),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
@@ -4091,6 +4136,7 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
cache = SimpleNamespace(
runtime=SimpleNamespace(
gather_prefill_cache=gather,
request_state_indices=torch.tensor([0], dtype=torch.int32),
),
view=SimpleNamespace(cache=resident_cache, block_size=1),
block_table=resident_block_table,
@@ -4113,10 +4159,6 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
assert result is staged
assert block_table is plan.block_table
torch.testing.assert_close(request_ids, metadata.req_id_per_token)
plan.ensure_gpu_sources.assert_called_once()
args = plan.ensure_gpu_sources.call_args.args
torch.testing.assert_close(args[0], resident_block_table)
assert args[1] == 1
assert calls == [(source, plan, resident_cache)]
@@ -4134,6 +4176,9 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
cache = SimpleNamespace(
view=SimpleNamespace(cache=resident_cache, block_size=4),
block_table=torch.tensor([[2, 3]], dtype=torch.int32),
runtime=SimpleNamespace(
request_state_indices=torch.tensor([0], dtype=torch.int32)
),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache]
@@ -4178,10 +4223,6 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
assert args.kwargs["host_cache"].data_ptr() == source.data_ptr()
assert args.kwargs["host_row_ids"] is plan.row_ids
assert args.kwargs["device_row_ids"] is plan.gpu_row_ids
plan.ensure_gpu_sources.assert_called_once()
ensure_args = plan.ensure_gpu_sources.call_args.args
torch.testing.assert_close(ensure_args[0], cache.block_table)
assert ensure_args[1] == 4
def test_sparse_impl_observes_repointed_indexer_buffer():
+72 -22
View File
@@ -279,6 +279,12 @@ def _allocate_scheduled(
return blocks
def _resident_block_ids(manager: KVCacheManager, request_id: str) -> list[int]:
"""The GPU block HiSparse reads each resident page from; 0 if host-only."""
resident = get_hisparse_coordinator(manager).resident_managers[0]
return [block.block_id for block in resident.get_residency_row(request_id)]
def test_hisparse_does_not_write_back_reprefillable_tokens():
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
@@ -319,7 +325,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
def test_hisparse_async_speculation_mirrors_uncertain_position_range():
"""Unresolved drafts must not leave gaps in the eager host mirror."""
coordinator = MagicMock(host_group_id=0)
coordinator.take_block_table_updates.return_value = {}
coordinator.take_residency_updates.return_value = {}
coordinator.build_offload_command.return_value = None
coordinator.build_row_mirrors.return_value = ()
scheduler = HiSparseConnectorScheduler(
@@ -332,7 +338,6 @@ def test_hisparse_async_speculation_mirrors_uncertain_position_range():
connector.connector_scheduler = scheduler
connector.update_state_after_alloc(request, None, 0)
scheduler_output = SimpleNamespace(
block_table_updates=None,
kv_cache_block_copies=None,
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
@@ -370,11 +375,66 @@ def test_hisparse_reports_when_context_is_fully_resident():
hot_manager.require_hot(request.request_id)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
_publish_hisparse_pages(manager)
coordinator.take_residency_updates([request.request_id])
core_row = manager.get_block_ids(request.request_id)[2]
pool = manager.block_pool
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
assert coordinator.take_block_table_updates().keys() == {request.request_id}
# The block-table row stays append-only; the worker learns of the lost
# pages from the residency update instead.
assert manager.get_block_ids(request.request_id)[2] == core_row
resident_ids = _resident_block_ids(manager, request.request_id)
lost = [page for page, block_id in enumerate(resident_ids) if block_id == 0]
assert lost
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == lost
assert update[request.request_id].block_ids[0] == [0] * len(lost)
def test_hisparse_residency_update_sends_only_changed_pages():
"""Losing one early page must not resend the pages after it: per-step
residency updates would otherwise grow with the context length."""
manager = make_hisparse_kv_cache_manager(32, 16)
tokens = list(range(6 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
coordinator = get_hisparse_coordinator(manager)
for hot_manager in coordinator.hot_managers:
hot_manager.require_hot(request.request_id)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
coordinator.take_residency_updates([request.request_id])
pool = manager.block_pool
num_never_used = pool.get_num_free_blocks()
# Pages unpin as their host copies complete, so page 0 is reused first.
_publish_hisparse_pages(manager)
pool.get_new_blocks(num_never_used + 1)
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == [0]
assert update[request.request_id].block_ids == ([0],)
def test_hisparse_readmitted_request_resends_its_full_residency():
"""A preempted request may resume in a worker state row holding another
request's residency, so its first update must cover every page again."""
manager = make_hisparse_kv_cache_manager(32, 16)
coordinator = get_hisparse_coordinator(manager)
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
assert request.request_id in coordinator.take_residency_updates(
[request.request_id]
)
manager.free(request)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == [0, 1]
assert update[request.request_id].block_ids[0] == _resident_block_ids(
manager, request.request_id
)
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
@@ -416,9 +476,7 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
assert len(source) == len(indexer) == len(resident) == 4
assert len(hot) == 2
assert not any(block.is_null for block in resident[:2])
assert not resident[2].is_null
assert not resident[3].is_null
assert 0 not in _resident_block_ids(manager, resumed.request_id)
coordinator = get_hisparse_coordinator(manager)
assert not coordinator.build_offload_command().page_transfers
coordinator.finish_host_import(resumed.request_id, failed=False)
@@ -666,9 +724,10 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
pool.free_blocks(held)
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
assert all(m.has_hot("first") for m in coordinator.hot_managers)
coordinator.take_residency_updates(["first"])
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
assert "first" in coordinator.take_block_table_updates()
assert "first" in coordinator.take_residency_updates(["first"])
def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
@@ -1204,11 +1263,7 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resident_blocks = manager.get_blocks("resumed").blocks[2]
assert [block.block_id for block in resident_blocks[:3]] == (
original_resident_ids[:3]
)
assert not any(block.is_null for block in resident_blocks[:3])
assert _resident_block_ids(manager, "resumed")[:3] == original_resident_ids[:3]
assert get_hisparse_coordinator(manager).all_context_pages_resident(
((resumed.request_id, num_computed, len(tokens) - num_computed),)
)
@@ -1249,17 +1304,13 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
new_computed_blocks=computed,
)
resumed_blocks = manager.get_blocks("resumed").blocks
assert [block.block_id for block in resumed_blocks[2][:3]] == [
resident_ids = _resident_block_ids(manager, "resumed")
assert resident_ids[:3] == [
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
for page in range(3)
]
gpu_ids = [
block.block_id
for group in resumed_blocks[1:]
for block in group
if not block.is_null
]
_, indexer, _, hot = manager.get_block_ids("resumed")
gpu_ids = [block_id for block_id in indexer + resident_ids + hot if block_id]
assert len(gpu_ids) == len(set(gpu_ids))
@@ -1488,8 +1539,7 @@ def test_hisparse_reused_copy_is_not_adopted():
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resident_blocks = manager.get_blocks("resumed").blocks[2]
assert all(block.is_null for block in resident_blocks[:3])
assert _resident_block_ids(manager, "resumed")[:3] == [0, 0, 0]
def test_hisparse_external_import_uses_hard_gpu_footprint():
@@ -28,10 +28,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiCo
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.hisparse import runtime as runtime_module
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
update_hisparse_residency,
)
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVPageTransfer,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.metrics.stats import KVCacheEvictionEvent
@@ -183,7 +187,7 @@ def test_full_graph_step_prepares_host_mirror_outside_model():
connector = object.__new__(HiSparseConnector)
connector.connector_worker = worker
connector._get_connector_metadata = MagicMock(
return_value=HiSparseConnectorMetadata(None, (), (), {}, True)
return_value=HiSparseConnectorMetadata(None, (), (), {}, True, {})
)
req_id_per_token = torch.tensor([0, 1], dtype=torch.int32)
attn_metadata = SimpleNamespace(
@@ -321,6 +325,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
)
},
True,
{},
)
)
# A verification step: four query tokens of one request fill the page.
@@ -354,7 +359,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
# The next step's start mirrors the drafter's rows, then hands the page over.
compute_stream.wait_event.reset_mock()
connector._get_connector_metadata.return_value = HiSparseConnectorMetadata(
None, (), (), {}, True
None, (), (), {}, True, {}
)
connector.start_load_kv(
SimpleNamespace(),
@@ -397,7 +402,8 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
def test_scheduled_prefix_hit_publishes_adopted_copies():
"""Copies adopted after scheduling must reach the worker's block table."""
"""Copies adopted after scheduling reach the worker as a residency update,
leaving the block-table row the scheduler output carries untouched."""
from tests.v1.core.test_prefix_caching import (
HISPARSE_BLOCK_SIZE,
_allocate_scheduled,
@@ -443,7 +449,39 @@ def test_scheduled_prefix_hit_publishes_adopted_copies():
num_scheduled_tokens={resumed.request_id: num_new_tokens},
)
scheduler.build_connector_meta(scheduler_output)
core_row = list(scheduler_output.scheduled_new_reqs[0].block_ids[2])
resident_ids = scheduler_output.block_table_updates[resumed.request_id][2]
assert resident_ids[:3] == copy_ids[:3]
metadata = scheduler.build_connector_meta(scheduler_output)
update = metadata.residency_updates[resumed.request_id]
assert update.pages == [0, 1, 2, 3]
assert update.block_ids[0][:3] == copy_ids[:3]
assert manager.get_block_ids(resumed.request_id)[2] == core_row
assert core_row[:3] == [0, 0, 0]
def test_residency_updates_persist_by_state_row():
"""Updates name requests by batch row but are stored by state row.
An update for a lost page and an appended page must leave the other pages
and the other request's row intact across steps that reorder the batch.
"""
table = torch.zeros((4, 2, 4), dtype=torch.int32)
update_hisparse_residency(
table,
{
"a": SparseKVResidencyUpdate([0, 1, 2], ([1, 2, 3], [11, 12, 13])),
"b": SparseKVResidencyUpdate([0, 1], ([4, 5], [14, 15])),
},
["a", "b"],
torch.tensor([2, 0], dtype=torch.int32),
)
update_hisparse_residency(
table,
{"a": SparseKVResidencyUpdate([1, 3], ([0, 6], [0, 16]))},
["b", "a"],
torch.tensor([0, 2], dtype=torch.int32),
)
assert table[[0, 2], 0].tolist() == [[4, 5, 0, 0], [1, 0, 3, 6]]
assert table[[0, 2], 1].tolist() == [[14, 15, 0, 0], [11, 0, 13, 16]]
+3
View File
@@ -1068,6 +1068,7 @@ def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
residency_updates={},
),
None,
)
@@ -1079,6 +1080,7 @@ def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
residency_updates={},
),
None,
)
@@ -1152,6 +1154,7 @@ def test_hisparse_empty_step_does_not_replay_stale_host_mirror(monkeypatch):
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
residency_updates={},
),
None,
)