[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -3241,7 +3241,8 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
|
||||
[[11, 12, 0, 13, 0, 0], [21, 0, 22, 23, 0, 0]], dtype=torch.int32
|
||||
)
|
||||
|
||||
plan.ensure_gpu_sources(resident_table, resident_block_size)
|
||||
state_indices = torch.arange(2, dtype=torch.int32)
|
||||
plan.ensure_gpu_sources(resident_table, state_indices, resident_block_size)
|
||||
|
||||
assert plan.gpu_row_ids is not None
|
||||
unique_hosts = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
|
||||
@@ -3271,12 +3272,51 @@ def test_hisparse_prefill_staging_plan_resolves_resident_sources():
|
||||
check_dtype=False,
|
||||
)
|
||||
|
||||
plan.ensure_gpu_sources(torch.zeros_like(resident_table), resident_block_size)
|
||||
plan.ensure_gpu_sources(
|
||||
torch.zeros_like(resident_table), state_indices, resident_block_size
|
||||
)
|
||||
|
||||
assert plan.gpu_row_ids is not None
|
||||
assert (plan.gpu_row_ids == -1).all()
|
||||
|
||||
|
||||
def test_hisparse_prefill_resident_sources_follow_each_resident_group():
|
||||
"""Layers of different resident groups share one batch staging plan.
|
||||
|
||||
Each group's layers must stage from that group's resident blocks. Keying
|
||||
the plan's cached resident sources on a per-call batch gather let a later
|
||||
group's gather reuse the freed address of an earlier one, so its layers
|
||||
read the earlier group's block ids.
|
||||
"""
|
||||
block_size = 4
|
||||
plan = build_hisparse_prefill_staging_plan(
|
||||
torch.tensor([[5, 2]], dtype=torch.int32),
|
||||
torch.tensor([8], dtype=torch.int32),
|
||||
block_size,
|
||||
staging_block_capacity=2,
|
||||
)
|
||||
# The prefill runs in request state row 1; one table holds both groups.
|
||||
# Rows are wide so that a cache keyed on a freed per-call gather of a row
|
||||
# would see the next gather reuse its address.
|
||||
state_rows = torch.zeros((4, 2, 1024), dtype=torch.int32)
|
||||
state_rows[1, 0, :2] = torch.tensor([11, 12])
|
||||
state_rows[1, 1, :2] = torch.tensor([21, 22])
|
||||
state_indices = torch.tensor([1], dtype=torch.int32)
|
||||
host_ids = (plan.row_ids[0].view(-1, block_size)[:, 0] // block_size).tolist()
|
||||
|
||||
for group in (0, 1, 0, 1):
|
||||
plan.ensure_gpu_sources(state_rows[:, group], state_indices, block_size)
|
||||
assert plan.gpu_row_ids is not None
|
||||
gpu_blocks = plan.gpu_row_ids[0].view(-1, block_size)[:, 0] // block_size
|
||||
staged = {
|
||||
host_id: gpu_block
|
||||
for host_id, gpu_block in zip(host_ids, gpu_blocks.tolist())
|
||||
if host_id > 0
|
||||
}
|
||||
first_page, second_page = state_rows[1, group, :2].tolist()
|
||||
assert staged == {5: first_page, 2: second_page}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_hisparse_gather_prefill_cache_prefers_resident_rows():
|
||||
"""Staged rows come from the resident cache when a shadow page exists."""
|
||||
@@ -3294,7 +3334,11 @@ def test_hisparse_gather_prefill_cache_prefers_resident_rows():
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
plan.ensure_gpu_sources(resident_table, resident_block_size)
|
||||
plan.ensure_gpu_sources(
|
||||
resident_table,
|
||||
torch.arange(2, dtype=torch.int32, device=device),
|
||||
resident_block_size,
|
||||
)
|
||||
|
||||
num_host_blocks, num_res_blocks = 10, 24
|
||||
host_cache = (
|
||||
@@ -3677,6 +3721,7 @@ def test_hisparse_resident_prefill_uses_attention_block_stride():
|
||||
all_context_pages_resident=True,
|
||||
view=SimpleNamespace(block_size=64, attention_block_stride=832),
|
||||
block_table=torch.tensor([[3]], dtype=torch.int32),
|
||||
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
|
||||
runtime=SimpleNamespace(max_swap_rows=1),
|
||||
)
|
||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||
@@ -4091,6 +4136,7 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
|
||||
cache = SimpleNamespace(
|
||||
runtime=SimpleNamespace(
|
||||
gather_prefill_cache=gather,
|
||||
request_state_indices=torch.tensor([0], dtype=torch.int32),
|
||||
),
|
||||
view=SimpleNamespace(cache=resident_cache, block_size=1),
|
||||
block_table=resident_block_table,
|
||||
@@ -4113,10 +4159,6 @@ def test_hisparse_prefill_reuses_builder_staging_plan():
|
||||
assert result is staged
|
||||
assert block_table is plan.block_table
|
||||
torch.testing.assert_close(request_ids, metadata.req_id_per_token)
|
||||
plan.ensure_gpu_sources.assert_called_once()
|
||||
args = plan.ensure_gpu_sources.call_args.args
|
||||
torch.testing.assert_close(args[0], resident_block_table)
|
||||
assert args[1] == 1
|
||||
assert calls == [(source, plan, resident_cache)]
|
||||
|
||||
|
||||
@@ -4134,6 +4176,9 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
|
||||
cache = SimpleNamespace(
|
||||
view=SimpleNamespace(cache=resident_cache, block_size=4),
|
||||
block_table=torch.tensor([[2, 3]], dtype=torch.int32),
|
||||
runtime=SimpleNamespace(
|
||||
request_state_indices=torch.tensor([0], dtype=torch.int32)
|
||||
),
|
||||
)
|
||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||
index_group.caches = [cache]
|
||||
@@ -4178,10 +4223,6 @@ def test_hisparse_fp8_prefill_gather_uses_dedicated_stream(monkeypatch):
|
||||
assert args.kwargs["host_cache"].data_ptr() == source.data_ptr()
|
||||
assert args.kwargs["host_row_ids"] is plan.row_ids
|
||||
assert args.kwargs["device_row_ids"] is plan.gpu_row_ids
|
||||
plan.ensure_gpu_sources.assert_called_once()
|
||||
ensure_args = plan.ensure_gpu_sources.call_args.args
|
||||
torch.testing.assert_close(ensure_args[0], cache.block_table)
|
||||
assert ensure_args[1] == 4
|
||||
|
||||
|
||||
def test_sparse_impl_observes_repointed_indexer_buffer():
|
||||
|
||||
@@ -279,6 +279,12 @@ def _allocate_scheduled(
|
||||
return blocks
|
||||
|
||||
|
||||
def _resident_block_ids(manager: KVCacheManager, request_id: str) -> list[int]:
|
||||
"""The GPU block HiSparse reads each resident page from; 0 if host-only."""
|
||||
resident = get_hisparse_coordinator(manager).resident_managers[0]
|
||||
return [block.block_id for block in resident.get_residency_row(request_id)]
|
||||
|
||||
|
||||
def test_hisparse_does_not_write_back_reprefillable_tokens():
|
||||
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
|
||||
@@ -319,7 +325,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
|
||||
def test_hisparse_async_speculation_mirrors_uncertain_position_range():
|
||||
"""Unresolved drafts must not leave gaps in the eager host mirror."""
|
||||
coordinator = MagicMock(host_group_id=0)
|
||||
coordinator.take_block_table_updates.return_value = {}
|
||||
coordinator.take_residency_updates.return_value = {}
|
||||
coordinator.build_offload_command.return_value = None
|
||||
coordinator.build_row_mirrors.return_value = ()
|
||||
scheduler = HiSparseConnectorScheduler(
|
||||
@@ -332,7 +338,6 @@ def test_hisparse_async_speculation_mirrors_uncertain_position_range():
|
||||
connector.connector_scheduler = scheduler
|
||||
connector.update_state_after_alloc(request, None, 0)
|
||||
scheduler_output = SimpleNamespace(
|
||||
block_table_updates=None,
|
||||
kv_cache_block_copies=None,
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
@@ -370,11 +375,66 @@ def test_hisparse_reports_when_context_is_fully_resident():
|
||||
hot_manager.require_hot(request.request_id)
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
coordinator.take_residency_updates([request.request_id])
|
||||
core_row = manager.get_block_ids(request.request_id)[2]
|
||||
pool = manager.block_pool
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
|
||||
assert not coordinator.all_context_pages_resident(scheduled)
|
||||
assert coordinator.take_block_table_updates().keys() == {request.request_id}
|
||||
# The block-table row stays append-only; the worker learns of the lost
|
||||
# pages from the residency update instead.
|
||||
assert manager.get_block_ids(request.request_id)[2] == core_row
|
||||
resident_ids = _resident_block_ids(manager, request.request_id)
|
||||
lost = [page for page, block_id in enumerate(resident_ids) if block_id == 0]
|
||||
assert lost
|
||||
update = coordinator.take_residency_updates([request.request_id])
|
||||
assert update[request.request_id].pages == lost
|
||||
assert update[request.request_id].block_ids[0] == [0] * len(lost)
|
||||
|
||||
|
||||
def test_hisparse_residency_update_sends_only_changed_pages():
|
||||
"""Losing one early page must not resend the pages after it: per-step
|
||||
residency updates would otherwise grow with the context length."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16)
|
||||
tokens = list(range(6 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
for hot_manager in coordinator.hot_managers:
|
||||
hot_manager.require_hot(request.request_id)
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
|
||||
coordinator.take_residency_updates([request.request_id])
|
||||
pool = manager.block_pool
|
||||
num_never_used = pool.get_num_free_blocks()
|
||||
# Pages unpin as their host copies complete, so page 0 is reused first.
|
||||
_publish_hisparse_pages(manager)
|
||||
pool.get_new_blocks(num_never_used + 1)
|
||||
|
||||
update = coordinator.take_residency_updates([request.request_id])
|
||||
|
||||
assert update[request.request_id].pages == [0]
|
||||
assert update[request.request_id].block_ids == ([0],)
|
||||
|
||||
|
||||
def test_hisparse_readmitted_request_resends_its_full_residency():
|
||||
"""A preempted request may resume in a worker state row holding another
|
||||
request's residency, so its first update must cover every page again."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, request, len(tokens)) is not None
|
||||
assert request.request_id in coordinator.take_residency_updates(
|
||||
[request.request_id]
|
||||
)
|
||||
manager.free(request)
|
||||
|
||||
assert _allocate_scheduled(manager, request, len(tokens)) is not None
|
||||
update = coordinator.take_residency_updates([request.request_id])
|
||||
assert update[request.request_id].pages == [0, 1]
|
||||
assert update[request.request_id].block_ids[0] == _resident_block_ids(
|
||||
manager, request.request_id
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
@@ -416,9 +476,7 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
|
||||
assert len(source) == len(indexer) == len(resident) == 4
|
||||
assert len(hot) == 2
|
||||
assert not any(block.is_null for block in resident[:2])
|
||||
assert not resident[2].is_null
|
||||
assert not resident[3].is_null
|
||||
assert 0 not in _resident_block_ids(manager, resumed.request_id)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert not coordinator.build_offload_command().page_transfers
|
||||
coordinator.finish_host_import(resumed.request_id, failed=False)
|
||||
@@ -666,9 +724,10 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
|
||||
pool.free_blocks(held)
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
|
||||
assert all(m.has_hot("first") for m in coordinator.hot_managers)
|
||||
coordinator.take_residency_updates(["first"])
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
assert not coordinator.all_context_pages_resident(scheduled)
|
||||
assert "first" in coordinator.take_block_table_updates()
|
||||
assert "first" in coordinator.take_residency_updates(["first"])
|
||||
|
||||
|
||||
def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
|
||||
@@ -1204,11 +1263,7 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
|
||||
num_new_computed_tokens=num_computed,
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
resident_blocks = manager.get_blocks("resumed").blocks[2]
|
||||
assert [block.block_id for block in resident_blocks[:3]] == (
|
||||
original_resident_ids[:3]
|
||||
)
|
||||
assert not any(block.is_null for block in resident_blocks[:3])
|
||||
assert _resident_block_ids(manager, "resumed")[:3] == original_resident_ids[:3]
|
||||
assert get_hisparse_coordinator(manager).all_context_pages_resident(
|
||||
((resumed.request_id, num_computed, len(tokens) - num_computed),)
|
||||
)
|
||||
@@ -1249,17 +1304,13 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
|
||||
resumed_blocks = manager.get_blocks("resumed").blocks
|
||||
assert [block.block_id for block in resumed_blocks[2][:3]] == [
|
||||
resident_ids = _resident_block_ids(manager, "resumed")
|
||||
assert resident_ids[:3] == [
|
||||
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
|
||||
for page in range(3)
|
||||
]
|
||||
gpu_ids = [
|
||||
block.block_id
|
||||
for group in resumed_blocks[1:]
|
||||
for block in group
|
||||
if not block.is_null
|
||||
]
|
||||
_, indexer, _, hot = manager.get_block_ids("resumed")
|
||||
gpu_ids = [block_id for block_id in indexer + resident_ids + hot if block_id]
|
||||
assert len(gpu_ids) == len(set(gpu_ids))
|
||||
|
||||
|
||||
@@ -1488,8 +1539,7 @@ def test_hisparse_reused_copy_is_not_adopted():
|
||||
num_new_computed_tokens=num_computed,
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
resident_blocks = manager.get_blocks("resumed").blocks[2]
|
||||
assert all(block.is_null for block in resident_blocks[:3])
|
||||
assert _resident_block_ids(manager, "resumed")[:3] == [0, 0, 0]
|
||||
|
||||
|
||||
def test_hisparse_external_import_uses_hard_gpu_footprint():
|
||||
|
||||
@@ -28,10 +28,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiCo
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.hisparse import runtime as runtime_module
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
update_hisparse_residency,
|
||||
)
|
||||
from vllm.v1.hisparse.types import (
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVPageTransfer,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.metrics.stats import KVCacheEvictionEvent
|
||||
@@ -183,7 +187,7 @@ def test_full_graph_step_prepares_host_mirror_outside_model():
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_worker = worker
|
||||
connector._get_connector_metadata = MagicMock(
|
||||
return_value=HiSparseConnectorMetadata(None, (), (), {}, True)
|
||||
return_value=HiSparseConnectorMetadata(None, (), (), {}, True, {})
|
||||
)
|
||||
req_id_per_token = torch.tensor([0, 1], dtype=torch.int32)
|
||||
attn_metadata = SimpleNamespace(
|
||||
@@ -321,6 +325,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
)
|
||||
},
|
||||
True,
|
||||
{},
|
||||
)
|
||||
)
|
||||
# A verification step: four query tokens of one request fill the page.
|
||||
@@ -354,7 +359,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
# The next step's start mirrors the drafter's rows, then hands the page over.
|
||||
compute_stream.wait_event.reset_mock()
|
||||
connector._get_connector_metadata.return_value = HiSparseConnectorMetadata(
|
||||
None, (), (), {}, True
|
||||
None, (), (), {}, True, {}
|
||||
)
|
||||
connector.start_load_kv(
|
||||
SimpleNamespace(),
|
||||
@@ -397,7 +402,8 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
|
||||
|
||||
def test_scheduled_prefix_hit_publishes_adopted_copies():
|
||||
"""Copies adopted after scheduling must reach the worker's block table."""
|
||||
"""Copies adopted after scheduling reach the worker as a residency update,
|
||||
leaving the block-table row the scheduler output carries untouched."""
|
||||
from tests.v1.core.test_prefix_caching import (
|
||||
HISPARSE_BLOCK_SIZE,
|
||||
_allocate_scheduled,
|
||||
@@ -443,7 +449,39 @@ def test_scheduled_prefix_hit_publishes_adopted_copies():
|
||||
num_scheduled_tokens={resumed.request_id: num_new_tokens},
|
||||
)
|
||||
|
||||
scheduler.build_connector_meta(scheduler_output)
|
||||
core_row = list(scheduler_output.scheduled_new_reqs[0].block_ids[2])
|
||||
|
||||
resident_ids = scheduler_output.block_table_updates[resumed.request_id][2]
|
||||
assert resident_ids[:3] == copy_ids[:3]
|
||||
metadata = scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
update = metadata.residency_updates[resumed.request_id]
|
||||
assert update.pages == [0, 1, 2, 3]
|
||||
assert update.block_ids[0][:3] == copy_ids[:3]
|
||||
assert manager.get_block_ids(resumed.request_id)[2] == core_row
|
||||
assert core_row[:3] == [0, 0, 0]
|
||||
|
||||
|
||||
def test_residency_updates_persist_by_state_row():
|
||||
"""Updates name requests by batch row but are stored by state row.
|
||||
|
||||
An update for a lost page and an appended page must leave the other pages
|
||||
and the other request's row intact across steps that reorder the batch.
|
||||
"""
|
||||
table = torch.zeros((4, 2, 4), dtype=torch.int32)
|
||||
update_hisparse_residency(
|
||||
table,
|
||||
{
|
||||
"a": SparseKVResidencyUpdate([0, 1, 2], ([1, 2, 3], [11, 12, 13])),
|
||||
"b": SparseKVResidencyUpdate([0, 1], ([4, 5], [14, 15])),
|
||||
},
|
||||
["a", "b"],
|
||||
torch.tensor([2, 0], dtype=torch.int32),
|
||||
)
|
||||
update_hisparse_residency(
|
||||
table,
|
||||
{"a": SparseKVResidencyUpdate([1, 3], ([0, 6], [0, 16]))},
|
||||
["b", "a"],
|
||||
torch.tensor([0, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert table[[0, 2], 0].tolist() == [[4, 5, 0, 0], [1, 0, 3, 6]]
|
||||
assert table[[0, 2], 1].tolist() == [[14, 15, 0, 0], [11, 0, 13, 16]]
|
||||
|
||||
@@ -1068,6 +1068,7 @@ def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer
|
||||
row_mirrors={},
|
||||
all_context_pages_resident=True,
|
||||
row_mirrors_from_resident=False,
|
||||
residency_updates={},
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -1079,6 +1080,7 @@ def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer
|
||||
row_mirrors={},
|
||||
all_context_pages_resident=True,
|
||||
row_mirrors_from_resident=False,
|
||||
residency_updates={},
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -1152,6 +1154,7 @@ def test_hisparse_empty_step_does_not_replay_stale_host_mirror(monkeypatch):
|
||||
row_mirrors={},
|
||||
all_context_pages_resident=True,
|
||||
row_mirrors_from_resident=False,
|
||||
residency_updates={},
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user