[Bugfix][HiSparse][NIXL] Import full blocks without tail prefill on D (#57049)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-17 15:26:10 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson
parent 41f9104fa6
commit 67e5b0acc9
8 changed files with 277 additions and 53 deletions
+95 -9
View File
@@ -325,7 +325,7 @@ def test_hisparse_reports_when_context_is_fully_resident():
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
"""Keep indexer-only imports host-backed in the resident block table."""
"""Restore the final host page when completing an indexer-only import."""
manager = make_hisparse_kv_cache_manager(
32,
16,
@@ -360,11 +360,15 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
assert len(source) == len(indexer) == len(resident) == 4
assert len(hot) == 2
# The local prefix is adopted from shadow pages (GPU-resident), while the
# externally imported page stays host-backed until its tail allocation.
assert not any(block.is_null for block in resident[:2])
assert resident[2].is_null
assert not resident[2].is_null
assert not resident[3].is_null
coordinator = get_hisparse_coordinator(manager)
assert not coordinator.build_offload_command().page_transfers
coordinator.finish_host_import(resumed.request_id, failed=False)
transfers = coordinator.build_offload_command().page_transfers
assert len(transfers) == 1 and transfers[0].restore
assert transfers[0].resident_block_ids == (resident[2].block_id,)
def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix():
@@ -465,7 +469,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
)
allocated = manager.get_blocks(resumed.request_id).blocks
assert len(allocated[0]) == host_blocks
assert allocated[2][1].is_null == (external > 0)
assert allocated[2][1].is_null == (host_blocks > 2)
assert not allocated[2][-1].is_null
manager.free(resumed)
# A surviving indexer offload must not make absent host KV look computed.
@@ -490,6 +495,85 @@ def allocate_external_prefix(
)
@pytest.mark.parametrize("num_tokens", [1, 15, 16, 17, 31, 32, 33, 127])
def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
"""Import every real token and restore the page used by last-token replay."""
from tests.v1.kv_connector.unit.utils import create_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
config = create_vllm_config(block_size=HISPARSE_BLOCK_SIZE)
connector = NixlPullConnectorScheduler(config, "test", manager.kv_cache_config)
request = make_request(
"import", list(range(num_tokens)), HISPARSE_BLOCK_SIZE, sha256
)
request.kv_transfer_params = {"do_remote_prefill": True}
count, is_async = connector.get_num_new_matched_tokens(request, 0)
assert count == num_tokens and is_async
assert allocate_external_prefix(manager, request, count) is not None
source, _, resident, _ = manager.get_blocks(request.request_id).blocks
assert len(source) == len(resident) == (num_tokens + 15) // 16
assert all(block.is_null for block in resident[:-1])
assert not resident[-1].is_null
coordinator = get_hisparse_coordinator(manager)
assert not coordinator.build_offload_command().page_transfers
coordinator.finish_host_import(request.request_id, failed=False)
transfers = coordinator.build_offload_command().page_transfers
assert len(transfers) == 1
restore = transfers[0]
assert restore.restore and not restore.after_forward
assert restore.host_block_id == source[-1].block_id
assert restore.resident_block_ids == (resident[-1].block_id,)
request.num_computed_tokens = count
scheduler = SimpleNamespace(
connector=connector,
kv_cache_manager=manager,
failed_recving_kv_req_ids=set(),
finished_recving_kv_req_ids={request.request_id},
)
Scheduler._update_waiting_for_remote_kv(scheduler, request)
assert request.num_tokens - request.num_computed_tokens == 1
assert manager.allocate_slots(request, num_new_tokens=1) is not None
assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)])
# A pending restore must not expose uninitialized GPU copies to prefix hits.
assert all(resident[-1] not in copies for copies in coordinator.copies.values())
coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1})
assert resident[-1].ref_cnt == 2
assert all(resident[-1] not in copies for copies in coordinator.copies.values())
coordinator.update_spills({}, {restore.transfer_id: 1})
assert resident[-1].ref_cnt == 1
assert coordinator.request_states[request.request_id].valid_pages == set(
range(num_tokens // HISPARSE_BLOCK_SIZE)
)
if num_tokens % HISPARSE_BLOCK_SIZE:
assert source[-1].block_hash is None
else:
assert coordinator.copies[source[-1].block_hash] == (resident[-1],)
def test_hisparse_aborted_tail_restore_retains_both_endpoints():
"""An abort cannot recycle storage still used by a queued tail restore."""
manager = make_hisparse_kv_cache_manager(16, 16)
request = make_request("import", list(range(17)), HISPARSE_BLOCK_SIZE, sha256)
assert allocate_external_prefix(manager, request, 17) is not None
source, _, resident, _ = manager.get_blocks(request.request_id).blocks
host_tail, gpu_tail = source[-1], resident[-1]
coordinator = get_hisparse_coordinator(manager)
coordinator.finish_host_import(request.request_id, failed=False)
restore = coordinator.build_offload_command().page_transfers[0]
manager.free(request)
assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1
coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1})
assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1
coordinator.update_spills({}, {restore.transfer_id: 1})
assert host_tail.ref_cnt == gpu_tail.ref_cnt == 0
assert not coordinator.has_pending_work()
@pytest.mark.parametrize("enable_caching", [False, True])
def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_caching):
"""Later admissions must not remove a scheduled request's only readable tier."""
@@ -669,7 +753,7 @@ def test_hisparse_host_cow_copy_is_drained_without_a_gpu_pool():
def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
"""An in-flight host import must reserve its unwritten resident pages."""
manager = make_hisparse_kv_cache_manager(
11,
12,
16,
transfer_device_cache=True,
)
@@ -1127,9 +1211,10 @@ def test_hisparse_external_import_uses_hard_gpu_footprint():
assert len(source) == num_prompt_blocks
assert len(indexer) == num_prompt_blocks
assert len(resident) == num_prompt_blocks
assert all(block.is_null for block in resident)
assert all(block.is_null for block in resident[:-1])
assert not resident[-1].is_null
assert len(hot) == 2
assert manager.block_pool.get_num_free_blocks() == 1
assert manager.block_pool.get_num_free_blocks() == 0
def test_hisparse_external_import_survives_capacity_retry():
@@ -1149,7 +1234,8 @@ def test_hisparse_external_import_survives_capacity_retry():
manager.free(first)
assert allocate_external_prefix(manager, second, len(tokens)) is not None
_, _, resident, hot = manager.get_blocks(second.request_id).blocks
assert all(block.is_null for block in resident)
assert all(block.is_null for block in resident[:-1])
assert not resident[-1].is_null
assert len(hot) == 2
+65
View File
@@ -531,6 +531,71 @@ def test_hisparse_eager_mirror_records_transfer_without_page_copy():
worker._enqueue_transfers.assert_not_called()
@pytest.mark.parametrize("is_host_writer", [False, True])
@pytest.mark.parametrize("device", ["cpu", "cuda"])
def test_hisparse_tail_restore_preserves_imported_rows(
monkeypatch, request, is_host_writer, device
):
"""Restore externally pinned host rows on every rank under sync checking."""
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA is required for the device copy check")
worker = _make_hisparse_worker()
worker.is_host_writer = is_host_writer
worker.kernel_block_size = 4
worker.host_caches = tuple(
torch.arange(24, dtype=torch.float32).reshape(12, 2) + layer * 100
for layer in range(2)
)
if device == "cuda":
# Match production's cudaHostRegister pool, which torch.is_pinned()
# does not recognize, rather than torch's caching pinned allocator.
host_pool, registered = hisparse_runtime_module.allocate_pinned_host_pool(
sum(cache.nbytes for cache in worker.host_caches)
)
request.addfinalizer(
lambda: hisparse_runtime_module.release_pinned_state([], [registered])
)
host_caches = host_pool.view(torch.float32).view(2, 12, 2).unbind()
for destination, source in zip(host_caches, worker.host_caches):
destination.copy_(source)
worker.host_caches = host_caches
worker.resident_caches = tuple(
torch.full((4, 4, 2), -1.0, device=device) for _ in range(2)
)
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(resident_source_index=i, eager_host_mirror=True)
)
for i in range(2)
]
worker._record_transfer_completion = MagicMock()
stream = torch.cuda.current_stream() if device == "cuda" else MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
transfer = SparseKVPageTransfer(7, 2, (1, 3), False, restore=True)
restore_pages = worker._restore_pages
if device == "cuda":
import vllm.utils.gpu_sync_debug as gpu_sync_debug
monkeypatch.setattr(gpu_sync_debug, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gpu_sync_debug, "_sync_check_enabled", False)
gpu_sync_debug.enable_gpu_sync_check()
restore_pages = gpu_sync_debug.with_gpu_sync_check(restore_pages)
restore_pages([transfer])
# Appending into unused capacity must preserve every imported prompt row.
for cache, host, block in zip(
worker.resident_caches, worker.host_caches, transfer.resident_block_ids
):
assert torch.equal(cache[block].cpu(), host[8:12])
cache[block, 2:] = 999
assert torch.equal(cache[block, :2].cpu(), host[8:10])
assert torch.all(cache[0] == -1)
worker._record_transfer_completion.assert_called_once_with(
[transfer], stream=stream
)
def test_hisparse_lazy_mirror_copies_transfer_pages():
worker = _make_hisparse_worker()
worker.cache_handles = [