[Bugfix][HiSparse] Fix a chunked-prefill preemption livelock (#59494)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-01 13:02:31 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 26bfdfb6bc
commit 3a69636645
3 changed files with 106 additions and 29 deletions
+60 -1
View File
@@ -1259,6 +1259,63 @@ def test_hisparse_cleanup_publishes_only_finalized_terminal_pages(
assert not coordinator.has_pending_work()
@pytest.mark.parametrize("num_pages,reads_host", [(4, True), (3, False)])
def test_hisparse_prefill_reads_host_once_it_fills_admission_window(
num_pages, reads_host
):
"""A prefill switches to host reads once it fills its admission window.
Admission caps each request's resident pages at the in-flight window,
assuming older pages move to host, but pinned pages were released only
once free blocks fell below a fixed watermark. A chunked prefill larger
than the watermark outgrew the pool first and was preempted.
"""
window_pages = 4
manager = make_hisparse_kv_cache_manager(
32,
16,
enable_caching=True,
max_in_flight_tokens=window_pages * HISPARSE_BLOCK_SIZE,
)
coordinator = get_hisparse_coordinator(manager)
tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE))
request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, num_pages * HISPARSE_BLOCK_SIZE) is not None
pool = coordinator.gpu_pool
assert pool is not None
assert pool.get_num_free_blocks() >= coordinator.transition_watermark
hot_manager = coordinator.hot_managers[0]
assert (request.request_id in hot_manager.hot_required) == reads_host
def test_hisparse_preempted_prefill_resumes_from_durable_prefix():
"""A prefill preempted mid-way must resume from the pages already on host.
Host writes trail a chunked prefill by a chunk, and host pages were only
published once the whole computed prefix was durable. Each chunk moved
that target past the finished writes, so nothing was published before the
prefill ended: a prefill preempted each time it outgrew the GPU pool
recomputed from the start forever.
"""
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
coordinator = get_hisparse_coordinator(manager)
chunk = 2 * HISPARSE_BLOCK_SIZE
tokens = list(range(3 * chunk))
request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.allocate_slots(request, chunk) is not None
request.num_computed_tokens = chunk
first_chunk_writes = coordinator.build_offload_command().page_transfers
assert manager.allocate_slots(request, chunk) is not None
request.num_computed_tokens = 2 * chunk
acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes}
coordinator.update_spills(acks, acks)
manager.free(request)
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.get_computed_blocks(resumed)[1] == chunk
def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity():
"""Old completions retain their pages through pressure and request-ID reuse."""
manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True)
@@ -1285,7 +1342,9 @@ def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity()
assert not {block.block_id for block in old_blocks[:2]} & {
block.block_id for block in pressure + new_blocks
}
assert all(block.block_hash is None for block in old_blocks)
# Only the page every worker finished writing is published.
assert old_blocks[0].block_hash is not None
assert all(block.block_hash is None for block in old_blocks[1:])
coordinator.update_spills({}, partial)
repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE