[Bugfix][HiSparse] Fix a chunked-prefill preemption livelock (#59494)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
26bfdfb6bc
commit
3a69636645
@@ -1259,6 +1259,63 @@ def test_hisparse_cleanup_publishes_only_finalized_terminal_pages(
|
||||
assert not coordinator.has_pending_work()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_pages,reads_host", [(4, True), (3, False)])
|
||||
def test_hisparse_prefill_reads_host_once_it_fills_admission_window(
|
||||
num_pages, reads_host
|
||||
):
|
||||
"""A prefill switches to host reads once it fills its admission window.
|
||||
|
||||
Admission caps each request's resident pages at the in-flight window,
|
||||
assuming older pages move to host, but pinned pages were released only
|
||||
once free blocks fell below a fixed watermark. A chunked prefill larger
|
||||
than the watermark outgrew the pool first and was preempted.
|
||||
"""
|
||||
window_pages = 4
|
||||
manager = make_hisparse_kv_cache_manager(
|
||||
32,
|
||||
16,
|
||||
enable_caching=True,
|
||||
max_in_flight_tokens=window_pages * HISPARSE_BLOCK_SIZE,
|
||||
)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
tokens = list(range(2 * window_pages * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("prefill", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, num_pages * HISPARSE_BLOCK_SIZE) is not None
|
||||
pool = coordinator.gpu_pool
|
||||
assert pool is not None
|
||||
assert pool.get_num_free_blocks() >= coordinator.transition_watermark
|
||||
|
||||
hot_manager = coordinator.hot_managers[0]
|
||||
assert (request.request_id in hot_manager.hot_required) == reads_host
|
||||
|
||||
|
||||
def test_hisparse_preempted_prefill_resumes_from_durable_prefix():
|
||||
"""A prefill preempted mid-way must resume from the pages already on host.
|
||||
|
||||
Host writes trail a chunked prefill by a chunk, and host pages were only
|
||||
published once the whole computed prefix was durable. Each chunk moved
|
||||
that target past the finished writes, so nothing was published before the
|
||||
prefill ended: a prefill preempted each time it outgrew the GPU pool
|
||||
recomputed from the start forever.
|
||||
"""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
chunk = 2 * HISPARSE_BLOCK_SIZE
|
||||
tokens = list(range(3 * chunk))
|
||||
request = make_request("long", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.allocate_slots(request, chunk) is not None
|
||||
request.num_computed_tokens = chunk
|
||||
first_chunk_writes = coordinator.build_offload_command().page_transfers
|
||||
assert manager.allocate_slots(request, chunk) is not None
|
||||
request.num_computed_tokens = 2 * chunk
|
||||
acks = {transfer.transfer_id: 1 for transfer in first_chunk_writes}
|
||||
coordinator.update_spills(acks, acks)
|
||||
manager.free(request)
|
||||
|
||||
resumed = make_request("resumed", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.get_computed_blocks(resumed)[1] == chunk
|
||||
|
||||
|
||||
def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity():
|
||||
"""Old completions retain their pages through pressure and request-ID reuse."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 8, enable_caching=True)
|
||||
@@ -1285,7 +1342,9 @@ def test_hisparse_terminal_prefix_waits_for_all_workers_and_preserves_identity()
|
||||
assert not {block.block_id for block in old_blocks[:2]} & {
|
||||
block.block_id for block in pressure + new_blocks
|
||||
}
|
||||
assert all(block.block_hash is None for block in old_blocks)
|
||||
# Only the page every worker finished writing is published.
|
||||
assert old_blocks[0].block_hash is not None
|
||||
assert all(block.block_hash is None for block in old_blocks[1:])
|
||||
coordinator.update_spills({}, partial)
|
||||
repeated = make_request("probe", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert manager.get_computed_blocks(repeated)[1] == 2 * HISPARSE_BLOCK_SIZE
|
||||
|
||||
Reference in New Issue
Block a user