[Bugfix][NIXL] Report a full prefix cache hit as finished (#56640)

Signed-off-by: Yizheng Jiao <jyizheng@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Yizheng Jiao
2026-09-14 20:15:19 -07:00
committed by GitHub
co-authored by Nick Hill
parent e6eb0d120c
commit 381c61e008
7 changed files with 142 additions and 6 deletions
@@ -341,6 +341,10 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
assert req_meta.remote.request_id == f"prefill-{42}"
# do_remote_prefill is consumed by request_finished to prevent re-issuing.
assert request.kv_transfer_params["do_remote_prefill"] is False
# The scheduler is not waiting on this recv -- the request is already gone
# from self.requests, so reporting it would trip `assert req_id in
# self.requests` in _update_from_kv_xfer_finished.
assert req_meta.awaiting_kvs is False
@patch(
@@ -2695,6 +2699,74 @@ class FailingNixlWrapper(FakeNixlWrapper):
return super().check_xfer_state(handle)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
)
@pytest.mark.parametrize("awaiting_kvs", [True, False])
@pytest.mark.parametrize("notification_fails", [False, True])
def test_empty_recv_is_reported_only_when_awaited(
default_vllm_config, dist_init, awaiting_kvs, notification_fails
):
"""An empty recv is reported iff the scheduler parked the request on it.
_read_blocks returns early when the local block list is empty -- D already
holds the KV and only the producer needs telling. Two kinds of caller reach
that path, and they need opposite handling:
Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the
request is named in neither _recving_transfers nor _failed_recv_reqs and
never reaches finished_recving. The scheduler has no other way to release a
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
skipped_waiting holding its blocks for the life of the process.
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
free P's blocks for a request aborted before it was scheduled, or a readback
on a request that stays RUNNING. Reporting either one crashes the engine
core -- _update_from_kv_xfer_finished asserts the id is still in
self.requests and that the request is parked or finished.
A failed notification changes neither: the KV is local either way, and the
producer frees its own blocks on a timeout.
"""
vllm_config = create_vllm_config()
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
connector.engine_id,
hand_shake_latency=0.0,
kv_cache_config=connector._kv_cache_config,
)
connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails
request_id = f"test_empty_recv_awaited_{awaiting_kvs}"
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=(), # empty: the whole prompt is already cached locally
kv_transfer_params={
"remote_block_ids": [[20, 21, 22]],
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
},
awaiting_kvs=awaiting_kvs,
)
connector.bind_connector_metadata(metadata)
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
connector.start_load_kv(dummy_ctx)
connector.bind_connector_metadata(NixlConnectorMetadata())
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
_, done_recving = connector.get_finished(finished_req_ids=set())
assert (request_id in done_recving) is awaiting_kvs
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
@@ -149,8 +149,49 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
_, _, local_num_computed_blocks, awaiting_kvs = scheduler._reqs_need_recv[
request.request_id
]
assert local_num_computed_blocks == (2, 1)
# There are external tokens to pull, so the scheduler parks the request and
# is waiting to see it in finished_recving.
assert awaiting_kvs is True
@pytest.mark.cpu_test
def test_full_local_hit_is_not_awaited_by_the_scheduler():
"""A recv with num_external_tokens == 0 must not be flagged as awaited.
The request keeps running -- there is nothing to pull, only a notification
owed to the producer -- so the worker must not report it in
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
request that is parked or already done."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
NixlPullConnectorScheduler,
)
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import KVCacheBlock
scheduler = object.__new__(NixlPullConnectorScheduler)
scheduler._reqs_in_batch = set()
scheduler._reqs_need_save = {}
scheduler._reqs_need_recv = {}
scheduler.use_host_buffer = False
scheduler.is_bidirectional_kv_xfer_enabled = False
scheduler._is_hma_required = False
scheduler.kv_cache_config = MagicMock(
select_transfer_block_ids=lambda block_ids: block_ids
)
blocks = KVCacheBlocks(blocks=([KVCacheBlock(block_id=0, _block_hash=object())],))
request = create_request(do_remote_prefill=True)
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=0)
_, local_block_ids, _, awaiting_kvs = scheduler._reqs_need_recv[request.request_id]
assert not local_block_ids # full local hit: nothing to pull
assert awaiting_kvs is False
@pytest.mark.cpu_test