[Bugfix][NIXL] Report a full prefix cache hit as finished (#56640)
Signed-off-by: Yizheng Jiao <jyizheng@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
e6eb0d120c
commit
381c61e008
@@ -341,6 +341,10 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
|
||||
assert req_meta.remote.request_id == f"prefill-{42}"
|
||||
# do_remote_prefill is consumed by request_finished to prevent re-issuing.
|
||||
assert request.kv_transfer_params["do_remote_prefill"] is False
|
||||
# The scheduler is not waiting on this recv -- the request is already gone
|
||||
# from self.requests, so reporting it would trip `assert req_id in
|
||||
# self.requests` in _update_from_kv_xfer_finished.
|
||||
assert req_meta.awaiting_kvs is False
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -2695,6 +2699,74 @@ class FailingNixlWrapper(FakeNixlWrapper):
|
||||
return super().check_xfer_state(handle)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
@pytest.mark.parametrize("awaiting_kvs", [True, False])
|
||||
@pytest.mark.parametrize("notification_fails", [False, True])
|
||||
def test_empty_recv_is_reported_only_when_awaited(
|
||||
default_vllm_config, dist_init, awaiting_kvs, notification_fails
|
||||
):
|
||||
"""An empty recv is reported iff the scheduler parked the request on it.
|
||||
|
||||
_read_blocks returns early when the local block list is empty -- D already
|
||||
holds the KV and only the producer needs telling. Two kinds of caller reach
|
||||
that path, and they need opposite handling:
|
||||
|
||||
Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the
|
||||
request is named in neither _recving_transfers nor _failed_recv_reqs and
|
||||
never reaches finished_recving. The scheduler has no other way to release a
|
||||
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
|
||||
skipped_waiting holding its blocks for the life of the process.
|
||||
|
||||
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
|
||||
free P's blocks for a request aborted before it was scheduled, or a readback
|
||||
on a request that stays RUNNING. Reporting either one crashes the engine
|
||||
core -- _update_from_kv_xfer_finished asserts the id is still in
|
||||
self.requests and that the request is parked or finished.
|
||||
|
||||
A failed notification changes neither: the KV is local either way, and the
|
||||
producer frees its own blocks on a timeout.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0.0,
|
||||
kv_cache_config=connector._kv_cache_config,
|
||||
)
|
||||
connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails
|
||||
|
||||
request_id = f"test_empty_recv_awaited_{awaiting_kvs}"
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=(), # empty: the whole prompt is already cached locally
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": [[20, 21, 22]],
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
connector.bind_connector_metadata(metadata)
|
||||
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
connector.bind_connector_metadata(NixlConnectorMetadata())
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert (request_id in done_recving) is awaiting_kvs
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
|
||||
@@ -149,8 +149,49 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
|
||||
|
||||
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
|
||||
_, _, local_num_computed_blocks, awaiting_kvs = scheduler._reqs_need_recv[
|
||||
request.request_id
|
||||
]
|
||||
assert local_num_computed_blocks == (2, 1)
|
||||
# There are external tokens to pull, so the scheduler parks the request and
|
||||
# is waiting to see it in finished_recving.
|
||||
assert awaiting_kvs is True
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_full_local_hit_is_not_awaited_by_the_scheduler():
|
||||
"""A recv with num_external_tokens == 0 must not be flagged as awaited.
|
||||
|
||||
The request keeps running -- there is nothing to pull, only a notification
|
||||
owed to the producer -- so the worker must not report it in
|
||||
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
|
||||
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
|
||||
request that is parked or already done."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
|
||||
scheduler = object.__new__(NixlPullConnectorScheduler)
|
||||
scheduler._reqs_in_batch = set()
|
||||
scheduler._reqs_need_save = {}
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.use_host_buffer = False
|
||||
scheduler.is_bidirectional_kv_xfer_enabled = False
|
||||
scheduler._is_hma_required = False
|
||||
scheduler.kv_cache_config = MagicMock(
|
||||
select_transfer_block_ids=lambda block_ids: block_ids
|
||||
)
|
||||
|
||||
blocks = KVCacheBlocks(blocks=([KVCacheBlock(block_id=0, _block_hash=object())],))
|
||||
request = create_request(do_remote_prefill=True)
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=0)
|
||||
|
||||
_, local_block_ids, _, awaiting_kvs = scheduler._reqs_need_recv[request.request_id]
|
||||
assert not local_block_ids # full local hit: nothing to pull
|
||||
assert awaiting_kvs is False
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
|
||||
Reference in New Issue
Block a user