[Bugfix][NIXL] Report a full prefix cache hit as finished (#56640)
Signed-off-by: Yizheng Jiao <jyizheng@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
e6eb0d120c
commit
381c61e008
@@ -341,6 +341,10 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv():
|
||||
assert req_meta.remote.request_id == f"prefill-{42}"
|
||||
# do_remote_prefill is consumed by request_finished to prevent re-issuing.
|
||||
assert request.kv_transfer_params["do_remote_prefill"] is False
|
||||
# The scheduler is not waiting on this recv -- the request is already gone
|
||||
# from self.requests, so reporting it would trip `assert req_id in
|
||||
# self.requests` in _update_from_kv_xfer_finished.
|
||||
assert req_meta.awaiting_kvs is False
|
||||
|
||||
|
||||
@patch(
|
||||
@@ -2695,6 +2699,74 @@ class FailingNixlWrapper(FakeNixlWrapper):
|
||||
return super().check_xfer_state(handle)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
)
|
||||
@pytest.mark.parametrize("awaiting_kvs", [True, False])
|
||||
@pytest.mark.parametrize("notification_fails", [False, True])
|
||||
def test_empty_recv_is_reported_only_when_awaited(
|
||||
default_vllm_config, dist_init, awaiting_kvs, notification_fails
|
||||
):
|
||||
"""An empty recv is reported iff the scheduler parked the request on it.
|
||||
|
||||
_read_blocks returns early when the local block list is empty -- D already
|
||||
holds the KV and only the producer needs telling. Two kinds of caller reach
|
||||
that path, and they need opposite handling:
|
||||
|
||||
Parked (awaiting_kvs=True, a full prefix cache hit). Without a report the
|
||||
request is named in neither _recving_transfers nor _failed_recv_reqs and
|
||||
never reaches finished_recving. The scheduler has no other way to release a
|
||||
WAITING_FOR_REMOTE_KVS request, and there is no timeout, so it sits in
|
||||
skipped_waiting holding its blocks for the life of the process.
|
||||
|
||||
Notify-only (awaiting_kvs=False): request_finished seeding an empty recv to
|
||||
free P's blocks for a request aborted before it was scheduled, or a readback
|
||||
on a request that stays RUNNING. Reporting either one crashes the engine
|
||||
core -- _update_from_kv_xfer_finished asserts the id is still in
|
||||
self.requests and that the request is parked or finished.
|
||||
|
||||
A failed notification changes neither: the KV is local either way, and the
|
||||
producer frees its own blocks on a timeout.
|
||||
"""
|
||||
vllm_config = create_vllm_config()
|
||||
connector = NixlConnector(
|
||||
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
|
||||
)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
connector.engine_id,
|
||||
hand_shake_latency=0.0,
|
||||
kv_cache_config=connector._kv_cache_config,
|
||||
)
|
||||
connector.connector_worker.nixl_wrapper.fail_send_notif = notification_fails
|
||||
|
||||
request_id = f"test_empty_recv_awaited_{awaiting_kvs}"
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id=request_id,
|
||||
local_block_ids=(), # empty: the whole prompt is already cached locally
|
||||
kv_transfer_params={
|
||||
"remote_block_ids": [[20, 21, 22]],
|
||||
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
|
||||
"remote_request_id": f"prefill-{request_id}",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1234,
|
||||
"remote_tp_size": 1,
|
||||
},
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
connector.bind_connector_metadata(metadata)
|
||||
dummy_ctx = ForwardContext(no_compile_layers={}, attn_metadata={}, slot_mapping={})
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
connector.bind_connector_metadata(NixlConnectorMetadata())
|
||||
time.sleep(0.1)
|
||||
connector.start_load_kv(dummy_ctx)
|
||||
|
||||
_, done_recving = connector.get_finished(finished_req_ids=set())
|
||||
assert (request_id in done_recving) is awaiting_kvs
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
|
||||
FailingNixlWrapper,
|
||||
|
||||
@@ -149,8 +149,49 @@ def test_update_state_after_alloc_tracks_cached_blocks_per_group():
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=2)
|
||||
|
||||
_, _, local_num_computed_blocks = scheduler._reqs_need_recv[request.request_id]
|
||||
_, _, local_num_computed_blocks, awaiting_kvs = scheduler._reqs_need_recv[
|
||||
request.request_id
|
||||
]
|
||||
assert local_num_computed_blocks == (2, 1)
|
||||
# There are external tokens to pull, so the scheduler parks the request and
|
||||
# is waiting to see it in finished_recving.
|
||||
assert awaiting_kvs is True
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_full_local_hit_is_not_awaited_by_the_scheduler():
|
||||
"""A recv with num_external_tokens == 0 must not be flagged as awaited.
|
||||
|
||||
The request keeps running -- there is nothing to pull, only a notification
|
||||
owed to the producer -- so the worker must not report it in
|
||||
finished_recving. Doing so trips `assert RequestStatus.is_finished` in
|
||||
_update_from_kv_xfer_finished, which only tolerates a finished recv for a
|
||||
request that is parked or already done."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlock
|
||||
|
||||
scheduler = object.__new__(NixlPullConnectorScheduler)
|
||||
scheduler._reqs_in_batch = set()
|
||||
scheduler._reqs_need_save = {}
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.use_host_buffer = False
|
||||
scheduler.is_bidirectional_kv_xfer_enabled = False
|
||||
scheduler._is_hma_required = False
|
||||
scheduler.kv_cache_config = MagicMock(
|
||||
select_transfer_block_ids=lambda block_ids: block_ids
|
||||
)
|
||||
|
||||
blocks = KVCacheBlocks(blocks=([KVCacheBlock(block_id=0, _block_hash=object())],))
|
||||
request = create_request(do_remote_prefill=True)
|
||||
|
||||
scheduler.update_state_after_alloc(request, blocks, num_external_tokens=0)
|
||||
|
||||
_, local_block_ids, _, awaiting_kvs = scheduler._reqs_need_recv[request.request_id]
|
||||
assert not local_block_ids # full local hit: nothing to pull
|
||||
assert awaiting_kvs is False
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
|
||||
@@ -113,7 +113,7 @@ class NixlBaseConnectorScheduler:
|
||||
# New requests are added by update_state_after_alloc in
|
||||
# the scheduler. Used to make metadata passed to Worker.
|
||||
self._reqs_need_recv: dict[
|
||||
ReqId, tuple[Request, BlockIds, tuple[int, ...]]
|
||||
ReqId, tuple[Request, BlockIds, tuple[int, ...], bool]
|
||||
] = {}
|
||||
self._reqs_need_save: dict[ReqId, Request] = {}
|
||||
# Reqs to send and their expiration time
|
||||
@@ -459,13 +459,19 @@ class NixlBaseConnectorScheduler:
|
||||
meta = NixlConnectorMetadata()
|
||||
|
||||
# Loop through scheduled reqs and convert to ReqMeta.
|
||||
for req_id, (req, block_ids, cached) in self._reqs_need_recv.items():
|
||||
for req_id, (
|
||||
req,
|
||||
block_ids,
|
||||
cached,
|
||||
awaiting_kvs,
|
||||
) in self._reqs_need_recv.items():
|
||||
assert req.kv_transfer_params is not None
|
||||
meta.add_new_req_to_recv(
|
||||
request_id=req_id,
|
||||
local_block_ids=block_ids,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
local_num_computed_blocks=cached,
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
|
||||
if self.use_host_buffer:
|
||||
|
||||
@@ -246,6 +246,9 @@ class ReqMeta:
|
||||
remote_block_size: int | None = None
|
||||
# Remote producer pipeline-parallel size (push mode, D side).
|
||||
pp_size: int = 1
|
||||
# True only when the scheduler parked the request in WAITING_FOR_REMOTE_KVS
|
||||
# and expects it in finished_recving; notify-only recvs must not be reported.
|
||||
awaiting_kvs: bool = False
|
||||
|
||||
|
||||
class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
@@ -275,6 +278,7 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
awaiting_kvs: bool = False,
|
||||
) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
local_block_ids=local_block_ids,
|
||||
@@ -285,6 +289,7 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
remote_block_size=kv_transfer_params.get("remote_block_size"),
|
||||
pp_size=kv_transfer_params.get("pp_size", 1),
|
||||
local_num_computed_blocks=local_num_computed_blocks,
|
||||
awaiting_kvs=awaiting_kvs,
|
||||
)
|
||||
|
||||
def add_new_req_to_save(
|
||||
@@ -303,11 +308,13 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
local_block_ids: BlockIds,
|
||||
kv_transfer_params: dict[str, Any],
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
awaiting_kvs: bool = False,
|
||||
):
|
||||
req = self._add_new_req(
|
||||
local_block_ids,
|
||||
kv_transfer_params,
|
||||
local_num_computed_blocks,
|
||||
awaiting_kvs,
|
||||
)
|
||||
req.remote = RemoteMeta(
|
||||
block_ids=kv_transfer_params["remote_block_ids"],
|
||||
|
||||
@@ -172,6 +172,9 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
request,
|
||||
local_block_ids,
|
||||
local_num_computed_blocks,
|
||||
# Parked in WAITING_FOR_REMOTE_KVS only when there is
|
||||
# something to pull; a full local hit stays RUNNING.
|
||||
num_external_tokens > 0,
|
||||
)
|
||||
|
||||
else:
|
||||
@@ -224,7 +227,7 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
# To avoid stranding the prefill blocks in the prefill instance,
|
||||
# we must add empty block_ids to _reqs_need_recv so that our
|
||||
# worker side will notify and free blocks in the prefill instance.
|
||||
self._reqs_need_recv[request.request_id] = (request, [], ())
|
||||
self._reqs_need_recv[request.request_id] = (request, [], (), False)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
|
||||
@@ -303,6 +303,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_dram_handle=local_dram_handle,
|
||||
remote_xfer_side_handle=remote_xfer_side_handle,
|
||||
expected_consumers=plan.local_consumers,
|
||||
awaiting_kvs=meta.awaiting_kvs,
|
||||
):
|
||||
return
|
||||
|
||||
@@ -327,6 +328,7 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_dram_handle: int | None,
|
||||
remote_xfer_side_handle: int,
|
||||
expected_consumers: int,
|
||||
awaiting_kvs: bool,
|
||||
) -> bool:
|
||||
"""
|
||||
Post a READ point-to-point xfer request from a single local worker to
|
||||
@@ -387,6 +389,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
remote_agent_name=agent_name,
|
||||
)
|
||||
self.xfer_stats.record_failed_notification()
|
||||
# Report even on notification failure: the KV is already local, and
|
||||
# an unreported parked request would hold its blocks forever.
|
||||
# Notify-only recvs must stay unreported (scheduler asserts).
|
||||
if awaiting_kvs:
|
||||
self._recving_transfers.setdefault(request_id, [])
|
||||
return True
|
||||
|
||||
if read_spec.block_ids_by_region:
|
||||
|
||||
@@ -198,7 +198,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
# ReqMeta without a KeyError — the actual remote block IDs are
|
||||
# learned by P over the NIXL handshake at WRITE time.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, local_block_ids, ())
|
||||
self._reqs_need_recv[request.request_id] = (request, local_block_ids, (), False)
|
||||
|
||||
# Mark as processed so a re-entry (e.g. preemption + reschedule)
|
||||
# doesn't re-stage the registration.
|
||||
@@ -240,7 +240,7 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
# recv so the worker emits a notif that lets P free them.
|
||||
# Seed remote_block_ids so add_new_req_to_recv won't KeyError.
|
||||
params["remote_block_ids"] = ()
|
||||
self._reqs_need_recv[request.request_id] = (request, [], ())
|
||||
self._reqs_need_recv[request.request_id] = (request, [], (), False)
|
||||
params["do_remote_prefill"] = False
|
||||
return False, None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user