[Bugfix][NIXL] Avoid receive reports for notification-only requests (#57570)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-18 13:33:02 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 63d9ad0a3a
commit 49e0427ca5
4 changed files with 91 additions and 2 deletions
+3
View File
@@ -311,6 +311,9 @@ steps:
PREFILLER_TP_SIZE: "2" PREFILLER_TP_SIZE: "2"
DECODER_TP_SIZE: "2" DECODER_TP_SIZE: "2"
MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct" MODEL_NAMES: "moonshotai/Kimi-Linear-48B-A3B-Instruct"
# Keep TP=2 prefill and TP=1 decode above both hybrid page-size minima.
PREFILL_BLOCK_SIZE: "2048"
DECODE_BLOCK_SIZE: "2048"
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768" VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--mamba-cache-mode align,--max-model-len 32768"
commands: commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
@@ -35,6 +35,39 @@ def scheduler():
return create_scheduler(vllm_config) return create_scheduler(vllm_config)
@pytest.mark.parametrize("policy", ["fail", "recompute"])
def test_failed_receive_completion_honors_failure_policy(policy):
"""Failed receives still need completion to free blocks or resume computation."""
scheduler = create_scheduler(create_vllm_config(kv_load_failure_policy=policy))
request = create_request(num_tokens=3 * scheduler.block_size)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_num_new_matched_tokens.side_effect = [
(2 * scheduler.block_size, True),
(0, False),
]
scheduler.connector.request_finished.return_value = (False, None)
scheduler.connector.take_events.return_value = ()
block_pool = scheduler.kv_cache_manager.block_pool
free_blocks_before = block_pool.get_num_free_blocks()
scheduler_output = scheduler.schedule()
assert request.status == RequestStatus.WAITING_FOR_REMOTE_KVS
output = create_model_runner_output([], finished_recving={request.request_id})
assert output.kv_connector_output is not None
output.kv_connector_output.failed_recving = {request.request_id}
scheduler.update_from_output(scheduler_output, output)
if policy == "fail":
assert request.status == RequestStatus.FINISHED_ERROR
assert request.request_id not in scheduler.requests
assert block_pool.get_num_free_blocks() == free_blocks_before
else:
assert request.num_computed_tokens == 0
next_output = scheduler.schedule()
assert request.request_id in next_output.num_scheduled_tokens
@pytest.mark.parametrize( @pytest.mark.parametrize(
"num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs", "num_prompt_blocks,num_external_computed_blocks,invalid_block_idxs",
[ [
@@ -10,6 +10,7 @@ import textwrap
import time import time
import uuid import uuid
from collections import defaultdict from collections import defaultdict
from concurrent.futures import Future
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any, cast from typing import Any, cast
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
@@ -2899,6 +2900,52 @@ def test_empty_recv_is_reported_only_when_awaited(
assert (request_id in done_recving) is awaiting_kvs assert (request_id in done_recving) is awaiting_kvs
@pytest.mark.parametrize("is_hma", [False, True])
@pytest.mark.parametrize(
"local_block_ids,awaiting_kvs",
[((), False), (([],), False), (([],), True), (([1, 2, 3],), True)],
)
def test_handshake_failure_reports_only_awaited_recvs(
recv_worker, is_hma, local_block_ids, awaiting_kvs
):
"""A failed cleanup handshake must not complete an already-aborted request."""
worker = recv_worker
worker._is_hma_required = is_hma
worker._recving_metadata.clear()
worker._remote_agents = {}
worker._handshake_lock = contextlib.nullcontext()
worker._ready_requests = queue.Queue()
worker._reqs_to_process = set()
worker.pcp_rank = 0
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id="request",
local_block_ids=local_block_ids,
kv_transfer_params={
"remote_block_ids": ([4, 5, 6],),
"remote_engine_id": "prefill",
"remote_request_id": "prefill-request",
"remote_host": "localhost",
"remote_port": 1234,
},
awaiting_kvs=awaiting_kvs,
)
handshake = Future[None]()
handshake.set_exception(RuntimeError("handshake failed"))
with patch.object(worker, "_ensure_handshake", return_value=handshake):
worker.start_load_kv(metadata)
results = worker.get_transfer_results()
expected = {"request"} if awaiting_kvs else set()
assert results.finished_recving == results.failed_recving == expected
assert worker.get_block_ids_with_load_errors() == (
set(local_block_ids[0]) if awaiting_kvs and not is_hma else set()
)
assert not worker._recving_metadata
assert not worker._recv_failures
assert not worker._pending_recv_notifs
@patch( @patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper, FailingNixlWrapper,
@@ -64,8 +64,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
len(meta.local_physical_block_ids), len(meta.local_physical_block_ids),
len(meta.remote.block_ids), len(meta.remote.block_ids),
) )
# always store metadata for failure recovery # Full local hits and aborted cleanup only notify P; no recv is awaited.
self._recving_metadata[req_id] = meta # On a full local hit:
# - Notification failure must not fail the request: its KV is local.
# - Receive completion must not be reported: num_external_tokens == 0,
# so the scheduler never entered WAITING_FOR_REMOTE_KVS to begin with.
# Aborted cleanup requests have already been removed from the scheduler.
if meta.awaiting_kvs or any(meta.local_block_ids):
self._recving_metadata[req_id] = meta
if remote_engine_id not in self._remote_agents: if remote_engine_id not in self._remote_agents:
# Initiate handshake with remote engine to exchange metadata. # Initiate handshake with remote engine to exchange metadata.
with self._handshake_lock: with self._handshake_lock: