[NIXL] Avoid reading expired blocks in bidirectional turn-2 read (#47021)

Signed-off-by: Tomer Gilad <tgilad@nvidia.com>
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
tomerg-nvidia
2026-07-14 13:03:41 +00:00
committed by GitHub
co-authored by NickLucche
parent b6754f536e
commit 7a74a9662b
10 changed files with 264 additions and 22 deletions
@@ -87,7 +87,7 @@ def _make_connector_with_fake_worker(
hand_shake_latency=0, cycles_before_done=0, do_handshake=True
):
"""Create a NixlConnector with FakeNixlConnectorWorker."""
vllm_config = create_vllm_config()
vllm_config = create_vllm_config(kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG)
kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=2)
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
@@ -100,7 +100,7 @@ def _make_connector_with_fake_worker(
assert isinstance(worker.nixl_wrapper, FakeNixlWrapper)
worker.kv_cache_layout = "HND"
if do_handshake:
remote_agents = worker._nixl_handshake(
remote_agents, _ = worker._nixl_handshake(
host="localhost",
port=1234,
remote_tp_size=1,
@@ -912,3 +912,164 @@ def test_remote_blocks_processed_flag_persists():
mro.kv_connector_output = KVConnectorOutput(finished_sending={req_id})
scheduler.update_from_output(so, mro)
assert_scheduler_empty(scheduler)
# 6. Turn-2 deadline expiry check (P declines reading expired D blocks)
_REMOTE = FakeNixlConnectorWorker.REMOTE_ENGINE_ID
@pytest.mark.parametrize(
("offset", "expiry_delta", "expect_declined"),
[
# offset known + deadline expired/near-expiry -> decline & recompute.
pytest.param(0.0, -10.0, True, id="expired"),
# offset applied before comparison: +50s in D's clock is expired locally
# when D is 100s ahead.
pytest.param(100.0, 50.0, True, id="offset_makes_expired"),
# within the default 5s safety margin -> treated as expired.
pytest.param(0.0, 2.0, True, id="near_expiry_within_margin"),
# far-future deadline -> read proceeds.
pytest.param(0.0, 1000.0, False, id="valid_far_deadline"),
# no deadline field (older peer / router did not forward) -> skipped.
pytest.param(0.0, None, False, id="missing_expiry_field"),
],
)
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_turn2_deadline_gate(dist_init, offset, expiry_delta, expect_declined):
"""P declines on the turn-2 readback if the deadline is known,
near-expiry and a handshake clock offset is known"""
connector, worker = _make_connector_with_fake_worker()
worker._engine_clock_offset[_REMOTE] = offset
expiry_time = None if expiry_delta is None else time.perf_counter() + expiry_delta
meta = NixlConnectorMetadata()
params = {
"do_remote_prefill": False,
"do_remote_decode": True,
"remote_block_ids": ([20, 21],),
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": "decode-req",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
}
if expiry_time is not None:
params["remote_blocks_expiry_time"] = expiry_time
meta.add_new_req_to_recv(
request_id="req",
local_block_ids=([10, 11],),
kv_transfer_params=params,
)
_do_load_kv(connector, meta)
assert worker.xfer_stats.data["num_kv_expired_reqs"] == (
[1] if expect_declined else []
)
_, done_recving = connector.get_finished(finished_req_ids=set())
assert "req" in done_recving
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_turn2_full_prefix_hit_with_expired_deadline_skips_gate(dist_init):
"""A full local prefix hit issues no READ. The expiry gate must be skipped
even when D's deadline is expired."""
connector, worker = _make_connector_with_fake_worker()
worker._engine_clock_offset[_REMOTE] = 0.0
meta = NixlConnectorMetadata()
meta.add_new_req_to_recv(
request_id="req",
local_block_ids=(), # full prefix hit -> zero local block groups
kv_transfer_params={
"do_remote_prefill": False,
"do_remote_decode": True,
"remote_block_ids": ([20, 21],),
"remote_engine_id": _REMOTE,
"remote_request_id": "decode-req",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
# Expired on D's clock; with offset 0 it is expired locally too.
"remote_blocks_expiry_time": time.perf_counter() - 10.0,
},
)
# Must not raise, and the gate must not fire for a no-read prefix hit.
_do_load_kv(connector, meta)
assert worker.xfer_stats.data["num_kv_expired_reqs"] == []
assert worker.xfer_stats.data["num_failed_transfers"] == []
def test_d_node_request_finished_exports_blocks_expiry_time():
"""D-node (do_remote_prefill request) exports a future float expiry time."""
vllm_config = create_vllm_config(kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG)
scheduler = create_scheduler(vllm_config)
BS = vllm_config.cache_config.block_size
req = create_request(
request_id=600, block_size=BS, num_tokens=int(BS * 2.5), do_remote_prefill=True
)
scheduler.add_request(req)
req_id = req.request_id
so = scheduler.schedule()
scheduler.update_from_output(
so, create_model_runner_output(reqs=[], finished_recving={req_id})
)
so = scheduler.schedule()
eco = scheduler.update_from_output(
so, create_model_runner_output(reqs=[req], use_eos=True)
)
kv = eco[0].outputs[0].kv_transfer_params
assert kv["do_remote_decode"] is True
assert isinstance(kv["remote_blocks_expiry_time"], float)
assert kv["remote_blocks_expiry_time"] > time.perf_counter()
def test_handshake_listener_appends_perf_counter_frame():
"""The handshake reply carries a 2nd frame with the listener's live
perf_counter."""
import threading
import msgspec
import zmq
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import (
NixlBaseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import GET_META_MSG
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx
from vllm.utils.network_utils import get_open_port, make_zmq_path
encoded_data = {(0, 0): b"payload-rank-0"}
ready_event = threading.Event()
stop_event = threading.Event()
host = "127.0.0.1"
port = get_open_port()
listener = threading.Thread(
target=NixlBaseConnectorScheduler._nixl_handshake_listener,
args=(encoded_data, ready_event, stop_event, host, port),
daemon=True,
)
listener.start()
try:
assert ready_event.wait(timeout=5)
path = make_zmq_path("tcp", host, port)
with zmq_ctx(zmq.REQ, path) as sock: # type: ignore[attr-defined]
sock.setsockopt(zmq.RCVTIMEO, 5000) # type: ignore[attr-defined]
t0 = time.perf_counter()
sock.send(msgspec.msgpack.encode((GET_META_MSG, 0, 0)))
parts = sock.recv_multipart()
t1 = time.perf_counter()
assert len(parts) == 2
assert parts[0] == b"payload-rank-0"
remote_perf = msgspec.msgpack.decode(parts[1])
assert isinstance(remote_perf, float)
# same-clock => offset is near zero.
offset = remote_perf - (t0 + t1) / 2
assert abs(offset) < 1.0
finally:
stop_event.set()
listener.join(timeout=5)
@@ -508,7 +508,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
expected_engine_id: str,
remote_pp_size: int = 1,
notif_agents_only: bool = False,
) -> dict[tuple[int, int], str]:
) -> tuple[dict[tuple[int, int], str], float]:
# Mimic slow _nixl_handshake, as well as bypass zmq communication.
time.sleep(self._hand_shake_latency)
# These should've been done in register_kv_caches(), called by
@@ -560,7 +560,8 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
remote_tp_size=remote_tp_size,
)
remote_agents[(0, remote_tp_rank)] = remote_agent_name
return remote_agents
# Handshake bypasses zmq, so report a zero clock offset to the peer.
return remote_agents, 0.0
class TestNixlHandshake:
@@ -768,7 +769,7 @@ class TestNixlHandshake:
range(tp_ratio)
)
remote_agents = worker._nixl_handshake(
remote_agents, _ = worker._nixl_handshake(
host="localhost",
port=1234,
remote_tp_size=4,
@@ -780,7 +781,7 @@ class TestNixlHandshake:
# discovered. This is not a scenario we actively support right now, but
# the connector allows it.
worker.REMOTE_ENGINE_ID = "remote_engine_2"
remote_agents = worker._nixl_handshake(
remote_agents, _ = worker._nixl_handshake(
host="localhost",
port=1234,
remote_tp_size=6,
@@ -2878,7 +2879,10 @@ def test_compatibility_hash_validation(
# Mock ZMQ socket to return our handshake payload
mock_socket = MagicMock()
mock_socket.recv.return_value = msgspec.msgpack.encode(handshake_payload)
mock_socket.recv_multipart.return_value = [
msgspec.msgpack.encode(handshake_payload),
msgspec.msgpack.encode(time.perf_counter()),
]
# Mock add_remote_agent to avoid actual NIXL operations
# Patch zmq_ctx to return our mock socket
@@ -2897,7 +2901,7 @@ def test_compatibility_hash_validation(
expected_engine_id=FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
)
else:
result = decode_worker._nixl_handshake(
result, _ = decode_worker._nixl_handshake(
host="localhost",
port=1234,
remote_tp_size=1,
@@ -2981,7 +2985,10 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario)
raise AssertionError(f"{error_scenario} not a valid scenario")
mock_socket = MagicMock()
mock_socket.recv.return_value = msg_bytes
mock_socket.recv_multipart.return_value = [
msg_bytes,
msgspec.msgpack.encode(time.perf_counter()),
]
with (
patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"),
patch.object(nixl.base_worker, "zmq_ctx") as mock_zmq_ctx,
@@ -209,6 +209,7 @@ def test_read_blocks_for_req_expands_remote_ids(
worker = object.__new__(NixlConnectorWorker)
worker._physical_blocks_per_logical_kv_block = local_physical_per_logical
worker._engine_last_active = {}
worker._bidirectional_kv_xfer_enabled = False
has_mamba = any(t is MambaSpec for t in resolved_types)
has_swa = any(t is SlidingWindowSpec for t in resolved_types)