[PD][NixlPush][Bugfix] Fix blocking handshake call on writer thread (#49221)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nicolò Lucchesi
2026-07-24 01:41:37 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5c5434e2d8
commit 589a5b884b
4 changed files with 196 additions and 80 deletions
@@ -25,6 +25,7 @@ import queue
import threading
import time
from collections import defaultdict
from concurrent.futures import Future
from typing import Any
from unittest.mock import MagicMock, patch
@@ -320,6 +321,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._finished_blocks_inbox = queue.Queue()
w._pending_completion_notifs = queue.Queue()
w._evict_finished_inbox = queue.Queue()
w._deferred_push_inbox = queue.Queue()
w._push_writer_wake = threading.Event()
w._push_writer_stop = threading.Event()
w._push_writer_thread = None
@@ -334,6 +336,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.world_size = 1
w.engine_id = "test-decode-engine"
w._remote_agents = {}
w._physical_blocks_per_logical_kv_block = 1
# Track _do_start_push_kv invocations.
calls: list[tuple[str, Any, dict[str, Any]]] = []
@@ -506,6 +509,112 @@ class TestPushWriterStartLoadKv:
assert w.start_push_calls == []
# The P→D handshake must run on the base worker's background executor, never
# blocking the writer thread: ``_do_start_push_kv`` defers the WRITE until the
# handshake resolves, then re-drives via ``_deferred_push_inbox``. These call
# the *real* ``_do_start_push_kv`` (the stub overrides it for matching tests).
def _real_do_start_push_kv(w, *args):
return NixlPushConnectorWorker._do_start_push_kv(w, *args)
def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
"""Full happy-path lifecycle: an in-flight handshake defers the WRITE (no
NIXL op from the writer or the executor callback); once it resolves the
request is re-queued on ``_deferred_push_inbox`` with the wake set; and on
re-drive with the handshake ready the WRITE is issued with a correct
ReqMeta."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x, ratio: x
xfer_calls: list[dict[str, Any]] = []
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
fut: Future = Future()
w._ensure_handshake = lambda *a, **k: fut
rd = _registration_data("req-hs", decode_engine_id="decode-engine")
_real_do_start_push_kv(w, "req-hs", ([1, 2, 3],), rd)
# Handshake pending -> nothing issued, nothing queued, no wake.
assert xfer_calls == []
assert w._deferred_push_inbox.qsize() == 0
assert not w._push_writer_wake.is_set()
# Handshake completes: request re-queued for the writer, wake set, but no
# *direct* WRITE from the callback (wrong thread for NIXL ops).
fut.set_result(({(0, 0): "agent"}, 0.0))
assert xfer_calls == []
assert w._push_writer_wake.is_set()
rid, blocks, reg = w._deferred_push_inbox.get_nowait()
assert (rid, blocks, reg) == ("req-hs", ([1, 2, 3],), rd)
# Re-drive on the writer with the handshake now ready -> WRITE issued.
w._ensure_handshake = lambda *a, **k: None
_real_do_start_push_kv(w, rid, blocks, reg)
assert len(xfer_calls) == 1
assert xfer_calls[0]["req_id"] == "req-hs"
meta = xfer_calls[0]["meta"]
assert meta.remote is not None
assert meta.remote.engine_id == "decode-engine"
# RemoteMeta.request_id is D's request id from the registration.
assert meta.remote.request_id == "req-hs"
def test_do_start_push_kv_drops_request_on_handshake_failure():
"""Handshake raises: the request is dropped (not re-queued, no WRITE) and
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
the old blocking behaviour."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x: x
xfer_calls: list[dict[str, Any]] = []
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
failures: list[dict[str, Any]] = []
w._log_failure = lambda **kw: failures.append(kw)
fut: Future = Future()
w._ensure_handshake = lambda *a, **k: fut
_real_do_start_push_kv(w, "req-fail", ([9],), _registration_data("req-fail"))
fut.set_exception(RuntimeError("handshake boom"))
assert w._deferred_push_inbox.qsize() == 0
assert xfer_calls == []
assert len(failures) == 1
assert failures[0]["failure_type"] == "push_handshake_failed"
def test_writer_loop_drains_deferred_push_inbox():
"""The writer loop drains ``_deferred_push_inbox`` and re-drives
``_do_start_push_kv`` for each entry (event-driven, no polling)."""
w = _StubWriterWorker.fresh()
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.get_new_notifs.return_value = {}
processed = threading.Event()
def _tracked(rid, blocks, rd):
w.start_push_calls.append((rid, blocks, rd))
processed.set()
w._do_start_push_kv = _tracked # type: ignore[method-assign]
w._deferred_push_inbox.put(
("req-retry", ([1, 2],), _registration_data("req-retry"))
)
w._push_writer_wake.set()
t = threading.Thread(target=w._push_writer_loop, daemon=True)
t.start()
try:
assert processed.wait(timeout=2.0), "writer did not drain deferred inbox"
finally:
w._push_writer_stop.set()
w._push_writer_wake.set()
t.join(timeout=2)
assert len(w.start_push_calls) == 1
assert w.start_push_calls[0][0] == "req-retry"
class TestPushWriterNotifs:
def test_get_new_notifs_processes_forwarded_completion_notif(self):
"""Non-PUSH_REG notifs forwarded by the writer thread are drained