[PD][NixlPush][Bugfix] Fix blocking handshake call on writer thread (#49221)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nicolò Lucchesi
2026-07-24 01:41:37 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5c5434e2d8
commit 589a5b884b
4 changed files with 196 additions and 80 deletions
+28 -14
View File
@@ -47,7 +47,8 @@ sequenceDiagram
else only one side present
PWriter->>PWriter: stash and wait, self-poll only when blocks unmatched
end
PWriter->>PWriter: ensure D handshake (one-time)
PWriter->>PWriter: _ensure_handshake to D (async; defer WRITE)
PWriter->>PWriter: handshake callback re-queues on _deferred_push_inbox, wake
PWriter->>DWriter: NIXL WRITE direct to D GPU + completion notif
note over DWorker,DWriter: D side - completion accounting
@@ -100,15 +101,20 @@ event:
D, completion notifs after a WRITE, late-arriving ``PUSH_REG``)
even when there is no new metadata to act on.
3. **Handshake-completion callback** (background handshake executor
thread) — when a deferred D→P handshake finishes successfully, the
future's done-callback re-enqueues the registration onto
``_reg_send_inbox`` and sets the wake so the corresponding
``send_notif`` runs on the writer (we never call ``send_notif`` from
the executor thread). On this second pass ``_ensure_handshake``
returns ``None`` (the agent is now connected), so the writer sends
the ``PUSH_REG`` directly. If the handshake *failed*, the callback
fails the request instead of re-enqueuing, so there is no retry
loop.
thread) — both handshakes run on the executor and never block the
writer; their done-callbacks re-enqueue the deferred op and set the
wake, since neither ``send_notif`` nor the NIXL WRITE may run off the
writer thread:
* the **D→P** handshake (before sending ``PUSH_REG``) re-enqueues the
registration onto ``_reg_send_inbox``;
* the **P→D** handshake (before a WRITE) re-enqueues the matched
``(req_id, blocks, reg_data)`` onto ``_deferred_push_inbox``.
On this second pass ``_ensure_handshake`` returns ``None`` (the agent
is now connected), so the writer sends the ``PUSH_REG`` / issues the
WRITE directly. If a handshake *failed*, the callback fails or drops
the request instead of re-enqueuing, so there is no retry loop (see
Failure handling).
In addition to event-driven wakes, the writer self-polls at
``_PUSH_WRITER_POLL_INTERVAL_MS = 1.0`` ms while there are P-side
@@ -227,6 +233,13 @@ Two per-request timers are armed on the scheduler:
* **D-side ``send_notif`` failure when shipping the PUSH_REG to P** —
identical handling: ``_handle_failed_transfer`` marks the recv as
failed.
* **P-side handshake failure (P→D handshake before a WRITE)** — the
future's done-callback logs ``push_handshake_failed`` and drops the
request without re-queuing. It deliberately does *not* call
``_handle_failed_transfer`` (there is no ``_recving_metadata`` entry to
invalidate on the producer side, same reasoning as the WRITE-submission
failure below). P's blocks are reclaimed by the ``_kv_lease_duration``
lease and D's stale registration by its watchdog.
* **P-side WRITE submission failure** — the WRITE handle (if any) is
released and ``xfer_stats.record_failed_transfer()`` bumps the
failure counter. We deliberately do not call
@@ -246,10 +259,11 @@ existing NIXL connector:
class — all subclasses of the existing base classes;
* one dedicated background thread per worker;
* a few cross-thread queues, each with a single consumer (the writer);
most have one producer, except ``_reg_send_inbox``, which is fed both
by the engine main thread (new registrations) and by the
handshake-completion callback (registrations replayed after their
D→P handshake finishes);
most have one producer, except the two replay queues fed by both the
engine main thread and a handshake-completion callback:
``_reg_send_inbox`` (registrations replayed after their D→P handshake)
and ``_deferred_push_inbox`` (matched pushes replayed after their P→D
handshake);
* one new notification type (`PUSH_REG:<msgpack>`).
Behavior on the engine main thread is otherwise unchanged. The writer
@@ -25,6 +25,7 @@ import queue
import threading
import time
from collections import defaultdict
from concurrent.futures import Future
from typing import Any
from unittest.mock import MagicMock, patch
@@ -320,6 +321,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w._finished_blocks_inbox = queue.Queue()
w._pending_completion_notifs = queue.Queue()
w._evict_finished_inbox = queue.Queue()
w._deferred_push_inbox = queue.Queue()
w._push_writer_wake = threading.Event()
w._push_writer_stop = threading.Event()
w._push_writer_thread = None
@@ -334,6 +336,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
w.world_size = 1
w.engine_id = "test-decode-engine"
w._remote_agents = {}
w._physical_blocks_per_logical_kv_block = 1
# Track _do_start_push_kv invocations.
calls: list[tuple[str, Any, dict[str, Any]]] = []
@@ -506,6 +509,112 @@ class TestPushWriterStartLoadKv:
assert w.start_push_calls == []
# The P→D handshake must run on the base worker's background executor, never
# blocking the writer thread: ``_do_start_push_kv`` defers the WRITE until the
# handshake resolves, then re-drives via ``_deferred_push_inbox``. These call
# the *real* ``_do_start_push_kv`` (the stub overrides it for matching tests).
def _real_do_start_push_kv(w, *args):
return NixlPushConnectorWorker._do_start_push_kv(w, *args)
def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
"""Full happy-path lifecycle: an in-flight handshake defers the WRITE (no
NIXL op from the writer or the executor callback); once it resolves the
request is re-queued on ``_deferred_push_inbox`` with the wake set; and on
re-drive with the handshake ready the WRITE is issued with a correct
ReqMeta."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x, ratio: x
xfer_calls: list[dict[str, Any]] = []
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
fut: Future = Future()
w._ensure_handshake = lambda *a, **k: fut
rd = _registration_data("req-hs", decode_engine_id="decode-engine")
_real_do_start_push_kv(w, "req-hs", ([1, 2, 3],), rd)
# Handshake pending -> nothing issued, nothing queued, no wake.
assert xfer_calls == []
assert w._deferred_push_inbox.qsize() == 0
assert not w._push_writer_wake.is_set()
# Handshake completes: request re-queued for the writer, wake set, but no
# *direct* WRITE from the callback (wrong thread for NIXL ops).
fut.set_result(({(0, 0): "agent"}, 0.0))
assert xfer_calls == []
assert w._push_writer_wake.is_set()
rid, blocks, reg = w._deferred_push_inbox.get_nowait()
assert (rid, blocks, reg) == ("req-hs", ([1, 2, 3],), rd)
# Re-drive on the writer with the handshake now ready -> WRITE issued.
w._ensure_handshake = lambda *a, **k: None
_real_do_start_push_kv(w, rid, blocks, reg)
assert len(xfer_calls) == 1
assert xfer_calls[0]["req_id"] == "req-hs"
meta = xfer_calls[0]["meta"]
assert meta.remote is not None
assert meta.remote.engine_id == "decode-engine"
# RemoteMeta.request_id is D's request id from the registration.
assert meta.remote.request_id == "req-hs"
def test_do_start_push_kv_drops_request_on_handshake_failure():
"""Handshake raises: the request is dropped (not re-queued, no WRITE) and
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
the old blocking behaviour."""
w = _StubWriterWorker.fresh()
w._logical_to_kernel_block_ids = lambda x: x
xfer_calls: list[dict[str, Any]] = []
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
failures: list[dict[str, Any]] = []
w._log_failure = lambda **kw: failures.append(kw)
fut: Future = Future()
w._ensure_handshake = lambda *a, **k: fut
_real_do_start_push_kv(w, "req-fail", ([9],), _registration_data("req-fail"))
fut.set_exception(RuntimeError("handshake boom"))
assert w._deferred_push_inbox.qsize() == 0
assert xfer_calls == []
assert len(failures) == 1
assert failures[0]["failure_type"] == "push_handshake_failed"
def test_writer_loop_drains_deferred_push_inbox():
"""The writer loop drains ``_deferred_push_inbox`` and re-drives
``_do_start_push_kv`` for each entry (event-driven, no polling)."""
w = _StubWriterWorker.fresh()
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.get_new_notifs.return_value = {}
processed = threading.Event()
def _tracked(rid, blocks, rd):
w.start_push_calls.append((rid, blocks, rd))
processed.set()
w._do_start_push_kv = _tracked # type: ignore[method-assign]
w._deferred_push_inbox.put(
("req-retry", ([1, 2],), _registration_data("req-retry"))
)
w._push_writer_wake.set()
t = threading.Thread(target=w._push_writer_loop, daemon=True)
t.start()
try:
assert processed.wait(timeout=2.0), "writer did not drain deferred inbox"
finally:
w._push_writer_stop.set()
w._push_writer_wake.set()
t.join(timeout=2)
assert len(w.start_push_calls) == 1
assert w.start_push_calls[0][0] == "req-retry"
class TestPushWriterNotifs:
def test_get_new_notifs_processes_forwarded_completion_notif(self):
"""Non-PUSH_REG notifs forwarded by the writer thread are drained
@@ -792,7 +792,7 @@ class NixlBaseConnectorWorker:
failure_type: str,
req_id: str | None,
msg: str = "",
error: Exception | None = None,
error: BaseException | None = None,
meta: ReqMeta | None = None,
**extra_context,
):
@@ -8,10 +8,13 @@ notifs are forwarded to the engine main thread), sends PUSH_REG via
``send_notif``, matches D registrations with P finished blocks, and
issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``.
The engine main thread feeds the writer through three queues:
The engine main thread feeds the writer through queues:
``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox``
(P-side blocks from metadata) and ``_pending_completion_notifs``
(non-PUSH_REG notifs forwarded back for HB / completion accounting).
The handshake-completion callback feeds ``_deferred_push_inbox`` with
matched pushes whose P→D handshake has finished so the writer can
(re-)issue the WRITE without ever blocking on the network.
Wake model: the writer self-polls every
``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched
@@ -21,8 +24,8 @@ event-driven: the engine main thread sets ``_push_writer_wake`` from
``start_load_kv`` (when handing it new work) and from ``get_finished``
(so each engine step gives the writer a chance to drain NIXL notifs);
the handshake-completion callback sets the same event after a deferred
PUSH_REG send has been queued. When a request's lease expires (the base
worker reports it via ``done_sending``) or the WRITE completes,
PUSH_REG send or a deferred push WRITE has been queued. When a request's
lease expires (the base worker reports it via ``done_sending``) or the WRITE completes,
``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so
the writer drops any leftover ``_push_finished_blocks`` /
``_pending_d_registrations`` and stops self-polling.
@@ -109,6 +112,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
# ``_push_finished_blocks`` so an unmatched entry doesn't keep the
# writer busy-polling forever.
self._evict_finished_inbox: queue.Queue[str] = queue.Queue()
# Handshakes that have just completed and are ready for the WRITE on wthread
self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]()
# Wake signal from engine main thread (start_load_kv / get_finished).
# Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has
@@ -206,7 +211,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
break
self._send_registration_to_p(rid, rd)
# 2. P-side finished blocks; match against pending regs.
# 2. Deferred P→D pushes whose handshake just completed; do xfer now
while True:
try:
rid, blocks, rd = self._deferred_push_inbox.get_nowait()
except queue.Empty:
break
self._do_start_push_kv(rid, blocks, rd)
# 3. P-side finished blocks; match against pending regs.
while True:
try:
rid, blocks = self._finished_blocks_inbox.get_nowait()
@@ -218,7 +231,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
else:
self._push_finished_blocks[rid] = blocks
# 2b. Evict finished blocks for requests that have either
# 3b. Evict finished blocks for requests that have either
# completed (WRITE acknowledged) or whose lease expired
# without a D registration. Drop pending registrations
# for the same reason so we don't leak state.
@@ -230,7 +243,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
self._push_finished_blocks.pop(rid, None)
self._pending_d_registrations.pop(rid, None)
# 3. NIXL notifs: route PUSH_REG; forward the rest.
# 4. NIXL notifs: route PUSH_REG; forward the rest.
for notifs in self.nixl_wrapper.get_new_notifs().values():
for notif in notifs:
if notif.startswith(PUSH_REG_NOTIF_PREFIX):
@@ -384,34 +397,53 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
) -> None:
"""Start push-based KV transfer from P worker to D node.
``local_block_ids`` are P's *logical* block IDs (from the P
scheduler's metadata). ``registration_data["local_block_ids"]``
are D's *logical* block IDs (from D's scheduler, sent over the
PUSH_REG notif). All conversion to physical block IDs is
deferred to ``_xfer_blocks_for_req`` so each side uses its own
physical-blocks-per-logical ratio (P uses
``self._physical_blocks_per_logical_kv_block``; D's ratio is
learned during the NIXL handshake)."""
decode_engine_id = registration_data["decode_engine_id"]
remote_block_ids = registration_data["local_block_ids"]
decode_host = registration_data["decode_host"]
decode_port = registration_data["decode_port"]
decode_request_id = registration_data["request_id"]
The P→D handshake runs on the base worker's background executor.
If it isn't ready yet we register a completion callback, defer the
WRITE, and re-drive this request via ``_deferred_push_inbox`` once
the handshake resolves -- so the writer thread never blocks on the
network (mirrors ``_send_registration_to_p``).
"""
if not local_block_ids:
logger.warning("No local blocks to push for request %s", request_id)
return
if not self._ensure_d_handshake(
# ``local_block_ids`` are P's logical block IDs; ``remote_block_ids``
# (D's, from the PUSH_REG notif) are also logical.
decode_engine_id = registration_data["decode_engine_id"]
remote_block_ids = registration_data["local_block_ids"]
decode_request_id = registration_data["request_id"]
# Runs on the background executor; defer the WRITE until it's ready.
fut = self._ensure_handshake(
decode_engine_id,
decode_host,
decode_port,
registration_data["decode_host"],
registration_data["decode_port"],
registration_data["decode_tp_size"],
request_id,
):
)
if fut is not None:
def _on_handshake(
f: Future[tuple[dict[tuple[int, int], str], float]],
rid: str = request_id,
blocks: BlockIds = local_block_ids,
rd: dict[str, Any] = registration_data,
) -> None:
if (e := f.exception()) is not None:
# The engine reclaims the blocks via the TTL so we dont free here
self._log_failure(
failure_type="push_handshake_failed", req_id=rid, error=e
)
return
self._deferred_push_inbox.put((rid, blocks, rd))
self._push_writer_wake.set()
fut.add_done_callback(_on_handshake)
return
# Both sides are kept in logical form here; ``_xfer_blocks_for_req``
# expands each side using the appropriate ratio.
# Both sides stay logical here; ``_xfer_blocks_for_req`` converts each
# to physical with its own physical-blocks-per-logical ratio -- P uses
# ``self._physical_blocks_per_logical_kv_block``, D's is learned during
# the NIXL handshake.
logical_local = self._as_grouped_block_ids(local_block_ids)
logical_remote = self._as_grouped_block_ids(remote_block_ids)
physical_local = self._logical_to_kernel_block_ids(
@@ -441,45 +473,6 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
elapsed_ms,
)
def _ensure_d_handshake(
self,
decode_engine_id: str,
decode_host: str,
decode_port: int,
decode_tp_size: int,
request_id: str,
) -> bool:
"""First-time P→D handshake. Blocking call on the writer thread.
Returns True iff the handshake succeeded (or had already been
completed). Returns False if the handshake raised; the request is
skipped in that case (the engine layer will reschedule or fail it
via the standard lease/timeout path)."""
if decode_engine_id in self._remote_agents:
return True
try:
remote_agents, _ = self._nixl_handshake(
decode_host,
decode_port,
decode_tp_size,
decode_engine_id,
)
except Exception:
logger.exception(
"Failed handshake to D %s for push %s",
decode_engine_id,
request_id,
)
return False
with self._handshake_lock:
self._remote_agents[decode_engine_id] = remote_agents
logger.info(
"Push handshake to D %s done (%d agents)",
decode_engine_id,
len(remote_agents),
)
return True
@staticmethod
def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds:
"""Normalise a sequence of block IDs to a tuple-of-groups shape.