[PD][NixlPush][Bugfix] Fix blocking handshake call on writer thread (#49221)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5c5434e2d8
commit
589a5b884b
@@ -47,7 +47,8 @@ sequenceDiagram
|
||||
else only one side present
|
||||
PWriter->>PWriter: stash and wait, self-poll only when blocks unmatched
|
||||
end
|
||||
PWriter->>PWriter: ensure D handshake (one-time)
|
||||
PWriter->>PWriter: _ensure_handshake to D (async; defer WRITE)
|
||||
PWriter->>PWriter: handshake callback re-queues on _deferred_push_inbox, wake
|
||||
PWriter->>DWriter: NIXL WRITE direct to D GPU + completion notif
|
||||
|
||||
note over DWorker,DWriter: D side - completion accounting
|
||||
@@ -100,15 +101,20 @@ event:
|
||||
D, completion notifs after a WRITE, late-arriving ``PUSH_REG``)
|
||||
even when there is no new metadata to act on.
|
||||
3. **Handshake-completion callback** (background handshake executor
|
||||
thread) — when a deferred D→P handshake finishes successfully, the
|
||||
future's done-callback re-enqueues the registration onto
|
||||
``_reg_send_inbox`` and sets the wake so the corresponding
|
||||
``send_notif`` runs on the writer (we never call ``send_notif`` from
|
||||
the executor thread). On this second pass ``_ensure_handshake``
|
||||
returns ``None`` (the agent is now connected), so the writer sends
|
||||
the ``PUSH_REG`` directly. If the handshake *failed*, the callback
|
||||
fails the request instead of re-enqueuing, so there is no retry
|
||||
loop.
|
||||
thread) — both handshakes run on the executor and never block the
|
||||
writer; their done-callbacks re-enqueue the deferred op and set the
|
||||
wake, since neither ``send_notif`` nor the NIXL WRITE may run off the
|
||||
writer thread:
|
||||
* the **D→P** handshake (before sending ``PUSH_REG``) re-enqueues the
|
||||
registration onto ``_reg_send_inbox``;
|
||||
* the **P→D** handshake (before a WRITE) re-enqueues the matched
|
||||
``(req_id, blocks, reg_data)`` onto ``_deferred_push_inbox``.
|
||||
|
||||
On this second pass ``_ensure_handshake`` returns ``None`` (the agent
|
||||
is now connected), so the writer sends the ``PUSH_REG`` / issues the
|
||||
WRITE directly. If a handshake *failed*, the callback fails or drops
|
||||
the request instead of re-enqueuing, so there is no retry loop (see
|
||||
Failure handling).
|
||||
|
||||
In addition to event-driven wakes, the writer self-polls at
|
||||
``_PUSH_WRITER_POLL_INTERVAL_MS = 1.0`` ms while there are P-side
|
||||
@@ -227,6 +233,13 @@ Two per-request timers are armed on the scheduler:
|
||||
* **D-side ``send_notif`` failure when shipping the PUSH_REG to P** —
|
||||
identical handling: ``_handle_failed_transfer`` marks the recv as
|
||||
failed.
|
||||
* **P-side handshake failure (P→D handshake before a WRITE)** — the
|
||||
future's done-callback logs ``push_handshake_failed`` and drops the
|
||||
request without re-queuing. It deliberately does *not* call
|
||||
``_handle_failed_transfer`` (there is no ``_recving_metadata`` entry to
|
||||
invalidate on the producer side, same reasoning as the WRITE-submission
|
||||
failure below). P's blocks are reclaimed by the ``_kv_lease_duration``
|
||||
lease and D's stale registration by its watchdog.
|
||||
* **P-side WRITE submission failure** — the WRITE handle (if any) is
|
||||
released and ``xfer_stats.record_failed_transfer()`` bumps the
|
||||
failure counter. We deliberately do not call
|
||||
@@ -246,10 +259,11 @@ existing NIXL connector:
|
||||
class — all subclasses of the existing base classes;
|
||||
* one dedicated background thread per worker;
|
||||
* a few cross-thread queues, each with a single consumer (the writer);
|
||||
most have one producer, except ``_reg_send_inbox``, which is fed both
|
||||
by the engine main thread (new registrations) and by the
|
||||
handshake-completion callback (registrations replayed after their
|
||||
D→P handshake finishes);
|
||||
most have one producer, except the two replay queues fed by both the
|
||||
engine main thread and a handshake-completion callback:
|
||||
``_reg_send_inbox`` (registrations replayed after their D→P handshake)
|
||||
and ``_deferred_push_inbox`` (matched pushes replayed after their P→D
|
||||
handshake);
|
||||
* one new notification type (`PUSH_REG:<msgpack>`).
|
||||
|
||||
Behavior on the engine main thread is otherwise unchanged. The writer
|
||||
|
||||
@@ -25,6 +25,7 @@ import queue
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import Future
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -320,6 +321,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w._finished_blocks_inbox = queue.Queue()
|
||||
w._pending_completion_notifs = queue.Queue()
|
||||
w._evict_finished_inbox = queue.Queue()
|
||||
w._deferred_push_inbox = queue.Queue()
|
||||
w._push_writer_wake = threading.Event()
|
||||
w._push_writer_stop = threading.Event()
|
||||
w._push_writer_thread = None
|
||||
@@ -334,6 +336,7 @@ class _StubWriterWorker(NixlPushConnectorWorker):
|
||||
w.world_size = 1
|
||||
w.engine_id = "test-decode-engine"
|
||||
w._remote_agents = {}
|
||||
w._physical_blocks_per_logical_kv_block = 1
|
||||
|
||||
# Track _do_start_push_kv invocations.
|
||||
calls: list[tuple[str, Any, dict[str, Any]]] = []
|
||||
@@ -506,6 +509,112 @@ class TestPushWriterStartLoadKv:
|
||||
assert w.start_push_calls == []
|
||||
|
||||
|
||||
# The P→D handshake must run on the base worker's background executor, never
|
||||
# blocking the writer thread: ``_do_start_push_kv`` defers the WRITE until the
|
||||
# handshake resolves, then re-drives via ``_deferred_push_inbox``. These call
|
||||
# the *real* ``_do_start_push_kv`` (the stub overrides it for matching tests).
|
||||
def _real_do_start_push_kv(w, *args):
|
||||
return NixlPushConnectorWorker._do_start_push_kv(w, *args)
|
||||
|
||||
|
||||
def test_do_start_push_kv_defers_then_writes_when_handshake_ready():
|
||||
"""Full happy-path lifecycle: an in-flight handshake defers the WRITE (no
|
||||
NIXL op from the writer or the executor callback); once it resolves the
|
||||
request is re-queued on ``_deferred_push_inbox`` with the wake set; and on
|
||||
re-drive with the handshake ready the WRITE is issued with a correct
|
||||
ReqMeta."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._logical_to_kernel_block_ids = lambda x, ratio: x
|
||||
xfer_calls: list[dict[str, Any]] = []
|
||||
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
|
||||
|
||||
fut: Future = Future()
|
||||
w._ensure_handshake = lambda *a, **k: fut
|
||||
|
||||
rd = _registration_data("req-hs", decode_engine_id="decode-engine")
|
||||
_real_do_start_push_kv(w, "req-hs", ([1, 2, 3],), rd)
|
||||
|
||||
# Handshake pending -> nothing issued, nothing queued, no wake.
|
||||
assert xfer_calls == []
|
||||
assert w._deferred_push_inbox.qsize() == 0
|
||||
assert not w._push_writer_wake.is_set()
|
||||
|
||||
# Handshake completes: request re-queued for the writer, wake set, but no
|
||||
# *direct* WRITE from the callback (wrong thread for NIXL ops).
|
||||
fut.set_result(({(0, 0): "agent"}, 0.0))
|
||||
assert xfer_calls == []
|
||||
assert w._push_writer_wake.is_set()
|
||||
rid, blocks, reg = w._deferred_push_inbox.get_nowait()
|
||||
assert (rid, blocks, reg) == ("req-hs", ([1, 2, 3],), rd)
|
||||
|
||||
# Re-drive on the writer with the handshake now ready -> WRITE issued.
|
||||
w._ensure_handshake = lambda *a, **k: None
|
||||
_real_do_start_push_kv(w, rid, blocks, reg)
|
||||
assert len(xfer_calls) == 1
|
||||
assert xfer_calls[0]["req_id"] == "req-hs"
|
||||
meta = xfer_calls[0]["meta"]
|
||||
assert meta.remote is not None
|
||||
assert meta.remote.engine_id == "decode-engine"
|
||||
# RemoteMeta.request_id is D's request id from the registration.
|
||||
assert meta.remote.request_id == "req-hs"
|
||||
|
||||
|
||||
def test_do_start_push_kv_drops_request_on_handshake_failure():
|
||||
"""Handshake raises: the request is dropped (not re-queued, no WRITE) and
|
||||
the failure is logged. Blocks are reclaimed by the lease/watchdog, matching
|
||||
the old blocking behaviour."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w._logical_to_kernel_block_ids = lambda x: x
|
||||
xfer_calls: list[dict[str, Any]] = []
|
||||
w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw)
|
||||
failures: list[dict[str, Any]] = []
|
||||
w._log_failure = lambda **kw: failures.append(kw)
|
||||
|
||||
fut: Future = Future()
|
||||
w._ensure_handshake = lambda *a, **k: fut
|
||||
|
||||
_real_do_start_push_kv(w, "req-fail", ([9],), _registration_data("req-fail"))
|
||||
fut.set_exception(RuntimeError("handshake boom"))
|
||||
|
||||
assert w._deferred_push_inbox.qsize() == 0
|
||||
assert xfer_calls == []
|
||||
assert len(failures) == 1
|
||||
assert failures[0]["failure_type"] == "push_handshake_failed"
|
||||
|
||||
|
||||
def test_writer_loop_drains_deferred_push_inbox():
|
||||
"""The writer loop drains ``_deferred_push_inbox`` and re-drives
|
||||
``_do_start_push_kv`` for each entry (event-driven, no polling)."""
|
||||
w = _StubWriterWorker.fresh()
|
||||
w.nixl_wrapper = MagicMock()
|
||||
w.nixl_wrapper.get_new_notifs.return_value = {}
|
||||
|
||||
processed = threading.Event()
|
||||
|
||||
def _tracked(rid, blocks, rd):
|
||||
w.start_push_calls.append((rid, blocks, rd))
|
||||
processed.set()
|
||||
|
||||
w._do_start_push_kv = _tracked # type: ignore[method-assign]
|
||||
|
||||
w._deferred_push_inbox.put(
|
||||
("req-retry", ([1, 2],), _registration_data("req-retry"))
|
||||
)
|
||||
w._push_writer_wake.set()
|
||||
|
||||
t = threading.Thread(target=w._push_writer_loop, daemon=True)
|
||||
t.start()
|
||||
try:
|
||||
assert processed.wait(timeout=2.0), "writer did not drain deferred inbox"
|
||||
finally:
|
||||
w._push_writer_stop.set()
|
||||
w._push_writer_wake.set()
|
||||
t.join(timeout=2)
|
||||
|
||||
assert len(w.start_push_calls) == 1
|
||||
assert w.start_push_calls[0][0] == "req-retry"
|
||||
|
||||
|
||||
class TestPushWriterNotifs:
|
||||
def test_get_new_notifs_processes_forwarded_completion_notif(self):
|
||||
"""Non-PUSH_REG notifs forwarded by the writer thread are drained
|
||||
|
||||
@@ -792,7 +792,7 @@ class NixlBaseConnectorWorker:
|
||||
failure_type: str,
|
||||
req_id: str | None,
|
||||
msg: str = "",
|
||||
error: Exception | None = None,
|
||||
error: BaseException | None = None,
|
||||
meta: ReqMeta | None = None,
|
||||
**extra_context,
|
||||
):
|
||||
|
||||
@@ -8,10 +8,13 @@ notifs are forwarded to the engine main thread), sends PUSH_REG via
|
||||
``send_notif``, matches D registrations with P finished blocks, and
|
||||
issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``.
|
||||
|
||||
The engine main thread feeds the writer through three queues:
|
||||
The engine main thread feeds the writer through queues:
|
||||
``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox``
|
||||
(P-side blocks from metadata) and ``_pending_completion_notifs``
|
||||
(non-PUSH_REG notifs forwarded back for HB / completion accounting).
|
||||
The handshake-completion callback feeds ``_deferred_push_inbox`` with
|
||||
matched pushes whose P→D handshake has finished so the writer can
|
||||
(re-)issue the WRITE without ever blocking on the network.
|
||||
|
||||
Wake model: the writer self-polls every
|
||||
``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched
|
||||
@@ -21,8 +24,8 @@ event-driven: the engine main thread sets ``_push_writer_wake`` from
|
||||
``start_load_kv`` (when handing it new work) and from ``get_finished``
|
||||
(so each engine step gives the writer a chance to drain NIXL notifs);
|
||||
the handshake-completion callback sets the same event after a deferred
|
||||
PUSH_REG send has been queued. When a request's lease expires (the base
|
||||
worker reports it via ``done_sending``) or the WRITE completes,
|
||||
PUSH_REG send or a deferred push WRITE has been queued. When a request's
|
||||
lease expires (the base worker reports it via ``done_sending``) or the WRITE completes,
|
||||
``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so
|
||||
the writer drops any leftover ``_push_finished_blocks`` /
|
||||
``_pending_d_registrations`` and stops self-polling.
|
||||
@@ -109,6 +112,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# ``_push_finished_blocks`` so an unmatched entry doesn't keep the
|
||||
# writer busy-polling forever.
|
||||
self._evict_finished_inbox: queue.Queue[str] = queue.Queue()
|
||||
# Handshakes that have just completed and are ready for the WRITE on wthread
|
||||
self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]()
|
||||
|
||||
# Wake signal from engine main thread (start_load_kv / get_finished).
|
||||
# Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has
|
||||
@@ -206,7 +211,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
break
|
||||
self._send_registration_to_p(rid, rd)
|
||||
|
||||
# 2. P-side finished blocks; match against pending regs.
|
||||
# 2. Deferred P→D pushes whose handshake just completed; do xfer now
|
||||
while True:
|
||||
try:
|
||||
rid, blocks, rd = self._deferred_push_inbox.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self._do_start_push_kv(rid, blocks, rd)
|
||||
|
||||
# 3. P-side finished blocks; match against pending regs.
|
||||
while True:
|
||||
try:
|
||||
rid, blocks = self._finished_blocks_inbox.get_nowait()
|
||||
@@ -218,7 +231,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
else:
|
||||
self._push_finished_blocks[rid] = blocks
|
||||
|
||||
# 2b. Evict finished blocks for requests that have either
|
||||
# 3b. Evict finished blocks for requests that have either
|
||||
# completed (WRITE acknowledged) or whose lease expired
|
||||
# without a D registration. Drop pending registrations
|
||||
# for the same reason so we don't leak state.
|
||||
@@ -230,7 +243,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._push_finished_blocks.pop(rid, None)
|
||||
self._pending_d_registrations.pop(rid, None)
|
||||
|
||||
# 3. NIXL notifs: route PUSH_REG; forward the rest.
|
||||
# 4. NIXL notifs: route PUSH_REG; forward the rest.
|
||||
for notifs in self.nixl_wrapper.get_new_notifs().values():
|
||||
for notif in notifs:
|
||||
if notif.startswith(PUSH_REG_NOTIF_PREFIX):
|
||||
@@ -384,34 +397,53 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
) -> None:
|
||||
"""Start push-based KV transfer from P worker to D node.
|
||||
|
||||
``local_block_ids`` are P's *logical* block IDs (from the P
|
||||
scheduler's metadata). ``registration_data["local_block_ids"]``
|
||||
are D's *logical* block IDs (from D's scheduler, sent over the
|
||||
PUSH_REG notif). All conversion to physical block IDs is
|
||||
deferred to ``_xfer_blocks_for_req`` so each side uses its own
|
||||
physical-blocks-per-logical ratio (P uses
|
||||
``self._physical_blocks_per_logical_kv_block``; D's ratio is
|
||||
learned during the NIXL handshake)."""
|
||||
decode_engine_id = registration_data["decode_engine_id"]
|
||||
remote_block_ids = registration_data["local_block_ids"]
|
||||
decode_host = registration_data["decode_host"]
|
||||
decode_port = registration_data["decode_port"]
|
||||
decode_request_id = registration_data["request_id"]
|
||||
The P→D handshake runs on the base worker's background executor.
|
||||
If it isn't ready yet we register a completion callback, defer the
|
||||
WRITE, and re-drive this request via ``_deferred_push_inbox`` once
|
||||
the handshake resolves -- so the writer thread never blocks on the
|
||||
network (mirrors ``_send_registration_to_p``).
|
||||
"""
|
||||
if not local_block_ids:
|
||||
logger.warning("No local blocks to push for request %s", request_id)
|
||||
return
|
||||
|
||||
if not self._ensure_d_handshake(
|
||||
# ``local_block_ids`` are P's logical block IDs; ``remote_block_ids``
|
||||
# (D's, from the PUSH_REG notif) are also logical.
|
||||
decode_engine_id = registration_data["decode_engine_id"]
|
||||
remote_block_ids = registration_data["local_block_ids"]
|
||||
decode_request_id = registration_data["request_id"]
|
||||
|
||||
# Runs on the background executor; defer the WRITE until it's ready.
|
||||
fut = self._ensure_handshake(
|
||||
decode_engine_id,
|
||||
decode_host,
|
||||
decode_port,
|
||||
registration_data["decode_host"],
|
||||
registration_data["decode_port"],
|
||||
registration_data["decode_tp_size"],
|
||||
request_id,
|
||||
):
|
||||
)
|
||||
if fut is not None:
|
||||
|
||||
def _on_handshake(
|
||||
f: Future[tuple[dict[tuple[int, int], str], float]],
|
||||
rid: str = request_id,
|
||||
blocks: BlockIds = local_block_ids,
|
||||
rd: dict[str, Any] = registration_data,
|
||||
) -> None:
|
||||
if (e := f.exception()) is not None:
|
||||
# The engine reclaims the blocks via the TTL so we dont free here
|
||||
self._log_failure(
|
||||
failure_type="push_handshake_failed", req_id=rid, error=e
|
||||
)
|
||||
return
|
||||
self._deferred_push_inbox.put((rid, blocks, rd))
|
||||
self._push_writer_wake.set()
|
||||
|
||||
fut.add_done_callback(_on_handshake)
|
||||
return
|
||||
|
||||
# Both sides are kept in logical form here; ``_xfer_blocks_for_req``
|
||||
# expands each side using the appropriate ratio.
|
||||
# Both sides stay logical here; ``_xfer_blocks_for_req`` converts each
|
||||
# to physical with its own physical-blocks-per-logical ratio -- P uses
|
||||
# ``self._physical_blocks_per_logical_kv_block``, D's is learned during
|
||||
# the NIXL handshake.
|
||||
logical_local = self._as_grouped_block_ids(local_block_ids)
|
||||
logical_remote = self._as_grouped_block_ids(remote_block_ids)
|
||||
physical_local = self._logical_to_kernel_block_ids(
|
||||
@@ -441,45 +473,6 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
elapsed_ms,
|
||||
)
|
||||
|
||||
def _ensure_d_handshake(
|
||||
self,
|
||||
decode_engine_id: str,
|
||||
decode_host: str,
|
||||
decode_port: int,
|
||||
decode_tp_size: int,
|
||||
request_id: str,
|
||||
) -> bool:
|
||||
"""First-time P→D handshake. Blocking call on the writer thread.
|
||||
|
||||
Returns True iff the handshake succeeded (or had already been
|
||||
completed). Returns False if the handshake raised; the request is
|
||||
skipped in that case (the engine layer will reschedule or fail it
|
||||
via the standard lease/timeout path)."""
|
||||
if decode_engine_id in self._remote_agents:
|
||||
return True
|
||||
try:
|
||||
remote_agents, _ = self._nixl_handshake(
|
||||
decode_host,
|
||||
decode_port,
|
||||
decode_tp_size,
|
||||
decode_engine_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed handshake to D %s for push %s",
|
||||
decode_engine_id,
|
||||
request_id,
|
||||
)
|
||||
return False
|
||||
with self._handshake_lock:
|
||||
self._remote_agents[decode_engine_id] = remote_agents
|
||||
logger.info(
|
||||
"Push handshake to D %s done (%d agents)",
|
||||
decode_engine_id,
|
||||
len(remote_agents),
|
||||
)
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds:
|
||||
"""Normalise a sequence of block IDs to a tuple-of-groups shape.
|
||||
|
||||
Reference in New Issue
Block a user