diff --git a/docs/design/nixl_kv_push_connector.md b/docs/design/nixl_kv_push_connector.md index b99ba6659f..6f66be5eb9 100644 --- a/docs/design/nixl_kv_push_connector.md +++ b/docs/design/nixl_kv_push_connector.md @@ -47,7 +47,8 @@ sequenceDiagram else only one side present PWriter->>PWriter: stash and wait, self-poll only when blocks unmatched end - PWriter->>PWriter: ensure D handshake (one-time) + PWriter->>PWriter: _ensure_handshake to D (async; defer WRITE) + PWriter->>PWriter: handshake callback re-queues on _deferred_push_inbox, wake PWriter->>DWriter: NIXL WRITE direct to D GPU + completion notif note over DWorker,DWriter: D side - completion accounting @@ -100,15 +101,20 @@ event: D, completion notifs after a WRITE, late-arriving ``PUSH_REG``) even when there is no new metadata to act on. 3. **Handshake-completion callback** (background handshake executor - thread) — when a deferred D→P handshake finishes successfully, the - future's done-callback re-enqueues the registration onto - ``_reg_send_inbox`` and sets the wake so the corresponding - ``send_notif`` runs on the writer (we never call ``send_notif`` from - the executor thread). On this second pass ``_ensure_handshake`` - returns ``None`` (the agent is now connected), so the writer sends - the ``PUSH_REG`` directly. If the handshake *failed*, the callback - fails the request instead of re-enqueuing, so there is no retry - loop. + thread) — both handshakes run on the executor and never block the + writer; their done-callbacks re-enqueue the deferred op and set the + wake, since neither ``send_notif`` nor the NIXL WRITE may run off the + writer thread: + * the **D→P** handshake (before sending ``PUSH_REG``) re-enqueues the + registration onto ``_reg_send_inbox``; + * the **P→D** handshake (before a WRITE) re-enqueues the matched + ``(req_id, blocks, reg_data)`` onto ``_deferred_push_inbox``. + + On this second pass ``_ensure_handshake`` returns ``None`` (the agent + is now connected), so the writer sends the ``PUSH_REG`` / issues the + WRITE directly. If a handshake *failed*, the callback fails or drops + the request instead of re-enqueuing, so there is no retry loop (see + Failure handling). In addition to event-driven wakes, the writer self-polls at ``_PUSH_WRITER_POLL_INTERVAL_MS = 1.0`` ms while there are P-side @@ -227,6 +233,13 @@ Two per-request timers are armed on the scheduler: * **D-side ``send_notif`` failure when shipping the PUSH_REG to P** — identical handling: ``_handle_failed_transfer`` marks the recv as failed. +* **P-side handshake failure (P→D handshake before a WRITE)** — the + future's done-callback logs ``push_handshake_failed`` and drops the + request without re-queuing. It deliberately does *not* call + ``_handle_failed_transfer`` (there is no ``_recving_metadata`` entry to + invalidate on the producer side, same reasoning as the WRITE-submission + failure below). P's blocks are reclaimed by the ``_kv_lease_duration`` + lease and D's stale registration by its watchdog. * **P-side WRITE submission failure** — the WRITE handle (if any) is released and ``xfer_stats.record_failed_transfer()`` bumps the failure counter. We deliberately do not call @@ -246,10 +259,11 @@ existing NIXL connector: class — all subclasses of the existing base classes; * one dedicated background thread per worker; * a few cross-thread queues, each with a single consumer (the writer); - most have one producer, except ``_reg_send_inbox``, which is fed both - by the engine main thread (new registrations) and by the - handshake-completion callback (registrations replayed after their - D→P handshake finishes); + most have one producer, except the two replay queues fed by both the + engine main thread and a handshake-completion callback: + ``_reg_send_inbox`` (registrations replayed after their D→P handshake) + and ``_deferred_push_inbox`` (matched pushes replayed after their P→D + handshake); * one new notification type (`PUSH_REG:`). Behavior on the engine main thread is otherwise unchanged. The writer diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py index ad5ccbbbe8..d670e5563b 100644 --- a/tests/v1/kv_connector/unit/test_nixl_push_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -25,6 +25,7 @@ import queue import threading import time from collections import defaultdict +from concurrent.futures import Future from typing import Any from unittest.mock import MagicMock, patch @@ -320,6 +321,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w._finished_blocks_inbox = queue.Queue() w._pending_completion_notifs = queue.Queue() w._evict_finished_inbox = queue.Queue() + w._deferred_push_inbox = queue.Queue() w._push_writer_wake = threading.Event() w._push_writer_stop = threading.Event() w._push_writer_thread = None @@ -334,6 +336,7 @@ class _StubWriterWorker(NixlPushConnectorWorker): w.world_size = 1 w.engine_id = "test-decode-engine" w._remote_agents = {} + w._physical_blocks_per_logical_kv_block = 1 # Track _do_start_push_kv invocations. calls: list[tuple[str, Any, dict[str, Any]]] = [] @@ -506,6 +509,112 @@ class TestPushWriterStartLoadKv: assert w.start_push_calls == [] +# The P→D handshake must run on the base worker's background executor, never +# blocking the writer thread: ``_do_start_push_kv`` defers the WRITE until the +# handshake resolves, then re-drives via ``_deferred_push_inbox``. These call +# the *real* ``_do_start_push_kv`` (the stub overrides it for matching tests). +def _real_do_start_push_kv(w, *args): + return NixlPushConnectorWorker._do_start_push_kv(w, *args) + + +def test_do_start_push_kv_defers_then_writes_when_handshake_ready(): + """Full happy-path lifecycle: an in-flight handshake defers the WRITE (no + NIXL op from the writer or the executor callback); once it resolves the + request is re-queued on ``_deferred_push_inbox`` with the wake set; and on + re-drive with the handshake ready the WRITE is issued with a correct + ReqMeta.""" + w = _StubWriterWorker.fresh() + w._logical_to_kernel_block_ids = lambda x, ratio: x + xfer_calls: list[dict[str, Any]] = [] + w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw) + + fut: Future = Future() + w._ensure_handshake = lambda *a, **k: fut + + rd = _registration_data("req-hs", decode_engine_id="decode-engine") + _real_do_start_push_kv(w, "req-hs", ([1, 2, 3],), rd) + + # Handshake pending -> nothing issued, nothing queued, no wake. + assert xfer_calls == [] + assert w._deferred_push_inbox.qsize() == 0 + assert not w._push_writer_wake.is_set() + + # Handshake completes: request re-queued for the writer, wake set, but no + # *direct* WRITE from the callback (wrong thread for NIXL ops). + fut.set_result(({(0, 0): "agent"}, 0.0)) + assert xfer_calls == [] + assert w._push_writer_wake.is_set() + rid, blocks, reg = w._deferred_push_inbox.get_nowait() + assert (rid, blocks, reg) == ("req-hs", ([1, 2, 3],), rd) + + # Re-drive on the writer with the handshake now ready -> WRITE issued. + w._ensure_handshake = lambda *a, **k: None + _real_do_start_push_kv(w, rid, blocks, reg) + assert len(xfer_calls) == 1 + assert xfer_calls[0]["req_id"] == "req-hs" + meta = xfer_calls[0]["meta"] + assert meta.remote is not None + assert meta.remote.engine_id == "decode-engine" + # RemoteMeta.request_id is D's request id from the registration. + assert meta.remote.request_id == "req-hs" + + +def test_do_start_push_kv_drops_request_on_handshake_failure(): + """Handshake raises: the request is dropped (not re-queued, no WRITE) and + the failure is logged. Blocks are reclaimed by the lease/watchdog, matching + the old blocking behaviour.""" + w = _StubWriterWorker.fresh() + w._logical_to_kernel_block_ids = lambda x: x + xfer_calls: list[dict[str, Any]] = [] + w._xfer_blocks_for_req = lambda **kw: xfer_calls.append(kw) + failures: list[dict[str, Any]] = [] + w._log_failure = lambda **kw: failures.append(kw) + + fut: Future = Future() + w._ensure_handshake = lambda *a, **k: fut + + _real_do_start_push_kv(w, "req-fail", ([9],), _registration_data("req-fail")) + fut.set_exception(RuntimeError("handshake boom")) + + assert w._deferred_push_inbox.qsize() == 0 + assert xfer_calls == [] + assert len(failures) == 1 + assert failures[0]["failure_type"] == "push_handshake_failed" + + +def test_writer_loop_drains_deferred_push_inbox(): + """The writer loop drains ``_deferred_push_inbox`` and re-drives + ``_do_start_push_kv`` for each entry (event-driven, no polling).""" + w = _StubWriterWorker.fresh() + w.nixl_wrapper = MagicMock() + w.nixl_wrapper.get_new_notifs.return_value = {} + + processed = threading.Event() + + def _tracked(rid, blocks, rd): + w.start_push_calls.append((rid, blocks, rd)) + processed.set() + + w._do_start_push_kv = _tracked # type: ignore[method-assign] + + w._deferred_push_inbox.put( + ("req-retry", ([1, 2],), _registration_data("req-retry")) + ) + w._push_writer_wake.set() + + t = threading.Thread(target=w._push_writer_loop, daemon=True) + t.start() + try: + assert processed.wait(timeout=2.0), "writer did not drain deferred inbox" + finally: + w._push_writer_stop.set() + w._push_writer_wake.set() + t.join(timeout=2) + + assert len(w.start_push_calls) == 1 + assert w.start_push_calls[0][0] == "req-retry" + + class TestPushWriterNotifs: def test_get_new_notifs_processes_forwarded_completion_notif(self): """Non-PUSH_REG notifs forwarded by the writer thread are drained diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py index 2352d9f13b..671f5272db 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -792,7 +792,7 @@ class NixlBaseConnectorWorker: failure_type: str, req_id: str | None, msg: str = "", - error: Exception | None = None, + error: BaseException | None = None, meta: ReqMeta | None = None, **extra_context, ): diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py index 859dbae00b..43b4f89390 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -8,10 +8,13 @@ notifs are forwarded to the engine main thread), sends PUSH_REG via ``send_notif``, matches D registrations with P finished blocks, and issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``. -The engine main thread feeds the writer through three queues: +The engine main thread feeds the writer through queues: ``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox`` (P-side blocks from metadata) and ``_pending_completion_notifs`` (non-PUSH_REG notifs forwarded back for HB / completion accounting). +The handshake-completion callback feeds ``_deferred_push_inbox`` with +matched pushes whose P→D handshake has finished so the writer can +(re-)issue the WRITE without ever blocking on the network. Wake model: the writer self-polls every ``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched @@ -21,8 +24,8 @@ event-driven: the engine main thread sets ``_push_writer_wake`` from ``start_load_kv`` (when handing it new work) and from ``get_finished`` (so each engine step gives the writer a chance to drain NIXL notifs); the handshake-completion callback sets the same event after a deferred -PUSH_REG send has been queued. When a request's lease expires (the base -worker reports it via ``done_sending``) or the WRITE completes, +PUSH_REG send or a deferred push WRITE has been queued. When a request's +lease expires (the base worker reports it via ``done_sending``) or the WRITE completes, ``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so the writer drops any leftover ``_push_finished_blocks`` / ``_pending_d_registrations`` and stops self-polling. @@ -109,6 +112,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): # ``_push_finished_blocks`` so an unmatched entry doesn't keep the # writer busy-polling forever. self._evict_finished_inbox: queue.Queue[str] = queue.Queue() + # Handshakes that have just completed and are ready for the WRITE on wthread + self._deferred_push_inbox = queue.Queue[tuple[str, BlockIds, dict[str, Any]]]() # Wake signal from engine main thread (start_load_kv / get_finished). # Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has @@ -206,7 +211,15 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): break self._send_registration_to_p(rid, rd) - # 2. P-side finished blocks; match against pending regs. + # 2. Deferred P→D pushes whose handshake just completed; do xfer now + while True: + try: + rid, blocks, rd = self._deferred_push_inbox.get_nowait() + except queue.Empty: + break + self._do_start_push_kv(rid, blocks, rd) + + # 3. P-side finished blocks; match against pending regs. while True: try: rid, blocks = self._finished_blocks_inbox.get_nowait() @@ -218,7 +231,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): else: self._push_finished_blocks[rid] = blocks - # 2b. Evict finished blocks for requests that have either + # 3b. Evict finished blocks for requests that have either # completed (WRITE acknowledged) or whose lease expired # without a D registration. Drop pending registrations # for the same reason so we don't leak state. @@ -230,7 +243,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): self._push_finished_blocks.pop(rid, None) self._pending_d_registrations.pop(rid, None) - # 3. NIXL notifs: route PUSH_REG; forward the rest. + # 4. NIXL notifs: route PUSH_REG; forward the rest. for notifs in self.nixl_wrapper.get_new_notifs().values(): for notif in notifs: if notif.startswith(PUSH_REG_NOTIF_PREFIX): @@ -384,34 +397,53 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): ) -> None: """Start push-based KV transfer from P worker to D node. - ``local_block_ids`` are P's *logical* block IDs (from the P - scheduler's metadata). ``registration_data["local_block_ids"]`` - are D's *logical* block IDs (from D's scheduler, sent over the - PUSH_REG notif). All conversion to physical block IDs is - deferred to ``_xfer_blocks_for_req`` so each side uses its own - physical-blocks-per-logical ratio (P uses - ``self._physical_blocks_per_logical_kv_block``; D's ratio is - learned during the NIXL handshake).""" - decode_engine_id = registration_data["decode_engine_id"] - remote_block_ids = registration_data["local_block_ids"] - decode_host = registration_data["decode_host"] - decode_port = registration_data["decode_port"] - decode_request_id = registration_data["request_id"] + The P→D handshake runs on the base worker's background executor. + If it isn't ready yet we register a completion callback, defer the + WRITE, and re-drive this request via ``_deferred_push_inbox`` once + the handshake resolves -- so the writer thread never blocks on the + network (mirrors ``_send_registration_to_p``). + """ if not local_block_ids: logger.warning("No local blocks to push for request %s", request_id) return - if not self._ensure_d_handshake( + # ``local_block_ids`` are P's logical block IDs; ``remote_block_ids`` + # (D's, from the PUSH_REG notif) are also logical. + decode_engine_id = registration_data["decode_engine_id"] + remote_block_ids = registration_data["local_block_ids"] + decode_request_id = registration_data["request_id"] + + # Runs on the background executor; defer the WRITE until it's ready. + fut = self._ensure_handshake( decode_engine_id, - decode_host, - decode_port, + registration_data["decode_host"], + registration_data["decode_port"], registration_data["decode_tp_size"], - request_id, - ): + ) + if fut is not None: + + def _on_handshake( + f: Future[tuple[dict[tuple[int, int], str], float]], + rid: str = request_id, + blocks: BlockIds = local_block_ids, + rd: dict[str, Any] = registration_data, + ) -> None: + if (e := f.exception()) is not None: + # The engine reclaims the blocks via the TTL so we dont free here + self._log_failure( + failure_type="push_handshake_failed", req_id=rid, error=e + ) + return + self._deferred_push_inbox.put((rid, blocks, rd)) + self._push_writer_wake.set() + + fut.add_done_callback(_on_handshake) return - # Both sides are kept in logical form here; ``_xfer_blocks_for_req`` - # expands each side using the appropriate ratio. + # Both sides stay logical here; ``_xfer_blocks_for_req`` converts each + # to physical with its own physical-blocks-per-logical ratio -- P uses + # ``self._physical_blocks_per_logical_kv_block``, D's is learned during + # the NIXL handshake. logical_local = self._as_grouped_block_ids(local_block_ids) logical_remote = self._as_grouped_block_ids(remote_block_ids) physical_local = self._logical_to_kernel_block_ids( @@ -441,45 +473,6 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker): elapsed_ms, ) - def _ensure_d_handshake( - self, - decode_engine_id: str, - decode_host: str, - decode_port: int, - decode_tp_size: int, - request_id: str, - ) -> bool: - """First-time P→D handshake. Blocking call on the writer thread. - - Returns True iff the handshake succeeded (or had already been - completed). Returns False if the handshake raised; the request is - skipped in that case (the engine layer will reschedule or fail it - via the standard lease/timeout path).""" - if decode_engine_id in self._remote_agents: - return True - try: - remote_agents, _ = self._nixl_handshake( - decode_host, - decode_port, - decode_tp_size, - decode_engine_id, - ) - except Exception: - logger.exception( - "Failed handshake to D %s for push %s", - decode_engine_id, - request_id, - ) - return False - with self._handshake_lock: - self._remote_agents[decode_engine_id] = remote_agents - logger.info( - "Push handshake to D %s done (%d agents)", - decode_engine_id, - len(remote_agents), - ) - return True - @staticmethod def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds: """Normalise a sequence of block IDs to a tuple-of-groups shape.