From 88ed63621866d1e4bdaacc560c911f7b8859c53d Mon Sep 17 00:00:00 2001 From: snadampal <87143774+snadampal@users.noreply.github.com> Date: Fri, 12 Jun 2026 05:38:41 -0500 Subject: [PATCH] [KV Connector]: Support KV push from Prefill to Decode node using Nixl KV Connector (#35264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sunita Nadampalli Signed-off-by: NickLucche Co-authored-by: Nicolò Lucchesi --- docs/design/nixl_kv_push_connector.md | 256 ++ .../disagg_proxy_pushconnector_demo.py | 429 +++ .../unit/test_bidirectional_kv_transfer.py | 8 +- .../kv_connector/unit/test_multi_connector.py | 7 +- .../kv_connector/unit/test_nixl_connector.py | 79 +- .../unit/test_nixl_connector_hma.py | 8 +- .../unit/test_nixl_push_connector.py | 815 +++++ .../unit/test_nixl_simple_cpu_offload.py | 2 +- .../unit/test_remote_prefill_lifecycle.py | 4 +- tests/v1/kv_connector/unit/utils.py | 63 + .../kv_transfer/kv_connector/factory.py | 12 + .../kv_transfer/kv_connector/v1/base.py | 12 + .../kv_connector/v1/multi_connector.py | 3 + .../kv_connector/v1/nixl/__init__.py | 30 + .../kv_connector/v1/nixl/base_scheduler.py | 455 +++ .../kv_connector/v1/nixl/base_worker.py | 2286 ++++++++++++++ .../kv_connector/v1/nixl/connector.py | 137 +- .../kv_connector/v1/nixl/metadata.py | 14 + .../kv_connector/v1/nixl/pull_scheduler.py | 275 ++ .../kv_connector/v1/nixl/pull_worker.py | 382 +++ .../kv_connector/v1/nixl/push_scheduler.py | 348 +++ .../kv_connector/v1/nixl/push_worker.py | 742 +++++ .../kv_connector/v1/nixl/scheduler.py | 674 +---- .../kv_transfer/kv_connector/v1/nixl/utils.py | 11 + .../kv_connector/v1/nixl/worker.py | 2640 +---------------- vllm/v1/core/sched/scheduler.py | 13 + 26 files changed, 6335 insertions(+), 3370 deletions(-) create mode 100644 docs/design/nixl_kv_push_connector.md create mode 100644 examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py create mode 100644 tests/v1/kv_connector/unit/test_nixl_push_connector.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py create mode 100644 vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py diff --git a/docs/design/nixl_kv_push_connector.md b/docs/design/nixl_kv_push_connector.md new file mode 100644 index 0000000000..b99ba6659f --- /dev/null +++ b/docs/design/nixl_kv_push_connector.md @@ -0,0 +1,256 @@ +# NIXL push-mode KV transfer + +The default NIXL connector is **pull-based**: the decode (D) instance +reads KV blocks from the prefill (P) instance via `NIXL READ` after +prefill completes. `NixlPushConnector` adds a **push-based** alternative +in which P writes the KV blocks directly into D's pre-allocated memory +via `NIXL WRITE`. + +This document describes the threading, queues, and scheduling +interactions specific to the push design. The pull-mode design is +unchanged; the push connector reuses the same handshake, NIXL agent +setup, and metadata path wherever possible. + +## High-level flow + +```mermaid +sequenceDiagram + autonumber + participant Client + participant Proxy + participant DSched as D Scheduler + participant DWorker as D Worker (main) + participant DWriter as D Writer + participant PWriter as P Writer + participant PWorker as P Worker (main) + participant PSched as P Scheduler + + Client->>Proxy: POST /v1/completions + Proxy->>PSched: prefill leg (do_remote_decode=True, max_tokens=1) + Proxy->>DSched: decode leg (do_remote_prefill=True, P coordinates) + + note over DSched,DWriter: D side - register blocks with P + DSched->>DSched: update_state_after_alloc, stash registration, arm watchdog + DSched->>DWorker: build_connector_meta -> meta.push_registrations + DWorker->>DWriter: enqueue (req_id, reg_data) on _reg_send_inbox + DWriter->>PWriter: NIXL send_notif PUSH_REG msgpack + + note over PSched,PWriter: P side - prefill, stage finished blocks + PSched->>PSched: request_finished, stash blocks + PSched->>PWorker: build_connector_meta -> meta.push_finished_blocks + PWorker->>PWriter: enqueue (req_id, blocks) on _finished_blocks_inbox + + note over PWriter: P writer matches and WRITEs + PWriter->>PWriter: get_new_notifs returns PUSH_REG, route via _handle_push_reg_notif + alt PUSH_REG and finished blocks both present + PWriter->>PWriter: pop matching pair, fire WRITE + else only one side present + PWriter->>PWriter: stash and wait, self-poll only when blocks unmatched + end + PWriter->>PWriter: ensure D handshake (one-time) + PWriter->>DWriter: NIXL WRITE direct to D GPU + completion notif + + note over DWorker,DWriter: D side - completion accounting + DWriter-->>DWorker: forward HB and completion notifs via _pending_completion_notifs + DWorker->>DWorker: _get_new_notifs drains, HB extends lease, completion marks recv done + DWorker->>DSched: update_connector_output(finished_recving) + DSched->>DSched: clear watchdog deadline + + note over PWorker,PWriter: P side - reclaim + PWorker->>PWorker: get_finished, drain _sending_transfers, queue eviction + PWriter->>PWriter: drain _evict_finished_inbox, drop stale state + PWorker->>PSched: update_connector_output(finished_sending) + PSched->>PSched: free lease + + DWorker-->>Proxy: stream decode tokens + Proxy-->>Client: response +``` + +## Threads + +``NixlPushConnectorWorker`` introduces a single dedicated background +thread per worker (i.e. per TP rank), named ``nixl-push-writer``. +Each owns the new push-specific NIXL operations on its rank: + +* ``nixl_wrapper.get_new_notifs()`` — receive notifications. +* ``nixl_wrapper.send_notif(...)`` for the ``PUSH_REG:`` (D + side) and for the per-WRITE completion notif (P side). +* ``nixl_wrapper.make_prepped_xfer(...) / transfer(...)`` — submit the + WRITE itself. + +Heartbeats continue to go out from the engine main thread via the +existing base-worker ``_send_heartbeats`` plumbing inside +``start_load_kv``. + +### Wake model + +The writer thread blocks on ``_push_writer_wake`` (a +``threading.Event``) when it has no work. Three callers set the +event: + +1. **``start_load_kv``** (worker main thread, called once per engine + step with the scheduler's metadata) — sets the wake only when the + step actually hands the writer new work, i.e. when + ``meta.push_registrations`` or ``meta.push_finished_blocks`` is + non-empty. This is the wake for new transfers. +2. **``get_finished``** (worker main thread, called once per engine + step to report completions) — always sets the wake. The writer is + the sole consumer of ``nixl_wrapper.get_new_notifs()`` for push, + so this gives it a chance to drain inbound notifs (heartbeats from + D, completion notifs after a WRITE, late-arriving ``PUSH_REG``) + even when there is no new metadata to act on. +3. **Handshake-completion callback** (background handshake executor + thread) — when a deferred D→P handshake finishes successfully, the + future's done-callback re-enqueues the registration onto + ``_reg_send_inbox`` and sets the wake so the corresponding + ``send_notif`` runs on the writer (we never call ``send_notif`` from + the executor thread). On this second pass ``_ensure_handshake`` + returns ``None`` (the agent is now connected), so the writer sends + the ``PUSH_REG`` directly. If the handshake *failed*, the callback + fails the request instead of re-enqueuing, so there is no retry + loop. + +In addition to event-driven wakes, the writer self-polls at +``_PUSH_WRITER_POLL_INTERVAL_MS = 1.0`` ms while there are P-side +finished blocks waiting for an unmatched ``PUSH_REG``. + +When a request completes on P (lease expires or the WRITE finishes), +``get_finished`` enqueues the request id onto ``_evict_finished_inbox``, +which the writer drains to drop stale ``_push_finished_blocks`` / +``_pending_d_registrations`` and stop self-polling. + +## Writer-local matching tables + +| Table | Owner | Holds | +|--------------------------------|------------------|------------------------------------------------------------------------| +| `_pending_d_registrations` | writer | D registrations received from a remote D, waiting for P's blocks | +| `_push_finished_blocks` | writer | P blocks staged by the scheduler, waiting for a remote D registration | + +Either side can arrive first. The writer matches in both directions: +when a ``PUSH_REG`` arrives we look up ``_push_finished_blocks``, and +when finished blocks arrive we look up ``_pending_d_registrations``. +Both lookups try an exact ``request_id`` match first, then fall back +to comparing the ids after stripping the trailing per-engine random +suffix (via ``get_base_request_id``). The fallback exists because the +proxy hands the same ``X-Request-Id`` to both legs, so P and D wrap it +into the same ``cmpl--`` form and differ only by the +8-hex randomization suffix that ``input_processor.assign_request_id`` +appends per engine. Stripping just that suffix normalizes both sides +to the same id while preserving the completion index (so multi-prompt +sub-requests stay distinct). It also works whether or not +``VLLM_DISABLE_REQUEST_ID_RANDOMIZATION`` is set, which matters since +that env var is slated for removal upstream. + +## Wire format + +A push registration is sent as a NIXL notification: + +```text +PUSH_REG: +``` + +Fields in the dict: + +| Field | Set by | Meaning | +|----------------------|--------|------------------------------------------------------------------------| +| ``request_id`` | D | D's own vLLM request id; P's match key, echoed in the completion notif | +| ``decode_engine_id`` | D | D's engine id (P uses this for the reverse handshake) | +| ``decode_host`` | D | D's NIXL side-channel host | +| ``decode_port`` | D | D's NIXL side-channel port | +| ``decode_tp_size`` | D | D's tensor-parallel size | +| ``local_block_ids`` | D | per-group lists of D's *logical* block ids (preallocated) | +| ``remote_engine_id`` | D | P's engine id (for the existing P-side handshake) | +| ``remote_host`` | D | P's NIXL side-channel host | +| ``remote_port`` | D | P's NIXL side-channel port | +| ``remote_tp_size`` | D | P's tensor-parallel size | + +D ships **logical** block ids; P expands them to physical block ids at +WRITE-submission time using the ratio learned during the NIXL +handshake (`remote_physical_blocks_per_logical`). This matches the +pull-mode contract — schedulers ship logical ids, workers expand to +physical at submission. + +The completion notif sent from P to D after a WRITE is the existing +`:` format used in pull mode (here ``request_id`` +is D's own request id, taken from the registration), so the D-side +accounting code is unchanged. + +## Scheduler-side responsibilities + +`NixlPushConnectorScheduler` extends the base scheduler with: + +* **D side** — `update_state_after_alloc` stashes registration data in + `_push_pending_registrations` and arms a soft watchdog + (`_push_registration_deadlines`). `build_connector_meta` drains the + stash into `meta.push_registrations` and any expired entries are + dropped with a warning. +* **P side** — `request_finished` stashes block IDs in + `_finished_request_blocks` (for the lease and for + `has_pending_push_work`) and `_newly_finished_push_blocks` (for the + next worker step via `meta.push_finished_blocks`). +* **Both sides** — `has_pending_push_work` keeps the engine main loop + stepping while there is in-flight push state, so the writer always + gets at least one wake per step. + +`update_connector_output`: + +* `finished_sending` (P side) clears the lease entry. +* `finished_recving` (D side) clears the watchdog deadline. + +## Timeouts and watchdogs + +Two per-request timers are armed on the scheduler: + +* **D-side registration watchdog** — ``_push_registration_deadlines``. + If a registered request does not see a push completion within + ``push_registration_timeout`` seconds (defaults to + ``decoder_kv_blocks_ttl``), ``build_connector_meta`` drops the stale + registration and the pending entry, logs a warning, and stops trying + to resend the registration. The corresponding request remains tracked + in ``_reqs_need_recv``; it is the engine's request-level abort path + (or the user / proxy timing out the HTTP call) that ultimately fails + the request. +* **P-side block lease** — same ``_kv_lease_duration`` used by pull + mode. ``request_finished`` sets the expiration in ``_reqs_need_send`` + and ``update_connector_output(finished_sending=...)`` clears it on + successful WRITE. Stale leases are reaped by ``get_finished`` in the + base worker, which then enqueues the eviction onto + ``_evict_finished_inbox`` so the writer also stops self-polling. + +## Failure handling + +* **D-side handshake failure (P→D handshake before sending PUSH_REG)** — + the future's done-callback calls ``_handle_failed_transfer(rid, None)``, + which marks D's pre-allocated blocks invalid and enqueues onto + ``_failed_recv_reqs`` so the next ``get_finished`` reports the + request as a failed recv. Same recv-side accounting as pull mode. +* **D-side ``send_notif`` failure when shipping the PUSH_REG to P** — + identical handling: ``_handle_failed_transfer`` marks the recv as + failed. +* **P-side WRITE submission failure** — the WRITE handle (if any) is + released and ``xfer_stats.record_failed_transfer()`` bumps the + failure counter. We deliberately do not call + ``_handle_failed_transfer`` here: ``req_id`` on the P side has no + entry in ``_recving_metadata`` (P is not the receiver), so the + helper would put a P-local request id into ``_failed_recv_reqs`` + and trip the assertion in the base worker's ``get_finished``. The + outbound WRITE is dropped on the floor; D's lease watchdog handles + the missing completion. + +## Summary + +The push design is a small, well-contained extension on top of the +existing NIXL connector: + +* one new connector class, one new scheduler class, one new worker + class — all subclasses of the existing base classes; +* one dedicated background thread per worker; +* a few cross-thread queues, each with a single consumer (the writer); + most have one producer, except ``_reg_send_inbox``, which is fed both + by the engine main thread (new registrations) and by the + handshake-completion callback (registrations replayed after their + D→P handshake finishes); +* one new notification type (`PUSH_REG:`). + +Behavior on the engine main thread is otherwise unchanged. The writer +thread is event-driven and idle when there is no push work. diff --git a/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py b/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py new file mode 100644 index 0000000000..9f1a0a7f41 --- /dev/null +++ b/examples/disaggregated/disaggregated_serving/disagg_proxy_pushconnector_demo.py @@ -0,0 +1,429 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Push-mode disaggregated prefilling proxy demo. + +Companion to ``disagg_proxy_demo.py`` (pull mode). The client-facing API is +the same; the difference is in how P and D coordinate the KV transfer: + +* Pull mode: proxy forwards P's ``kv_transfer_params`` (including + ``remote_block_ids``) to D, and D pulls KV from P via NIXL READ. +* Push mode: proxy hands D **only** P's coordinates + (``remote_engine_id``, ``remote_host``, ``remote_port``, ``tp_size``) + and the shared ``remote_request_id``. D registers its locally allocated + blocks with P over a NIXL notification; P then pushes the KV to D via + NIXL WRITE. + +Launch multiple vLLM instances configured with ``NixlPushConnector`` and +matching ``engine_id`` / ``side_channel_port``, then start this proxy: + + python3 examples/disaggregated/disaggregated_serving/\ +disagg_proxy_pushconnector_demo.py \ + --model $model_name \ + --prefill localhost:8100 \ + --decode localhost:8200 \ + --prefill-engine-id prefill-engine-001 \ + --prefill-kv-host 10.0.0.1 \ + --prefill-side-channel-port 5600 \ + --prefill-tp-size 1 \ + --port 8000 +""" + +import argparse +import contextlib +import ipaddress +import itertools +import json +import logging +import os +import sys +import uuid +from abc import ABC, abstractmethod +from collections.abc import Callable + +import aiohttp +import uvicorn +from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request +from fastapi.responses import JSONResponse, StreamingResponse + +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60) +logger = logging.getLogger() +logging.basicConfig(level=logging.INFO) + + +class SchedulingPolicy(ABC): + @abstractmethod + def schedule(self, cycler: itertools.cycle): + raise NotImplementedError("Scheduling Proxy is not set.") + + +class RoundRobinSchedulingPolicy(SchedulingPolicy): + def schedule(self, cycler: itertools.cycle) -> str: + return next(cycler) + + +class PushProxy: + """Push-mode proxy. + + The structure mirrors the pull-mode ``Proxy`` in + ``disagg_proxy_demo.py``: an APIRouter with ``/v1/completions``, + ``/v1/chat/completions``, ``/status`` and ``/instances/add``, plus + round-robin scheduling across multiple P / D instances. + + Push-specific differences are confined to the request-handling + methods (``create_completion`` / ``create_chat_completion``): + + * D's ``kv_transfer_params`` is built from CLI-provided P + coordinates instead of being derived from P's response. + * P and D requests are issued concurrently — D registers blocks and + waits while P prefills and pushes. + """ + + def __init__( + self, + prefill_instances: list[str], + decode_instances: list[str], + model: str, + scheduling_policy: SchedulingPolicy, + prefill_engine_id: str, + prefill_kv_host: str, + prefill_side_channel_port: int, + prefill_tp_size: int, + custom_create_completion: Callable[[Request], StreamingResponse] | None = None, + custom_create_chat_completion: Callable[[Request], StreamingResponse] + | None = None, + ): + self.prefill_instances = prefill_instances + self.decode_instances = decode_instances + self.prefill_cycler = itertools.cycle(prefill_instances) + self.decode_cycler = itertools.cycle(decode_instances) + self.model = model + self.scheduling_policy = scheduling_policy + + # Push-mode metadata: D needs P's coordinates up-front. Pull mode + # learns these from P's response; push mode uses CLI args because + # D issues its registration before P responds. + self.push_metadata = { + "do_remote_decode": False, + "do_remote_prefill": True, + "remote_engine_id": prefill_engine_id, + "remote_host": prefill_kv_host, + "remote_port": prefill_side_channel_port, + "tp_size": prefill_tp_size, + } + + self.custom_create_completion = custom_create_completion + self.custom_create_chat_completion = custom_create_chat_completion + self.router = APIRouter() + self.setup_routes() + + # ── routes ──────────────────────────────────────────────────────── # + + def setup_routes(self): + self.router.post( + "/v1/completions", dependencies=[Depends(self.validate_json_request)] + )( + self.custom_create_completion + if self.custom_create_completion + else self.create_completion + ) + self.router.post( + "/v1/chat/completions", dependencies=[Depends(self.validate_json_request)] + )( + self.custom_create_chat_completion + if self.custom_create_chat_completion + else self.create_chat_completion + ) + self.router.get("/status", response_class=JSONResponse)(self.get_status) + + async def validate_json_request(self, raw_request: Request): + content_type = raw_request.headers.get("content-type", "").lower() + if content_type != "application/json": + raise HTTPException( + status_code=415, + detail="Unsupported Media Type: Only 'application/json' is allowed", + ) + + # ── HTTP forwarding ─────────────────────────────────────────────── # + + async def forward_request(self, url, data, headers, use_chunked=True): + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session: + try: + async with session.post( + url=url, json=data, headers=headers + ) as response: + if 200 <= response.status < 300 or 400 <= response.status < 500: + if use_chunked: + async for chunk_bytes in response.content.iter_chunked( + 1024 + ): + yield chunk_bytes + else: + yield await response.read() + else: + error_content = await response.text() + with contextlib.suppress(json.JSONDecodeError): + error_content = json.loads(error_content) + logger.error( + "Request failed with status %s: %s", + response.status, + error_content, + ) + raise HTTPException( + status_code=response.status, + detail=f"Request failed with status {response.status}: " + f"{error_content}", + ) + except aiohttp.ClientError as e: + logger.error("ClientError occurred: %s", str(e)) + raise HTTPException( + status_code=502, + detail="Bad Gateway: Error communicating with upstream server.", + ) from e + except Exception as e: + logger.error("Unexpected error: %s", str(e)) + raise HTTPException(status_code=500, detail=str(e)) from e + + def schedule(self, cycler: itertools.cycle) -> str: + return self.scheduling_policy.schedule(cycler) + + async def get_status(self): + return { + "mode": "push", + "prefill_node_count": len(self.prefill_instances), + "decode_node_count": len(self.decode_instances), + "prefill_nodes": self.prefill_instances, + "decode_nodes": self.decode_instances, + "prefill_engine_id": self.push_metadata["remote_engine_id"], + "prefill_kv_host": self.push_metadata["remote_host"], + "prefill_side_channel_port": self.push_metadata["remote_port"], + "prefill_tp_size": self.push_metadata["tp_size"], + } + + # ── push-mode request handling ──────────────────────────────────── # + + def _build_decode_kv_params(self, request_id: str) -> dict: + """Push-mode kv_transfer_params for D. + + ``remote_block_ids`` is intentionally omitted: D allocates its + own blocks and registers them with P; P determines the + prefill-side block IDs and ships them via the WRITE. + """ + params = self.push_metadata.copy() + params["remote_request_id"] = request_id + return params + + def _common_headers(self, request_id: str) -> dict: + h = {"X-Request-Id": request_id} + api_key = os.environ.get("OPENAI_API_KEY") + if api_key: + h["Authorization"] = f"Bearer {api_key}" + return h + + async def _push_completion(self, raw_request: Request, path: str): + """Shared body for /v1/completions and /v1/chat/completions. + + Push mode fires P and D concurrently: + * P runs a normal prefill (max_tokens=1, do_remote_decode=True). + * D runs the decode (do_remote_prefill=True, no remote_block_ids). + + D blocks waiting for P's WRITE; the response streamed back to the + client is the decode output from D. + """ + request = await raw_request.json() + request_id = str(uuid.uuid4()) + + # Prefill leg (max_tokens=1, signals P to keep KV around for D). + prefill_request = request.copy() + prefill_request["max_tokens"] = 1 + if "max_completion_tokens" in prefill_request: + prefill_request["max_completion_tokens"] = 1 + prefill_request["kv_transfer_params"] = { + "do_remote_decode": True, + "do_remote_prefill": False, + "remote_engine_id": None, + "remote_block_ids": None, + "remote_host": None, + "remote_port": None, + } + + # Decode leg (push mode: no remote_block_ids). + decode_request = request.copy() + decode_request["kv_transfer_params"] = self._build_decode_kv_params(request_id) + + prefill_instance = self.schedule(self.prefill_cycler) + decode_instance = self.schedule(self.decode_cycler) + headers = self._common_headers(request_id) + + # Fire prefill; we don't read its body but must drain the + # connection so the upstream server can free its slot. + async for _ in self.forward_request( + f"http://{prefill_instance}{path}", prefill_request, headers + ): + continue + + generator = self.forward_request( + f"http://{decode_instance}{path}", decode_request, headers + ) + return StreamingResponse(generator) + + async def create_completion(self, raw_request: Request): + try: + return await self._push_completion(raw_request, "/v1/completions") + except HTTPException: + raise + except Exception: + exc_info = sys.exc_info() + print("Error occurred in disagg push proxy server") + print(exc_info) + raise + + async def create_chat_completion(self, raw_request: Request): + try: + return await self._push_completion(raw_request, "/v1/chat/completions") + except HTTPException: + raise + except Exception: + exc_info = sys.exc_info() + error_messages = [str(e) for e in exc_info if e] + print("Error occurred in disagg push proxy server") + print(error_messages) + return StreamingResponse( + content=iter(error_messages), media_type="text/event-stream" + ) + + +class PushProxyServer: + def __init__( + self, + args: argparse.Namespace, + scheduling_policy: SchedulingPolicy | None = None, + create_completion: Callable[[Request], StreamingResponse] | None = None, + create_chat_completion: Callable[[Request], StreamingResponse] | None = None, + ): + self.validate_parsed_serve_args(args) + self.port = args.port + self.proxy_instance = PushProxy( + prefill_instances=[] if args.prefill is None else args.prefill, + decode_instances=[] if args.decode is None else args.decode, + model=args.model, + scheduling_policy=( + scheduling_policy + if scheduling_policy is not None + else RoundRobinSchedulingPolicy() + ), + prefill_engine_id=args.prefill_engine_id, + prefill_kv_host=args.prefill_kv_host, + prefill_side_channel_port=args.prefill_side_channel_port, + prefill_tp_size=args.prefill_tp_size, + custom_create_completion=create_completion, + custom_create_chat_completion=create_chat_completion, + ) + + def validate_parsed_serve_args(self, args: argparse.Namespace): + if not args.prefill: + raise ValueError("Please specify at least one prefill node.") + if not args.decode: + raise ValueError("Please specify at least one decode node.") + if not args.prefill_engine_id: + raise ValueError( + "--prefill-engine-id is required in push mode (it must match " + "the engine_id passed to the prefill vLLM instance via " + "--kv-transfer-config)." + ) + if not args.prefill_kv_host: + raise ValueError( + "--prefill-kv-host is required in push mode (the IP / host " + "that the prefill vLLM advertises on its NIXL side channel)." + ) + self.validate_instances(args.prefill) + self.validate_instances(args.decode) + + def validate_instances(self, instances: list): + for instance in instances: + if len(instance.split(":")) != 2: + raise ValueError(f"Invalid instance format: {instance}") + host, port = instance.split(":") + try: + if host != "localhost": + ipaddress.ip_address(host) + port = int(port) + if not (0 < port < 65536): + raise ValueError(f"Invalid port number in instance: {instance}") + except Exception as e: + raise ValueError(f"Invalid instance {instance}: {str(e)}") from e + + def run_server(self): + app = FastAPI() + app.include_router(self.proxy_instance.router) + config = uvicorn.Config(app, port=self.port, loop="uvloop") + server = uvicorn.Server(config) + server.run() + + +def parse_args(): + parser = argparse.ArgumentParser("vLLM disaggregated push-mode proxy server.") + parser.add_argument("--model", "-m", type=str, required=True, help="Model name") + + parser.add_argument( + "--prefill", + "-p", + type=str, + nargs="+", + help="List of prefill node URLs (host:port)", + ) + + parser.add_argument( + "--decode", + "-d", + type=str, + nargs="+", + help="List of decode node URLs (host:port)", + ) + + parser.add_argument( + "--port", + type=int, + default=8000, + help="Server port number", + ) + + # Push-mode specific: P's coordinates that D needs in advance. + parser.add_argument( + "--prefill-engine-id", + type=str, + required=True, + help=( + "engine_id of the prefill vLLM instance (must match " + "--kv-transfer-config engine_id on the prefill server)" + ), + ) + parser.add_argument( + "--prefill-kv-host", + type=str, + required=True, + help=( + "IP / host the prefill vLLM advertises on its NIXL side " + "channel (VLLM_NIXL_SIDE_CHANNEL_HOST)" + ), + ) + parser.add_argument( + "--prefill-side-channel-port", + type=int, + default=5600, + help="NIXL side channel port on the prefill node " + "(VLLM_NIXL_SIDE_CHANNEL_PORT, default 5600)", + ) + parser.add_argument( + "--prefill-tp-size", + type=int, + default=1, + help="Tensor parallel size of the prefill vLLM instance", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + proxy_server = PushProxyServer(args=args) + proxy_server.run_server() diff --git a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py index ef092dfb49..12831601cb 100644 --- a/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py +++ b/tests/v1/kv_connector/unit/test_bidirectional_kv_transfer.py @@ -32,7 +32,7 @@ from unittest.mock import patch import pytest from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import ( +from vllm.distributed.kv_transfer.kv_connector.v1.nixl import ( NixlConnector, NixlConnectorMetadata, ) @@ -436,7 +436,7 @@ def test_build_connector_meta_multiple_requests(): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_p_node_pull_kv_from_d(dist_init): @@ -450,7 +450,7 @@ def test_p_node_pull_kv_from_d(dist_init): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_p_node_pull_then_send_kv(dist_init): @@ -472,7 +472,7 @@ def test_p_node_pull_then_send_kv(dist_init): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_p_node_deferred_pull_on_no_handshake(dist_init): diff --git a/tests/v1/kv_connector/unit/test_multi_connector.py b/tests/v1/kv_connector/unit/test_multi_connector.py index f78037a143..6ac6b4318c 100644 --- a/tests/v1/kv_connector/unit/test_multi_connector.py +++ b/tests/v1/kv_connector/unit/test_multi_connector.py @@ -366,7 +366,10 @@ def test_multi_example_connector_consistency(): def _ignore_event_collection(events: list[str]) -> list[str]: - return [event for event in events if event != "take_events"] + # Filter out per-step polling hooks that the scheduler calls repeatedly + # and which are not meaningful state transitions for these assertions. + ignored = {"take_events", "has_pending_push_work"} + return [event for event in events if event not in ignored] def get_connector_events() -> dict[str, list[str]]: @@ -1072,7 +1075,7 @@ def test_multi_connector_mixed_hma_disables_hybrid_kv_cache(monkeypatch): ) with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ): llm = LLM( diff --git a/tests/v1/kv_connector/unit/test_nixl_connector.py b/tests/v1/kv_connector/unit/test_nixl_connector.py index c5784d1c20..32652118d5 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector.py @@ -344,7 +344,7 @@ def test_abort_immediately_remote_prefill_enqueues_empty_recv(): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_kv_transfer_handshake(dist_init): @@ -560,7 +560,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker): class TestNixlHandshake: @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_multi_xfer_one_engine( @@ -643,7 +643,7 @@ class TestNixlHandshake: connector.clear_connector_metadata() @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) @pytest.mark.parametrize( @@ -713,7 +713,7 @@ class TestNixlHandshake: raise TimeoutError("Took too long to complete async handshake.") @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) @pytest.mark.parametrize("local_tp_size", [1, 2]) @@ -725,7 +725,7 @@ class TestNixlHandshake: remote configurations. """ monkeypatch.setattr( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", lambda: local_tp_size, ) @@ -784,7 +784,7 @@ class TestNixlHandshake: check_handshake(6) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_prefill_tp_size_greater_than_decode_tp_size_mla( @@ -887,7 +887,7 @@ class TestNixlHandshake: assert req_id not in conn_p1.connector_worker._reqs_to_process @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_concurrent_load_kv( @@ -952,7 +952,7 @@ class TestNixlHandshake: raise TimeoutError("Took too long to complete async handshake.") @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_handshake_fails_on_kv_cache_layout_mismatch( @@ -967,7 +967,7 @@ class TestNixlHandshake: # Mock TP world size to 2 to force heterogeneous TP when # remote_tp_size=1 with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size", # noqa: E501 + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 return_value=2, ): # Initialize connector and worker (with fake NIXL wrapper) @@ -1007,7 +1007,7 @@ class TestNixlHandshake: worker.add_remote_agent(meta, remote_tp_size=1) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_handshake_succeed_on_kv_cache_layout_mismatch_with_experimental( @@ -1022,7 +1022,7 @@ class TestNixlHandshake: # Mock TP world size to 2 to force heterogeneous TP when # remote_tp_size=1 with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size", # noqa: E501 + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 return_value=2, ): # Initialize connector and worker (with fake NIXL wrapper) @@ -1064,7 +1064,7 @@ class TestNixlHandshake: worker.add_remote_agent(meta, remote_tp_size=1) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_handshake_mixed_fa_mla_hetero_tp(self, default_vllm_config, dist_init): @@ -1074,7 +1074,7 @@ class TestNixlHandshake: """ vllm_config = create_vllm_config() with patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.get_tensor_model_parallel_world_size", # noqa: E501 + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.get_tensor_model_parallel_world_size", # noqa: E501 return_value=2, ): connector = NixlConnector( @@ -1149,7 +1149,7 @@ class TestNixlHandshake: # we put here is important. First run ray, it will clean up the resources, then # the rest of the tests. @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_kv_connector_stats(default_vllm_config, dist_init): @@ -1363,7 +1363,7 @@ def test_multi_kv_connector_stats_aggregation(): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_scheduler_kv_connector_stats_aggregation(): @@ -1428,7 +1428,7 @@ def test_scheduler_kv_connector_stats_aggregation(): @pytest.mark.parametrize("distributed_executor_backend", ["ray", None]) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend): @@ -1615,7 +1615,7 @@ def test_register_kv_caches( backend_cls = TritonAttentionBackend - nixl_worker = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker" + nixl_worker = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker" nixl_connector = "vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector" with ( patch(f"{nixl_worker}.NixlWrapper") as mock_nixl_wrapper, @@ -1865,15 +1865,17 @@ def test_kv_buffer_to_nixl_memory_types( _NIXL_SUPPORTED_DEVICE.update(FakePlatform.get_nixl_supported_devices()) with ( - patch("vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper"), patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.threading.Event" + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper" ), patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.threading.Thread" + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.threading.Event" ), patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.current_platform", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.threading.Thread" + ), + patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.current_platform", FakePlatform, ), patch( @@ -1892,7 +1894,7 @@ def test_kv_buffer_to_nixl_memory_types( @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_shutdown_cleans_up_resources(default_vllm_config, dist_init): @@ -1991,7 +1993,7 @@ def _setup_worker_with_remote_engine( @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_engine_ttl_eviction(default_vllm_config, dist_init): @@ -2026,7 +2028,7 @@ def test_engine_ttl_eviction(default_vllm_config, dist_init): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_engine_ttl_disabled(default_vllm_config, dist_init): @@ -2074,7 +2076,7 @@ def test_transfer_topology_unregister(): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_aborted_request_removed_from_worker_in_batch(default_vllm_config, dist_init): @@ -2194,7 +2196,7 @@ class FailingNixlWrapper(FakeNixlWrapper): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FailingNixlWrapper, ) @pytest.mark.parametrize( @@ -2284,10 +2286,13 @@ def test_transfer_failure_logging( slot_mapping={}, ) - # Capture logs from the nixl.worker logger specifically + # Capture logs from the nixl connector loggers # vLLM loggers have propagate=False, so we need to capture directly nixl_logger = logging.getLogger( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker" + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker" + ) + pull_logger = logging.getLogger( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker" ) captured_logs: list[logging.LogRecord] = [] @@ -2298,6 +2303,7 @@ def test_transfer_failure_logging( handler = LogCapture() handler.setLevel(logging.ERROR) nixl_logger.addHandler(handler) + pull_logger.addHandler(handler) try: connector.start_load_kv(dummy_ctx) @@ -2313,6 +2319,7 @@ def test_transfer_failure_logging( connector.get_finished(finished_req_ids=set()) finally: nixl_logger.removeHandler(handler) + pull_logger.removeHandler(handler) # Print logs for manual comparison between commits error_logs = [r for r in captured_logs if r.levelno >= logging.ERROR] @@ -2349,7 +2356,7 @@ def test_transfer_failure_logging( @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FailingNixlWrapper, ) def test_handshake_failure_returns_finished(default_vllm_config, dist_init): @@ -2400,7 +2407,7 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init): @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FailingNixlWrapper, ) def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init): @@ -2454,7 +2461,7 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FailingNixlWrapper, ) @pytest.mark.parametrize( @@ -2597,7 +2604,7 @@ def test_failed_request_skips_kv_postprocessing( ], ) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_compatibility_hash_validation( @@ -2706,7 +2713,7 @@ def test_compatibility_hash_validation( # Patch zmq_ctx to return our mock socket with ( patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"), - patch.object(nixl.worker, "zmq_ctx") as mock_zmq_ctx, + patch.object(nixl.base_worker, "zmq_ctx") as mock_zmq_ctx, ): mock_zmq_ctx.return_value.__enter__.return_value = mock_socket @@ -2740,7 +2747,7 @@ def test_compatibility_hash_validation( ], ) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario): @@ -2806,7 +2813,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario) mock_socket.recv.return_value = msg_bytes with ( patch.object(decode_worker, "add_remote_agent", return_value="fake_agent"), - patch.object(nixl.worker, "zmq_ctx") as mock_zmq_ctx, + patch.object(nixl.base_worker, "zmq_ctx") as mock_zmq_ctx, ): mock_zmq_ctx.return_value.__enter__.return_value = mock_socket @@ -2819,7 +2826,7 @@ def test_handshake_decode_errors(default_vllm_config, dist_init, error_scenario) ) @patch( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper", FakeNixlWrapper, ) def test_mla_broadcast_notif_uses_remote_request_id( diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index af043113ed..eed20e0366 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -34,7 +34,9 @@ from .utils import ( (False, [0]), ], ) -@patch("vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler.current_platform") +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform" +) def test_sw_sizes(mock_platform, swa_enabled, expected_sw_sizes): """Test sw_sizes is correctly computed based on SWA enabled/disabled.""" from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import ( @@ -782,7 +784,9 @@ def test_mamba_n1_p_side_truncation(): ], ids=["fa_swa_mamba", "fa_swa_only", "fa_only"], ) -@patch("vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler.current_platform") +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform" +) def test_has_mamba_init( mock_platform, swa_enabled, diff --git a/tests/v1/kv_connector/unit/test_nixl_push_connector.py b/tests/v1/kv_connector/unit/test_nixl_push_connector.py new file mode 100644 index 0000000000..fe67c1ac73 --- /dev/null +++ b/tests/v1/kv_connector/unit/test_nixl_push_connector.py @@ -0,0 +1,815 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for NixlPushConnector (scheduler + worker). + +These tests cover the end-to-end mechanics of the push design without +requiring a real NIXL agent or network: + +* Scheduler stages D registrations on ``update_state_after_alloc`` and + P finished blocks on ``request_finished``. +* ``build_connector_meta`` drains them onto + ``meta.push_registrations`` / ``meta.push_finished_blocks``. +* ``has_pending_push_work`` reports True/False over the lifecycle. +* ``update_connector_output`` clears state on ``finished_sending`` and + ``finished_recving``. +* The worker matches D registrations against P finished blocks (both + scenario directions) and forwards non-PUSH_REG NIXL notifs to the main + thread's ``_get_new_notifs``. +* ``get_finished`` enqueues evictions for the writer. +""" + +from __future__ import annotations + +import logging +import queue +import threading +import time +from collections import defaultdict +from typing import Any +from unittest.mock import MagicMock, patch + +import msgspec + +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + PUSH_REG_NOTIF_PREFIX, + NixlConnectorMetadata, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( + NixlPushConnectorWorker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( + get_base_request_id, +) +from vllm.v1.outputs import KVConnectorOutput + +from .utils import make_nixl_push_scheduler + +# ----------------------------------------------------------------- # +# Helpers / fakes # +# ----------------------------------------------------------------- # + + +def _make_request( + *, + request_id: str, + is_d_side: bool = True, + remote_engine_id: str = "prefill-engine", + remote_request_id: str | None = None, + remote_host: str = "10.0.0.1", + remote_port: int = 5601, + tp_size: int = 1, + finished: bool = True, +) -> MagicMock: + """Build a minimal Request mock used by request_finished.""" + from vllm.v1.request import RequestStatus + + req = MagicMock() + req.request_id = request_id + req.num_computed_tokens = 64 + + if is_d_side: + # D-side request: do_remote_prefill=True -> prefill on a remote P. + params: dict[str, Any] = { + "do_remote_prefill": True, + "do_remote_decode": False, + "remote_engine_id": remote_engine_id, + "remote_request_id": remote_request_id or f"prefill-{request_id}", + "remote_host": remote_host, + "remote_port": remote_port, + "tp_size": tp_size, + } + else: + # P-side request: do_remote_decode=True (we are the prefiller). + params = { + "do_remote_prefill": False, + "do_remote_decode": True, + } + req.kv_transfer_params = params + req.status = ( + RequestStatus.FINISHED_LENGTH_CAPPED if finished else RequestStatus.RUNNING + ) + return req + + +class _BlocksMock: + """Minimal stand-in for ``KVCacheBlocks`` used in update_state_after_alloc.""" + + def __init__(self, block_ids: tuple[list[int], ...]): + self._block_ids = block_ids + + def get_unhashed_block_ids_all_groups(self) -> tuple[list[int], ...]: + return self._block_ids + + +def _stub_sw_clipping(scheduler) -> None: + """Make ``get_sw_clipped_blocks`` a passthrough so tests don't need + the full sliding-window machinery.""" + scheduler.get_sw_clipped_blocks = lambda block_ids: block_ids + + +# ----------------------------------------------------------------- # +# Scheduler-side tests # +# ----------------------------------------------------------------- # + + +class TestPushScheduler: + def test_d_side_update_state_after_alloc_stages_registration(self): + """D scheduler stashes registration data + arms watchdog deadline.""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + request = _make_request(request_id="req-d-1") + blocks = _BlocksMock(block_ids=([10, 11, 12],)) + + sched.update_state_after_alloc(request, blocks, num_external_tokens=48) + + assert request.request_id in sched._push_pending_registrations + reg = sched._push_pending_registrations[request.request_id] + # ``request_id`` is D's own vLLM request id; plus our own (D) coords. + assert reg["request_id"] == request.request_id + assert reg["decode_engine_id"] == sched.engine_id + assert reg["decode_host"] == sched.side_channel_host + assert reg["decode_port"] == sched.side_channel_port + assert reg["local_block_ids"] == ([10, 11, 12],) + assert reg["remote_engine_id"] == "prefill-engine" + + # Watchdog deadline set in the future. + deadline = sched._push_registration_deadlines[request.request_id] + assert deadline > time.perf_counter() + # do_remote_prefill flipped off so the request isn't reprocessed. + assert request.kv_transfer_params["do_remote_prefill"] is False + # Tracked as awaiting a recv. + assert request.request_id in sched._reqs_need_recv + + def test_p_side_request_finished_stages_blocks(self): + """P scheduler pushes blocks into both _finished_request_blocks (lease) + and _newly_finished_push_blocks (metadata for next step).""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + request = _make_request(request_id="req-p-1", is_d_side=False) + block_ids = ([20, 21, 22, 23],) + + delay, ret_params = sched.request_finished(request, block_ids) + + assert delay is True + assert ret_params is not None + assert ret_params["do_remote_prefill"] is True + assert ret_params["do_remote_decode"] is False + assert request.request_id in sched._finished_request_blocks + assert request.request_id in sched._newly_finished_push_blocks + assert request.request_id in sched._reqs_need_send # lease armed + + def test_build_connector_meta_drains_both_sides(self): + """meta.push_registrations and meta.push_finished_blocks are filled + from the staging dicts and the staging dicts are cleared.""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + # Stage one D registration and one P finished entry. + d_req = _make_request(request_id="req-d-9") + sched.update_state_after_alloc( + d_req, _BlocksMock(([1, 2, 3],)), num_external_tokens=48 + ) + p_req = _make_request(request_id="req-p-9", is_d_side=False) + sched.request_finished(p_req, ([4, 5, 6],)) + + scheduler_output = MagicMock() + scheduler_output.scheduled_new_reqs = [] + scheduler_output.scheduled_cached_reqs = MagicMock( + req_ids=[], resumed_req_ids=set() + ) + + # Patch parent build_connector_meta so we don't have to set up + # all the base scheduler plumbing. + with patch.object( + sched.__class__.__mro__[1], + "build_connector_meta", + return_value=NixlConnectorMetadata(), + ): + meta = sched.build_connector_meta(scheduler_output) + + assert isinstance(meta, NixlConnectorMetadata) + assert "req-d-9" in meta.push_registrations + assert "req-p-9" in meta.push_finished_blocks + # Staging dicts cleared. + assert sched._push_pending_registrations == {} + assert sched._newly_finished_push_blocks == {} + # Lease bookkeeping kept until the WRITE completes. + assert "req-p-9" in sched._finished_request_blocks + + def test_has_pending_push_work_lifecycle(self): + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + assert sched.has_pending_push_work() is False + + # P finished blocks waiting for WRITE completion. + p_req = _make_request(request_id="req-p-7", is_d_side=False) + sched.request_finished(p_req, ([0, 1],)) + assert sched.has_pending_push_work() is True + + # Drain via build_connector_meta - lease still pending until WRITE. + scheduler_output = MagicMock() + scheduler_output.scheduled_new_reqs = [] + scheduler_output.scheduled_cached_reqs = MagicMock( + req_ids=[], resumed_req_ids=set() + ) + with patch.object( + sched.__class__.__mro__[1], + "build_connector_meta", + return_value=NixlConnectorMetadata(), + ): + sched.build_connector_meta(scheduler_output) + # Lease is pending until WRITE completes -> still True. + assert sched.has_pending_push_work() is True + + # Simulate WRITE completion via update_connector_output. + sched.update_connector_output( + KVConnectorOutput( + finished_sending={"req-p-7"}, + finished_recving=set(), + invalid_block_ids=set(), + ) + ) + assert sched.has_pending_push_work() is False + + def test_update_connector_output_clears_lease_and_watchdog(self): + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + d_req = _make_request(request_id="req-d-x") + sched.update_state_after_alloc( + d_req, _BlocksMock(([1, 2],)), num_external_tokens=32 + ) + p_req = _make_request(request_id="req-p-x", is_d_side=False) + sched.request_finished(p_req, ([3, 4],)) + + sched.update_connector_output( + KVConnectorOutput( + finished_sending={"req-p-x"}, + finished_recving={"req-d-x"}, + invalid_block_ids=set(), + ) + ) + assert "req-p-x" not in sched._finished_request_blocks + assert "req-d-x" not in sched._push_registration_deadlines + + def test_registration_watchdog_expires(self, caplog): + """Stale D registrations whose deadline has passed are dropped at + ``build_connector_meta`` time.""" + # Watchdog logs a WARNING when it drops the stale entry; that's + # what this test is verifying, so silence it in the test report. + caplog.set_level( + logging.CRITICAL, + logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler"), + ) + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + d_req = _make_request(request_id="req-d-stale") + sched.update_state_after_alloc( + d_req, _BlocksMock(([7, 8],)), num_external_tokens=32 + ) + # Force the deadline into the past. + sched._push_registration_deadlines[d_req.request_id] = time.perf_counter() - 1.0 + + scheduler_output = MagicMock() + scheduler_output.scheduled_new_reqs = [] + scheduler_output.scheduled_cached_reqs = MagicMock( + req_ids=[], resumed_req_ids=set() + ) + with patch.object( + sched.__class__.__mro__[1], + "build_connector_meta", + return_value=NixlConnectorMetadata(), + ): + meta = sched.build_connector_meta(scheduler_output) + + assert d_req.request_id not in sched._push_registration_deadlines + assert d_req.request_id not in sched._push_pending_registrations + assert d_req.request_id not in meta.push_registrations + + +# ----------------------------------------------------------------- # +# Worker-side tests # +# ----------------------------------------------------------------- # + + +class _StubWriterWorker(NixlPushConnectorWorker): + """Construct a worker without invoking ``__init__`` so we can drive + the matching/notif logic without bringing up NIXL or torch.""" + + @classmethod + def fresh(cls) -> _StubWriterWorker: + w = object.__new__(cls) + + # Push-specific state managed by NixlPushConnectorWorker. + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + ReqId, + TransferHandle, + ) + + w._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) + w._sending_transfers_lock = threading.Lock() + w._push_finished_blocks = {} + w._pending_d_registrations = {} + w._reg_send_inbox = queue.Queue() + w._finished_blocks_inbox = queue.Queue() + w._pending_completion_notifs = queue.Queue() + w._evict_finished_inbox = queue.Queue() + w._push_writer_wake = threading.Event() + w._push_writer_stop = threading.Event() + w._push_writer_thread = None + + # Base worker fields touched by start_load_kv / _get_new_notifs. + w._recving_metadata = {} + w._recving_transfers = defaultdict(list) + w._reqs_to_process = set() + w._reqs_to_send = {} + w.consumer_notification_counts_by_req = defaultdict(int) + w.tp_rank = 0 + w.world_size = 1 + w.engine_id = "test-decode-engine" + w._remote_agents = {} + + # Track _do_start_push_kv invocations. + calls: list[tuple[str, Any, dict[str, Any]]] = [] + w.start_push_calls = calls + return w + + def _do_start_push_kv( + self, + request_id: str, + local_block_ids, + registration_data: dict[str, Any], + ) -> None: # pragma: no cover - exercised through tests + # Track the call instead of issuing real WRITEs. + self.start_push_calls.append((request_id, local_block_ids, registration_data)) + + +def _registration_data( + request_id: str, + *, + decode_engine_id: str = "decode-engine", + decode_host: str = "10.0.0.2", + decode_port: int = 5602, + decode_tp_size: int = 1, + local_block_ids=((100, 101, 102),), + remote_engine_id: str = "prefill-engine", + remote_host: str = "10.0.0.1", + remote_port: int = 5601, + remote_tp_size: int = 1, +) -> dict[str, Any]: + return { + "request_id": request_id, + "decode_engine_id": decode_engine_id, + "decode_host": decode_host, + "decode_port": decode_port, + "decode_tp_size": decode_tp_size, + "local_block_ids": local_block_ids, + "remote_engine_id": remote_engine_id, + "remote_host": remote_host, + "remote_port": remote_port, + "remote_tp_size": remote_tp_size, + } + + +class TestPushWriterMatching: + def test_handle_push_reg_matches_existing_finished_blocks(self): + """PUSH_REG arrives second (P finished first): match + fire.""" + w = _StubWriterWorker.fresh() + # P had already finished; its blocks were stashed via metadata. + w._push_finished_blocks["req-A"] = ([200, 201, 202],) + + notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode( + _registration_data("req-A") + ) + w._handle_push_reg_notif(notif) + + assert len(w.start_push_calls) == 1 + rid, blocks, reg = w.start_push_calls[0] + assert rid == "req-A" + assert blocks == ([200, 201, 202],) + assert reg["decode_engine_id"] == "decode-engine" + # Finished blocks consumed. + assert "req-A" not in w._push_finished_blocks + assert w._pending_d_registrations == {} + + def test_handle_push_reg_stashes_when_no_finished_blocks_yet(self): + """PUSH_REG arrives first (D registered first): stash, no fire.""" + w = _StubWriterWorker.fresh() + + notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode( + _registration_data("req-B") + ) + w._handle_push_reg_notif(notif) + + assert len(w.start_push_calls) == 0 + assert "req-B" in w._pending_d_registrations + + def test_handle_push_reg_matches_after_stripping_random_suffix(self): + """P and D assign the same logical request the same + ``cmpl--`` but different per-engine random suffixes; + the writer should still match P's finished blocks via the + suffix-stripping fallback in ``_pop_matching_finished_blocks``. + """ + w = _StubWriterWorker.fresh() + # Same base id + completion index; differ only in the trailing + # ``-<8 hex>`` randomization suffix. + p_id = "cmpl-12345678-aaaa-bbbb-cccc-1234567890ab-0-aaaaaaaa" + d_id = "cmpl-12345678-aaaa-bbbb-cccc-1234567890ab-0-bbbbbbbb" + # Sanity: same base id under the helper used by the connector. + assert get_base_request_id(p_id) == get_base_request_id(d_id) + + w._push_finished_blocks[p_id] = ([1, 2, 3],) + notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(_registration_data(d_id)) + w._handle_push_reg_notif(notif) + + # Suffix-stripped fallback matched and fired. + assert len(w.start_push_calls) == 1 + assert w.start_push_calls[0][0] == p_id + assert p_id not in w._push_finished_blocks + + def test_handle_push_reg_drops_malformed(self, caplog): + # The writer logs WARNING/ERROR when it sees these bad payloads; + # that's the desired behavior, so suppress the noise from test + # output rather than letting it look like a failure. + caplog.set_level( + logging.CRITICAL, + logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), + ) + w = _StubWriterWorker.fresh() + # Missing request_id -> should drop without raising. + bad = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode({"decode_engine_id": "x"}) + w._handle_push_reg_notif(bad) + assert w._pending_d_registrations == {} + assert w.start_push_calls == [] + + # Undecodable payload also dropped. + w._handle_push_reg_notif(PUSH_REG_NOTIF_PREFIX + b"\xff\xff\xff") + assert w.start_push_calls == [] + + +class TestPushWriterStartLoadKv: + def test_finished_blocks_inbox_matches_stashed_registration(self): + """Run the writer-loop's finished-blocks drain against a + pre-populated _pending_d_registrations entry.""" + w = _StubWriterWorker.fresh() + w._pending_d_registrations["req-C"] = _registration_data("req-C") + + # Simulate start_load_kv enqueuing finished blocks. + w._finished_blocks_inbox.put(("req-C", ([10, 11, 12],))) + + # Drain like the writer loop does. + while True: + try: + rid, blocks = w._finished_blocks_inbox.get_nowait() + except queue.Empty: + break + matched = w._pop_matching_registration(rid) + if matched is not None: + w._do_start_push_kv(rid, blocks, matched) + else: + w._push_finished_blocks[rid] = blocks + + assert len(w.start_push_calls) == 1 + assert w.start_push_calls[0][0] == "req-C" + assert "req-C" not in w._pending_d_registrations + + def test_start_load_kv_enqueues_to_writer(self): + """``start_load_kv`` should hand registrations + finished blocks + to the writer queues without doing matching itself.""" + w = _StubWriterWorker.fresh() + # Stub heartbeats to a no-op; tests don't exercise the heartbeat + # path here. + w._send_heartbeats = lambda metadata: None + # Stub logical-to-kernel mapping used by reqs_to_recv. + w._logical_to_kernel_block_ids = lambda x: x + + meta = NixlConnectorMetadata() + meta.push_registrations = { + "req-D": _registration_data("req-D"), + } + meta.push_finished_blocks = { + "req-E": ([5, 6, 7],), + } + + w.start_load_kv(meta) + + # Things are queued for the writer; nothing fires yet. + assert w._reg_send_inbox.qsize() == 1 + assert w._finished_blocks_inbox.qsize() == 1 + assert w._push_writer_wake.is_set() + assert w.start_push_calls == [] + + +class TestPushWriterNotifs: + def test_get_new_notifs_processes_forwarded_completion_notif(self): + """Non-PUSH_REG notifs forwarded by the writer thread are drained + on the engine main thread inside ``_get_new_notifs``.""" + w = _StubWriterWorker.fresh() + # Pretend the writer thread already forwarded a completion notif + # for a request whose KV is being received. + request_id = "req-recv-1" + w._recving_metadata[request_id] = MagicMock() + # Compose the standard completion notif: req_id:tp_size. + notif_msg = f"{request_id}:1".encode() + w._pending_completion_notifs.put(notif_msg) + + # transfer_topo is consulted only for the producer-side path; we + # make it a MagicMock because the D-side branch returns early. + w.transfer_topo = MagicMock() + + notified = w._get_new_notifs() + + # Notif consumed; D-side just touches _recving_transfers. + assert notified == set() + assert request_id in w._recving_transfers + + def test_get_finished_evicts_completed_state(self): + """``get_finished`` should enqueue evictions and wake the writer.""" + w = _StubWriterWorker.fresh() + + # Stub the base ``get_finished`` to return one done_sending entry. + # Patch via the MRO's parent class. + with patch.object( + NixlPushConnectorWorker.__mro__[1], + "get_finished", + return_value=({"req-done"}, set()), + ): + done_sending, done_recving = w.get_finished() + + assert "req-done" in done_sending + assert done_recving == set() + # Eviction enqueued for the writer. + evicted = [] + while True: + try: + evicted.append(w._evict_finished_inbox.get_nowait()) + except queue.Empty: + break + assert evicted == ["req-done"] + assert w._push_writer_wake.is_set() + + +# ----------------------------------------------------------------- # +# Negative / error-path tests # +# ----------------------------------------------------------------- # + + +class TestPushSchedulerNegative: + """Failure / no-op paths on the scheduler side.""" + + def test_update_state_after_alloc_no_kv_transfer_params_is_noop(self): + """Requests without kv_transfer_params must not register anything.""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + request = MagicMock() + request.request_id = "req-no-params" + request.kv_transfer_params = None + + sched.update_state_after_alloc( + request, _BlocksMock(([1, 2, 3],)), num_external_tokens=64 + ) + + assert sched._push_pending_registrations == {} + assert sched._push_registration_deadlines == {} + assert sched._reqs_need_recv == {} + + def test_update_state_after_alloc_zero_external_tokens_does_not_register(self): + """num_external_tokens=0 should not stage a D registration.""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + request = _make_request(request_id="req-zero-ext") + sched.update_state_after_alloc( + request, _BlocksMock(([1, 2, 3],)), num_external_tokens=0 + ) + + assert sched._push_pending_registrations == {} + assert sched._push_registration_deadlines == {} + + def test_request_finished_unfinished_status_does_not_stage(self): + """If a request is still RUNNING, request_finished must not stash + blocks for the worker (no push needed).""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + request = _make_request( + request_id="req-running", is_d_side=False, finished=False + ) + + delay, ret = sched.request_finished(request, ([1, 2, 3],)) + + assert delay is False + assert ret is None + assert sched._finished_request_blocks == {} + assert sched._newly_finished_push_blocks == {} + + def test_request_finished_empty_blocks_does_not_arm_lease(self): + """Empty block-id groups should still complete cleanly without + arming the lease/finished maps.""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + request = _make_request(request_id="req-empty", is_d_side=False) + delay, ret = sched.request_finished(request, ((),)) + + assert delay is False + assert ret is not None + assert "req-empty" not in sched._finished_request_blocks + assert "req-empty" not in sched._newly_finished_push_blocks + assert "req-empty" not in sched._reqs_need_send + + def test_update_connector_output_unknown_request_is_noop(self): + """Idempotent cleanup: clearing a request that was never staged + must not raise or mutate other state.""" + sched = make_nixl_push_scheduler() + _stub_sw_clipping(sched) + + # Stage one real request to ensure it's NOT touched. + live = _make_request(request_id="req-live", is_d_side=False) + sched.request_finished(live, ([1],)) + + sched.update_connector_output( + KVConnectorOutput( + finished_sending={"unknown-1"}, + finished_recving={"unknown-2"}, + invalid_block_ids=set(), + ) + ) + + # Live entry untouched. + assert "req-live" in sched._finished_request_blocks + + +class TestPushWriterNegative: + """Failure / drop / idempotence paths in the writer thread.""" + + def test_pop_matching_registration_returns_none_when_empty(self): + w = _StubWriterWorker.fresh() + assert w._pop_matching_registration("nope") is None + + def test_pop_matching_finished_blocks_returns_none_when_empty(self): + w = _StubWriterWorker.fresh() + assert w._pop_matching_finished_blocks("nope") is None + + def test_pop_matching_registration_no_match_when_base_ids_differ(self): + """A registration whose base id (after stripping the random suffix) + does NOT match the lookup request_id must not be popped.""" + w = _StubWriterWorker.fresh() + # Two unrelated requests: different base UUIDs, so stripping the + # trailing ``-<8 hex>`` suffix still yields different base ids. + unrelated_d = "cmpl-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa-0-11111111" + lookup = "cmpl-bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb-0-22222222" + assert get_base_request_id(unrelated_d) != get_base_request_id(lookup) + + w._pending_d_registrations[unrelated_d] = _registration_data(unrelated_d) + result = w._pop_matching_registration(lookup) + assert result is None + # Original entry untouched. + assert unrelated_d in w._pending_d_registrations + + def test_handle_push_reg_with_non_dict_payload_is_dropped(self, caplog): + """msgpack-encoded non-dict payload (e.g. a list) should be + dropped without raising.""" + caplog.set_level( + logging.CRITICAL, + logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), + ) + w = _StubWriterWorker.fresh() + bad = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode([1, 2, 3]) + w._handle_push_reg_notif(bad) + assert w._pending_d_registrations == {} + assert w.start_push_calls == [] + + def test_handle_push_reg_with_non_string_request_id_is_dropped(self, caplog): + """request_id must be a str; integers, None, etc. must drop.""" + caplog.set_level( + logging.CRITICAL, + logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), + ) + w = _StubWriterWorker.fresh() + for bogus_rid in (123, None, 4.5, b"bytes-not-str"): + payload = _registration_data("placeholder") + payload["request_id"] = bogus_rid # type: ignore[assignment] + notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(payload) + w._handle_push_reg_notif(notif) + assert w._pending_d_registrations == {} + assert w.start_push_calls == [] + + def test_handle_push_reg_idempotent_for_same_request_id(self): + """Receiving the same PUSH_REG twice (e.g. P retries after a + flake) keeps the entry staged exactly once and never fires.""" + w = _StubWriterWorker.fresh() + notif = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode( + _registration_data("req-dup") + ) + w._handle_push_reg_notif(notif) + w._handle_push_reg_notif(notif) + assert "req-dup" in w._pending_d_registrations + assert len(w._pending_d_registrations) == 1 + assert w.start_push_calls == [] + + def test_get_finished_enqueues_eviction_for_each_done_request(self): + """``get_finished`` must enqueue an eviction for every request + in ``done_sending`` so the writer can drop stale matching state. + Unlike the happy-path test, this verifies the *cardinality*: N + completed requests -> N evictions, in order.""" + w = _StubWriterWorker.fresh() + with patch.object( + NixlPushConnectorWorker.__mro__[1], + "get_finished", + return_value=({"req-1", "req-2", "req-3"}, set()), + ): + done_sending, _ = w.get_finished() + assert done_sending == {"req-1", "req-2", "req-3"} + + evicted: list[str] = [] + while True: + try: + evicted.append(w._evict_finished_inbox.get_nowait()) + except queue.Empty: + break + assert sorted(evicted) == ["req-1", "req-2", "req-3"] + + def test_get_finished_with_no_completions_does_not_enqueue_eviction(self): + """If there's nothing newly done, no eviction should be enqueued. + The wake event IS still set because ``get_finished`` always wakes + the writer to drain notifs.""" + w = _StubWriterWorker.fresh() + with patch.object( + NixlPushConnectorWorker.__mro__[1], + "get_finished", + return_value=(set(), set()), + ): + done_sending, done_recving = w.get_finished() + assert done_sending == set() + assert done_recving == set() + assert w._evict_finished_inbox.qsize() == 0 + # Wake set so the writer drains NIXL notifs even when idle. + assert w._push_writer_wake.is_set() + + def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog): + """A completion notif for a request the worker doesn't know + about should be logged but not crash.""" + caplog.set_level( + logging.CRITICAL, + logger=("vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker"), + ) + w = _StubWriterWorker.fresh() + w.transfer_topo = MagicMock() + # Forward a completion notif for an unknown request_id. + w._pending_completion_notifs.put(b"never-heard-of-you:1") + + notified = w._get_new_notifs() + assert notified == set() + # Did not register anywhere. + assert "never-heard-of-you" not in w._recving_transfers + + def test_start_load_kv_with_empty_metadata_is_noop(self): + """Empty metadata must not wake the writer or enqueue anything.""" + w = _StubWriterWorker.fresh() + w._send_heartbeats = lambda metadata: None + w._logical_to_kernel_block_ids = lambda x: x + + meta = NixlConnectorMetadata() + w.start_load_kv(meta) + + assert w._reg_send_inbox.qsize() == 0 + assert w._finished_blocks_inbox.qsize() == 0 + # Wake should NOT be set if there was nothing to push. + assert not w._push_writer_wake.is_set() + + def test_get_new_notifs_extends_lease_on_heartbeat(self): + """``HB:`` notifs forwarded by the writer thread must extend the + leases of tracked P-side requests on the engine main thread, and + ignore request IDs that aren't being tracked.""" + w = _StubWriterWorker.fresh() + w.transfer_topo = MagicMock() + # _handle_heartbeat reads ``self._lease_extension`` (set in the + # real ``__init__``). + w._lease_extension = 10 + + # Tracked P-side requests with a lease about to expire. + old_expiry = time.perf_counter() - 5.0 + w._reqs_to_send["req-a"] = old_expiry + w._reqs_to_send["req-b"] = old_expiry + + # Forwarded heartbeat covers a tracked request, an unknown one, + # and another tracked one. + w._pending_completion_notifs.put(b"HB:req-a,req-unknown,req-b") + + notified = w._get_new_notifs() + assert notified == set() + + # Tracked leases were renewed strictly forward in time. + now = time.perf_counter() + for rid in ("req-a", "req-b"): + assert w._reqs_to_send[rid] > old_expiry + # New expiry must be roughly now + _lease_extension. + assert w._reqs_to_send[rid] >= now + # Unknown request must not be inserted by the heartbeat path. + assert "req-unknown" not in w._reqs_to_send diff --git a/tests/v1/kv_connector/unit/test_nixl_simple_cpu_offload.py b/tests/v1/kv_connector/unit/test_nixl_simple_cpu_offload.py index 0760d7141e..78e9e1196f 100644 --- a/tests/v1/kv_connector/unit/test_nixl_simple_cpu_offload.py +++ b/tests/v1/kv_connector/unit/test_nixl_simple_cpu_offload.py @@ -44,7 +44,7 @@ from vllm.v1.simple_kv_offload.metadata import ( ) NIXL_WRAPPER_PATCH = ( - "vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker.NixlWrapper" + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper" ) diff --git a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py index d92b632676..95e8254fe4 100644 --- a/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py +++ b/tests/v1/kv_connector/unit/test_remote_prefill_lifecycle.py @@ -587,7 +587,9 @@ def test_cannot_recv(): assert_scheduler_empty(scheduler) -@patch("vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler.current_platform") +@patch( + "vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler.current_platform" +) def test_p_side_chunked_prefill_mamba(mock_platform): """P-side integration: Mamba N-1 truncation + chunked prefill completes. diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index c5411be620..7df9e20e6a 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -524,3 +524,66 @@ def make_nixl_scheduler( sched.blocks_per_sw = [] sched.is_bidirectional_kv_xfer_enabled = False return sched + + +def make_nixl_push_scheduler( + *, + decoder_kv_blocks_ttl: float = 30.0, + push_registration_timeout: float | None = None, + is_bidirectional_kv_xfer_enabled: bool = False, + has_mamba: bool = False, +): + """Create a NixlPushConnectorScheduler via __new__ (skipping __init__). + + The push scheduler can't reuse :func:`make_nixl_scheduler` because it + is a different class (``NixlPushConnectorScheduler`` vs + ``NixlConnectorScheduler``) and carries push-specific state. Only the + fields touched by the unit tests are populated. + """ + from unittest.mock import MagicMock + + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( + NixlPushConnectorScheduler, + ) + + sched = object.__new__(NixlPushConnectorScheduler) + + # Base scheduler fields (shared with pull / heartbeat path). + sched._reqs_need_recv = {} + sched._reqs_need_send = {} + sched._reqs_in_batch = set() + sched._reqs_not_processed = set() + sched._reqs_need_save = {} + sched._kv_lease_duration = 30 + sched.decoder_kv_blocks_ttl = decoder_kv_blocks_ttl + sched.use_host_buffer = False + sched.engine_id = "decode-engine" + sched.side_channel_host = "127.0.0.1" + sched.side_channel_port = 5600 + sched.is_bidirectional_kv_xfer_enabled = is_bidirectional_kv_xfer_enabled + sched._has_mamba = has_mamba + + # vllm_config is consulted for parallel_config.tensor_parallel_size. + vllm_config = MagicMock() + vllm_config.parallel_config.tensor_parallel_size = 1 + sched.vllm_config = vllm_config + + # Push-specific state. + sched._push_pending_registrations = {} + sched._push_registration_deadlines = {} + sched._finished_request_blocks = {} + sched._newly_finished_push_blocks = {} + sched._push_registration_timeout = ( + push_registration_timeout + if push_registration_timeout is not None + else decoder_kv_blocks_ttl + ) + + # Heartbeat fields touched by base request_finished / + # update_connector_output. + sched._heartbeat_by_engine = {} + sched._heartbeat_req_engine = {} + sched._last_heartbeat_time = 0.0 + sched.blocks_per_sw = [] + + return sched diff --git a/vllm/distributed/kv_transfer/kv_connector/factory.py b/vllm/distributed/kv_transfer/kv_connector/factory.py index 75290f6a01..aad7999d08 100644 --- a/vllm/distributed/kv_transfer/kv_connector/factory.py +++ b/vllm/distributed/kv_transfer/kv_connector/factory.py @@ -179,6 +179,18 @@ KVConnectorFactory.register_connector( "NixlConnector", ) +KVConnectorFactory.register_connector( + "NixlPullConnector", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl", + "NixlPullConnector", +) + +KVConnectorFactory.register_connector( + "NixlPushConnector", + "vllm.distributed.kv_transfer.kv_connector.v1.nixl", + "NixlPushConnector", +) + KVConnectorFactory.register_connector( "MultiConnector", "vllm.distributed.kv_transfer.kv_connector.v1.multi_connector", diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/base.py b/vllm/distributed/kv_transfer/kv_connector/v1/base.py index 71d89f43a7..954fedafe8 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/base.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/base.py @@ -569,6 +569,18 @@ class KVConnectorBase_V1(ABC): """ return () + def has_pending_push_work(self) -> bool: + """Return True if the connector has push-mode work that requires + the engine main loop to keep stepping (e.g. a P-side request whose + KV blocks are waiting to be WRITTEN to a D node). + + Connectors that don't implement push-based KV transfer should + leave this as False. + """ + # TODO: replace with a more general connector hook for keeping the + # scheduler alive (e.g. extend has_unfinished_requests). + return False + @classmethod def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None: """ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py index 46354337e6..bfb6ee466a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py @@ -538,6 +538,9 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA): for c in self._connectors: yield from c.take_events() + def has_pending_push_work(self) -> bool: + return any(c.has_pending_push_work() for c in self._connectors) + @classmethod def get_required_kvcache_layout(cls, vllm_config: "VllmConfig") -> str | None: """ diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py index ed5c892fb9..fd5996f64b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/__init__.py @@ -2,14 +2,35 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """NIXL KV-cache transfer connector (disaggregated prefill / decode).""" +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( + NixlBaseConnectorScheduler, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( + NixlBaseConnectorWorker, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.connector import ( + NixlBaseConnector, NixlConnector, + NixlPullConnector, + NixlPushConnector, ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( NixlAgentMetadata, NixlConnectorMetadata, NixlHandshakePayload, ) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import ( + NixlPullConnectorWorker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( + NixlPushConnectorScheduler, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( + NixlPushConnectorWorker, +) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import ( NixlConnectorScheduler, ) @@ -22,10 +43,19 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( __all__ = [ "NixlAgentMetadata", + "NixlBaseConnector", + "NixlBaseConnectorScheduler", + "NixlBaseConnectorWorker", "NixlConnector", "NixlConnectorMetadata", "NixlConnectorScheduler", "NixlConnectorWorker", "NixlHandshakePayload", "NixlKVConnectorStats", + "NixlPullConnector", + "NixlPullConnectorScheduler", + "NixlPullConnectorWorker", + "NixlPushConnector", + "NixlPushConnectorScheduler", + "NixlPushConnectorWorker", ] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py new file mode 100644 index 0000000000..cba81cadd8 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_scheduler.py @@ -0,0 +1,455 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Base scheduler-side logic for the NIXL connector.""" + +import threading +import time +from typing import TYPE_CHECKING, Any + +import msgspec +import zmq + +from vllm import envs +from vllm.distributed.kv_transfer.kv_connector.utils import ( + BlockIds, + EngineId, + yield_req_data, +) +from vllm.distributed.kv_transfer.kv_connector.v1.base import ( + KVConnectorHandshakeMetadata, + KVConnectorMetadata, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + GET_META_MSG, + HeartbeatInfo, + NixlConnectorMetadata, + NixlHandshakePayload, + ReqId, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils.math_utils import cdiv +from vllm.utils.network_utils import make_zmq_path +from vllm.v1.core.sched.output import SchedulerOutput +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + MambaSpec, + SlidingWindowSpec, +) + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + from vllm.v1.kv_cache_interface import KVCacheConfig + from vllm.v1.outputs import KVConnectorOutput + from vllm.v1.request import Request + +logger = init_logger(__name__) + + +class NixlBaseConnectorScheduler: + """Base implementation of Scheduler side methods shared by pull and push.""" + + def __init__( + self, + vllm_config: "VllmConfig", + engine_id: str, + kv_cache_config: "KVCacheConfig", + ): + self.vllm_config = vllm_config + self.block_size = vllm_config.cache_config.block_size + self.engine_id: EngineId = engine_id + self.kv_cache_config = kv_cache_config + self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST + self.side_channel_port = ( + envs.VLLM_NIXL_SIDE_CHANNEL_PORT + + vllm_config.parallel_config.data_parallel_index + ) + assert vllm_config.kv_transfer_config is not None + self._kv_lease_duration: int = ( + vllm_config.kv_transfer_config.get_from_extra_config( + "kv_lease_duration", 30 + ) + ) + # NOTE (NickLucche): For now we use a hardcoded value for a simpler interface. + self._heartbeat_interval = self._kv_lease_duration // 6 + if current_platform.device_type == "cpu": + self.use_host_buffer = False + else: + self.use_host_buffer = ( + vllm_config.kv_transfer_config.kv_buffer_device == "cpu" + ) + self._is_hma_required = ( + not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager + # Also handle unlikely SW-only model case instead of checking num_groups>1. + and any( + not isinstance(g.kv_cache_spec, FullAttentionSpec) + for g in kv_cache_config.kv_cache_groups + ) + ) + self._has_mamba = any( + isinstance(g.kv_cache_spec, MambaSpec) + for g in kv_cache_config.kv_cache_groups + ) + + logger.info("Initializing NIXL Scheduler %s", engine_id) + if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager: + logger.info("Hybrid Memory Allocator is enabled with NIXL") + + # Background thread for handling new handshake requests. + self._nixl_handshake_listener_t: threading.Thread | None = None + self._stop_event = threading.Event() + + # Requests that need to start recv/send. + # New requests are added by update_state_after_alloc in + # the scheduler. Used to make metadata passed to Worker. + self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {} + self._reqs_need_save: dict[ReqId, Request] = {} + # Reqs to send and their expiration time + self._reqs_need_send: dict[ReqId, float] = {} + self._reqs_in_batch: set[ReqId] = set() + # Reqs to remove from processed set because they're not to send after + # remote prefill or aborted. + self._reqs_not_processed: set[ReqId] = set() + + # Heartbeat tracking: requests needing periodic lease-renewal heartbeats to + # remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine + self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {} + # Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal + self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {} + self._last_heartbeat_time: float = 0.0 + + # Gather Sliding Window sizes for each kv cache group (if any) in number of + # blocks per KV cache group. This is used to clip the local attention window. + sw_sizes_tokens: list[tuple[int, int]] = [ + (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) + if isinstance(g.kv_cache_spec, SlidingWindowSpec) + else (0, self.block_size) + for g in kv_cache_config.kv_cache_groups + ] + # cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively + # account for boundary overlap eg window isn't fully aligned with blocks. + self.blocks_per_sw = [ + cdiv(n_tokens, block_size) + 1 if n_tokens else 0 + for n_tokens, block_size in sw_sizes_tokens + ] + + # Threshold to decide whether to compute kv cache locally + # or pull from a remote node: minimum number of remote + # tokens to amortize the xfer latencies + self.kv_recompute_threshold: int = int( + vllm_config.kv_transfer_config.get_from_extra_config( + "kv_recompute_threshold", 64 + ) + ) + + # Bi-directional KV transfer feature supports KV block + # transfers from D node to P node + self.is_bidirectional_kv_xfer_enabled = ( + vllm_config.kv_transfer_config.get_from_extra_config( + "bidirectional_kv_xfer", False + ) + ) + self.decoder_kv_blocks_ttl = ( + vllm_config.kv_transfer_config.get_from_extra_config( + "decoder_kv_blocks_ttl", 480 + ) + ) + + if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0: + logger.info( + "Bidirectional KV transfer is enabled and the kv " + "recompute threshold is set to %d tokens." + "KV blocks on D are released after a TTL of %d seconds.", + self.kv_recompute_threshold, + self.decoder_kv_blocks_ttl, + ) + + def shutdown(self): + self._stop_event.set() + if self._nixl_handshake_listener_t is not None: + self._nixl_handshake_listener_t.join() + self._nixl_handshake_listener_t = None + + def on_new_request(self, request: "Request") -> None: + """Track a request that may need heartbeats.""" + params = request.kv_transfer_params + # NOTE (NickLucche) This excludes request meant for P, ie heartbeats are + # effectively disabled for Bidirectional KV transfer. + if params is None or not params.get("do_remote_prefill"): + return + # Only track if all required remote fields are present. + remote_engine_id = params.get("remote_engine_id") + remote_request_id = params.get("remote_request_id") + host = params.get("remote_host") + port = params.get("remote_port") + tp_size = params.get("tp_size") + if ( + remote_engine_id is None + or remote_request_id is None + or host is None + or port is None + or tp_size is None + ): + return + if remote_engine_id not in self._heartbeat_by_engine: + self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo( + req_ids=set(), + host=host, + port=port, + tp_size=tp_size, + ) + self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id) + self._heartbeat_req_engine[request.request_id] = ( + remote_engine_id, + remote_request_id, + ) + + def _stop_heartbeat(self, req_id: ReqId) -> None: + """Remove *req_id* from heartbeat tracking (if tracked).""" + if key := self._heartbeat_req_engine.pop(req_id, None): + engine_id, remote_id = key + if info := self._heartbeat_by_engine.get(engine_id): + info.req_ids.discard(remote_id) + if not info.req_ids: + # Clean up empty engines so we don't leak a key when remote dies. + del self._heartbeat_by_engine[engine_id] + + def get_sw_clipped_blocks(self, block_ids: BlockIds) -> BlockIds: + """ + Clip the number of blocks to the sliding window size for each kv cache group + that employs SWA. + This is necessary because the KV Cache manager initially allocates blocks for + the entire sequence length, and successively cleans up blocks that are outside + the window prior to the `request_finished_all_groups` hook. + """ + if len(block_ids) == 0 or not self._is_hma_required: + # No blocks to clip eg Full prefix cache hit or not a hybrid model. + return block_ids + # NOTE (NickLucche) This logic is currently handled at the connector level + # because offloading connectors might want to receive the whole sequence even + # for SWA groups. We will abstract this logic once the interface is more stable + assert len(block_ids) == len(self.blocks_per_sw), ( + "Number of KV cache groups must match" + ) + # For non-SWA groups, blocks_per_sw is 0 so we return all block_ids unchanged + return tuple( + [ + blocks[-self.blocks_per_sw[i] :] + if self.blocks_per_sw[i] > 0 + else blocks + for i, blocks in enumerate(block_ids) + ] + ) + + def set_xfer_handshake_metadata( + self, metadata: dict[int, KVConnectorHandshakeMetadata] + ) -> None: + """ + Set the KV connector handshake metadata for this connector. + + Args: + metadata (dict): the handshake metadata to set. + """ + encoded_data: dict[int, bytes] = {} + encoder = msgspec.msgpack.Encoder() + for tp_rank, rank_metadata in metadata.items(): + if not isinstance(rank_metadata, NixlHandshakePayload): + raise ValueError( + "NixlConnectorScheduler expects NixlHandshakePayload for " + "handshake metadata." + ) + encoded_data[tp_rank] = encoder.encode(rank_metadata) + logger.debug( + "Tp rank %d: encoded NixlHandshakePayload size: %s bytes", + tp_rank, + str(len(encoded_data[tp_rank])), + ) + + # Only start the listener when we have metadata to serve. + if self._nixl_handshake_listener_t is None: + ready_event = threading.Event() + self._nixl_handshake_listener_t = threading.Thread( + target=self._nixl_handshake_listener, + args=( + encoded_data, + ready_event, + self._stop_event, + self.side_channel_host, + self.side_channel_port, + ), + daemon=True, + name="nixl_handshake_listener", + ) + self._nixl_handshake_listener_t.start() + ready_event.wait() # Wait for listener ZMQ socket to be ready. + + @staticmethod + def _nixl_handshake_listener( + encoded_data: dict[int, Any], + ready_event: threading.Event, + stop_event: threading.Event, + host: str, + port: int, + ): + """Background thread for getting new NIXL handshakes.""" + # NOTE(rob): this is a simple implementation. We will move + # to a better approach via HTTP endpoint soon. + + # Listen for new requests for metadata. + path = make_zmq_path("tcp", host, port) + logger.debug("Starting listening on path: %s", path) + with zmq_ctx(zmq.ROUTER, path) as sock: + sock.setsockopt(zmq.RCVTIMEO, 1000) + ready_event.set() + while True: + try: + identity, _, msg = sock.recv_multipart() + except zmq.Again: + if stop_event.is_set(): + break + continue + # Decode the message which contains (GET_META_MSG, rank) + msg, target_tp_rank = msgspec.msgpack.decode(msg) + logger.debug( + "Received message for tp rank %s", + target_tp_rank, + ) + if msg != GET_META_MSG: + logger.warning("Connection listener got unexpected message %s", msg) + sock.send_multipart((identity, b"", encoded_data[target_tp_rank])) + + def _mamba_prefill_token_count(self, num_prompt_tokens: int) -> int: + """D-side only. Returns N-1 for Mamba models since the decoder + always recomputes the last token and must start from h(N-1).""" + if self._has_mamba and num_prompt_tokens > 1: + return num_prompt_tokens - 1 + return num_prompt_tokens + + def _truncate_mamba_request_for_prefill(self, request: "Request") -> None: + """P-side only: drop the last prompt token so the prefiller computes + h(N-1) instead of h(N). The decoder recomputes the last token to + derive h(N) correctly. + + Guarded by ``_p_side_truncated`` to avoid repeated truncation if the + request is preempted and rescheduled.""" + params = request.kv_transfer_params + if ( + params is not None + # Guard against repeated truncation after preemption/reschedule. + and not params.get("_p_side_truncated") + and request.num_prompt_tokens > 1 + ): + if request.prompt_token_ids is not None: + request.prompt_token_ids.pop() + elif request.prompt_embeds is not None: + request.prompt_embeds = request.prompt_embeds[:-1] + else: + return + + request._all_token_ids.pop() + request.num_prompt_tokens -= 1 + request.max_tokens = 1 + params["_p_side_truncated"] = True + + def _build_save_meta( + self, + meta: NixlConnectorMetadata, + scheduler_output: SchedulerOutput, + ) -> None: + # only called when use_host_buffer is True to build the save metadata + + # NOTE: For the prefill side, there might be a chance that an early added + # request is a chunked prefill, so we need to check if new blocks are added + for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output): + req_to_save = self._reqs_need_save.get(req_id) + if req_to_save is None or new_block_id_groups is None: + continue + req = req_to_save + + assert req.kv_transfer_params is not None + clipped_block_id_groups = self.get_sw_clipped_blocks(new_block_id_groups) + meta.add_new_req_to_save( + request_id=req_id, + local_block_ids=clipped_block_id_groups, + kv_transfer_params=req.kv_transfer_params, + ) + assert scheduler_output.num_scheduled_tokens is not None + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] + is_partial = ( + req.num_computed_tokens + num_scheduled_tokens + ) < req.num_prompt_tokens + if not is_partial: + # For non-partial prefills, once new req_meta is scheduled, it + # can be removed from _reqs_need_save. + # For partial prefill case, we will retain the request in + # _reqs_need_save until all blocks are scheduled with req_meta. + # Therefore, only pop if `not is_partial`. + self._reqs_need_save.pop(req_id) + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + ) -> KVConnectorMetadata: + meta = NixlConnectorMetadata() + + # Loop through scheduled reqs and convert to ReqMeta. + for req_id, (req, block_ids) in self._reqs_need_recv.items(): + assert req.kv_transfer_params is not None + meta.add_new_req_to_recv( + request_id=req_id, + local_block_ids=block_ids, + kv_transfer_params=req.kv_transfer_params, + ) + + if self.use_host_buffer: + self._build_save_meta(meta, scheduler_output) + + meta.reqs_to_send = self._reqs_need_send + meta.reqs_in_batch = self._reqs_in_batch + meta.reqs_not_processed = self._reqs_not_processed + + # Package heartbeats, throttled by heartbeat_interval. + if self._heartbeat_by_engine: + now = time.perf_counter() + if now - self._last_heartbeat_time >= self._heartbeat_interval: + self._last_heartbeat_time = now + meta.heartbeat_by_engine = self._heartbeat_by_engine + + # Clear the list once workers start the transfers + self._reqs_need_recv.clear() + self._reqs_in_batch = set() + self._reqs_not_processed = set() + self._reqs_need_send = {} + + return meta + + def update_connector_output(self, connector_output: "KVConnectorOutput") -> None: + """Stop heartbeating for requests whose KV transfer completed.""" + for req_id in connector_output.finished_recving or (): + self._stop_heartbeat(req_id) + + def has_pending_push_work(self) -> bool: + return False + + ############################################################ + # Abstract methods that subclasses must implement + ############################################################ + + def get_num_new_matched_tokens( + self, request: "Request", num_computed_tokens: int + ) -> tuple[int, bool]: + raise NotImplementedError + + def update_state_after_alloc( + self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int + ): + raise NotImplementedError + + def request_finished( + self, + request: "Request", + block_ids: BlockIds, + ) -> tuple[bool, dict[str, Any] | None]: + raise NotImplementedError diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py new file mode 100644 index 0000000000..e587b0cd1f --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py @@ -0,0 +1,2286 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Base worker-side logic for the NIXL connector.""" + +import logging +import os +import queue +import threading +import time +import uuid +from collections import defaultdict +from collections.abc import Iterator +from concurrent.futures import Future, ThreadPoolExecutor +from typing import TYPE_CHECKING, Any, cast + +import msgspec +import numpy as np +import torch +import zmq + +from vllm.distributed.kv_transfer.kv_connector.utils import ( + BlockIds, + EngineId, + EngineTransferInfo, + TransferTopology, + get_current_attn_backends, + kv_postprocess_blksize_and_layout_on_receive, + kv_postprocess_blksize_on_receive, + kv_postprocess_layout_on_receive, +) +from vllm.distributed.kv_transfer.kv_connector.v1.base import CopyBlocksOp +from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + GET_META_MSG, + NixlAgentMetadata, + NixlConnectorMetadata, + NixlHandshakePayload, + ReqId, + ReqMeta, + TransferHandle, + compute_nixl_compatibility_hash, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( + NixlKVConnectorStats, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + TPMapping, + _is_attention_spec, + _is_ssm_spec, + compute_tp_mapping, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( + _NIXL_SUPPORTED_DEVICE, + get_representative_spec_type, + zmq_ctx, +) +from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( + MambaConvSplitInfo, + derive_mamba_conv_split, +) +from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config +from vllm.distributed.parallel_state import ( + get_tensor_model_parallel_rank, + get_tensor_model_parallel_world_size, +) +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils.network_utils import make_zmq_path +from vllm.v1.attention.backends.utils import get_kv_cache_layout +from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + MambaSpec, + MLAAttentionSpec, + UniformTypeKVCacheSpecs, +) +from vllm.v1.worker.block_table import BlockTable +from vllm.v1.worker.utils import select_common_block_size + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.kv_cache_interface import KVCacheConfig + +logger = init_logger(__name__) + + +class NixlBaseConnectorWorker: + """Base implementation of Worker side methods shared by pull and push.""" + + def _compute_desc_ids( + self, + block_ids: BlockIds, + dst_num_blocks: int, + block_size_ratio: float | None, + physical_blocks_per_logical: int, + ) -> np.ndarray: + """Compute NIXL descriptor IDs for given block IDs.""" + num_fa_regions = self.num_regions + num_ssm_regions = len(self.block_len_per_layer) * 4 if self._has_mamba else 0 + + num_blocks = dst_num_blocks + if block_size_ratio is not None: + num_blocks = int(num_blocks * block_size_ratio) + num_fa_descs = num_fa_regions * num_blocks + + # All-attention fast path: single vectorized broadcast. + if num_ssm_regions == 0: + # NOTE (NickLucche) With HMA, every kv group has the same number of layers + # and layers from different groups share the same kv tensor. + # eg block_ids=[[1, 2], [3]]->blocks [1, 2] need to be + # read across all regions, same for [3], but group0-group1 blocks will + # always differ (different areas). Therefore we can just flatten the + # block_ids and compute the descs ids for all groups at once. + block_arr = np.concatenate(block_ids)[None, :] + region_ids = np.arange(num_fa_regions)[:, None] + return (region_ids * num_blocks + block_arr).flatten() + + # Compute desc ids per group using the right stride: FA descs have + # num_blocks entries per region (kernel granularity), SSM descs have + # logical_blocks entries per region (no kernel splitting). + logical_blocks = num_blocks // physical_blocks_per_logical + all_descs: list[np.ndarray] = [] + for i, group in enumerate(block_ids): + group_arr = np.asarray(group) + if _is_attention_spec(self._group_spec_types[i]): + fa_region_ids = np.arange(num_fa_regions)[:, None] + all_descs.append( + (fa_region_ids * num_blocks + group_arr[None, :]).flatten() + ) + elif _is_ssm_spec(self._group_spec_types[i]): + # NOTE (NickLucche) SSM and Attention block regions can + # be exchanged arbitrarily by manager. Therefore, descs + # are laid out as: + # [descs_fa (all regions) | descs_ssm (all regions)]. + # num_fa_descs offset must be computed per-engine since + # P and D can have different num_blocks (and thus + # different FA desc counts). + ssm_region_ids = np.arange(num_ssm_regions)[:, None] + all_descs.append( + ( + ssm_region_ids * logical_blocks + + group_arr[None, :] + + num_fa_descs + ).flatten() + ) + else: + raise ValueError( + f"Unknown spec type {self._group_spec_types[i]} at index {i}" + ) + + return np.concatenate(all_descs) + + def _build_local_splits_from_plan( + self, + plan: TPMapping, + src_blocks_data: list[tuple[int, int, int]], + num_fa_descs: int, + ) -> Iterator[list[tuple[int, int, int]]]: + """Build split handle data for P_TP > D_TP scenario. + + num_fa_descs is the boundary between FA and SSM descriptors. + Split counts are derived from source_ranks_per_group lengths. + FA uses rank_to_attention_slot for the slot offset; + SSM uses the rank's positional index. + """ + fa_idx = next( + i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) + ) + fa_num_splits = len(plan.source_ranks_per_group[fa_idx]) + + has_ssm_descs = num_fa_descs < len(src_blocks_data) + ssm_idx = next( + (i for i, t in enumerate(self._group_spec_types) if _is_ssm_spec(t)), + None, + ) + ssm_num_splits = ( + len(plan.source_ranks_per_group[ssm_idx]) + if has_ssm_descs and ssm_idx is not None + else 0 + ) + + # Per-FA-descriptor replicate flag, in _build_fa_local emission order. + fa_desc_replicated = self._fa_desc_replicated(num_fa_descs) + + for p_idx, p_rank in enumerate(plan.all_source_ranks): + fa_slot = plan.rank_to_attention_slot.get(p_rank, 0) + + handle: list[tuple[int, int, int]] = [] + for j, (addr, local_len, dev) in enumerate(src_blocks_data): + if j < num_fa_descs: + if fa_desc_replicated[j]: + # REPLICATE (MLA): whole block written on every rank. + handle.append((addr, local_len, dev)) + else: + # SPLIT (full-attn): this rank's head slice. + chunk = local_len // fa_num_splits + handle.append((addr + fa_slot * chunk, chunk, dev)) + else: + chunk = local_len // ssm_num_splits + handle.append((addr + p_idx * chunk, chunk, dev)) + yield handle + + def _fa_desc_replicated(self, num_fa_descs: int) -> list[bool]: + """Per-FA-descriptor replicate flag, in _build_fa_local emission order + (region-major; K then optional V per region). Length ``num_fa_descs``. + """ + assert self.transfer_topo is not None + n_regions = len(self.block_len_per_layer) + if n_regions == 0 or self.num_regions == 0: + return [False] * num_fa_descs + nblk = num_fa_descs // self.num_regions + virtually_split = self.transfer_topo.virtually_split_kv_in_blocks + flags: list[bool] = [] + for i in range(n_regions): + replicated = self._is_region_replicated(i) + num_streams = 1 if replicated or not virtually_split else 2 + flags.extend([replicated] * (num_streams * nblk)) + assert len(flags) == num_fa_descs, ( + f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}" + ) + return flags + + def _is_region_replicated(self, region_idx: int) -> bool: + """Whether region ``region_idx`` is transferred REPLICATE vs SPLIT. + + REPLICATE (MLA): identical on every rank, whole block read from one + rank at offset 0, key-only. SPLIT (full-attn): head-sharded across TP. + Defaults to SPLIT when the per-region map is unset (e.g. tests that set + block_len_per_layer without register_kv_caches). + """ + return region_idx < len(self._region_is_mla) and self._region_is_mla[region_idx] + + def __init__( + self, + vllm_config: "VllmConfig", + engine_id: str, + kv_cache_config: "KVCacheConfig", + ): + nixl_wrapper_cls = NixlWrapper + if nixl_wrapper_cls is None: + logger.error("NIXL is not available") + raise RuntimeError("NIXL is not available") + logger.info("Initializing NIXL wrapper") + logger.info("Initializing NIXL worker %s", engine_id) + + # Config. + self.vllm_config = vllm_config + # mypy will complain on re-assignment otherwise. + self.block_size: int = cast(int, vllm_config.cache_config.block_size) + + if vllm_config.kv_transfer_config is None: + raise ValueError("kv_transfer_config must be set for NixlConnector") + self.kv_transfer_config = vllm_config.kv_transfer_config + + self.nixl_backends = vllm_config.kv_transfer_config.get_from_extra_config( + "backends", ["UCX"] + ) + kv_lease_duration: int = vllm_config.kv_transfer_config.get_from_extra_config( + "kv_lease_duration", 30 + ) + # NOTE (NickLucche): For now we use a hardcoded value for a simpler interface. + self._lease_extension = kv_lease_duration * 2 // 3 + + self._is_hma_required = ( + not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager + and any( + not isinstance(g.kv_cache_spec, FullAttentionSpec) + for g in kv_cache_config.kv_cache_groups + ) + ) + self.kv_cache_config = kv_cache_config + self._layer_specs = { + layer: group.kv_cache_spec + for group in kv_cache_config.kv_cache_groups + for layer in group.layer_names + } + self.hma_group_size = len(kv_cache_config.kv_cache_tensors) + + # ---- Model state (derived from model config) ---- + mamba_ssm_size = (0, 0) + # Conv state sub-projection decomposition (None when no Mamba). + # The 3-read transfer requires DS (dim, state_len) conv layout so + # that x/B/C sub-projections are contiguous in memory. + self._conv_decomp: MambaConvSplitInfo | None = None + self._has_mamba = any( + isinstance(g.kv_cache_spec, MambaSpec) + for g in kv_cache_config.kv_cache_groups + ) + if self._has_mamba: + assert self._is_hma_required + from vllm.model_executor.layers.mamba.mamba_utils import ( + is_conv_state_dim_first, + ) + + assert is_conv_state_dim_first(), ( + "3-read Mamba conv transfer requires DS conv state layout. " + "Set VLLM_SSM_CONV_STATE_LAYOUT=DS" + ) + mamba_spec = next( + spec + for spec in self._layer_specs.values() + if isinstance(spec, MambaSpec) + ) + self._conv_decomp = derive_mamba_conv_split( + mamba_spec, + vllm_config.parallel_config.tensor_parallel_size, + ) + mamba_ssm_size = self._conv_decomp.ssm_sizes + self._mamba_ssm_size = mamba_ssm_size + + # Agent. + non_ucx_backends = [b for b in self.nixl_backends if b != "UCX"] + # Configure NIXL num_threads to avoid UAR exhaustion on Mellanox NICs. + # Each UCX thread allocates UARs (doorbell pages) via DevX, and + # excessive NIXL UAR usage can exhaust NIC UAR space. This can cause + # components like NVSHMEM (used by DeepEP kernels) to fail during RDMA + # initialization with "mlx5dv_devx_alloc_uar" errors. + # Ref: https://network.nvidia.com/files/doc-2020/ethernet-adapters-programming-manual.pdf#page=63 + num_threads = vllm_config.kv_transfer_config.get_from_extra_config( + "num_threads", 4 + ) + if nixl_agent_config is None: + config = None + else: + # Enable telemetry by default for NIXL 0.7.1 and above. + config = ( + nixl_agent_config(backends=self.nixl_backends, capture_telemetry=True) + if len(non_ucx_backends) > 0 + else nixl_agent_config(num_threads=num_threads, capture_telemetry=True) + ) + + self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config) + # Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}. + self._remote_agents: dict[EngineId, dict[int, str]] = defaultdict(dict) + + # Metadata. + self.engine_id: EngineId = engine_id + self.tp_rank = get_tensor_model_parallel_rank() + self.world_size = get_tensor_model_parallel_world_size() + + self.num_blocks = kv_cache_config.num_blocks + self.enable_permute_local_kv = False + self.enable_heterogeneous_attn_post_process = False + + # KV Caches and nixl tracking data. + self.device_type = current_platform.device_type + self.kv_buffer_device: str = vllm_config.kv_transfer_config.kv_buffer_device + if self.device_type not in _NIXL_SUPPORTED_DEVICE: + raise RuntimeError(f"{self.device_type} is not supported.") + elif self.kv_buffer_device not in _NIXL_SUPPORTED_DEVICE[self.device_type]: + raise RuntimeError( + f"{self.device_type} with {self.kv_buffer_device} kv_buffer " + "is not supported." + ) + self.device_kv_caches: dict[str, torch.Tensor] = {} + + # cpu kv buffer for xfer + # used when device memory can not be registered under nixl + self.host_xfer_buffers: dict[str, torch.Tensor] = {} + if self.device_type == "cpu": + self.use_host_buffer = False + else: + self.use_host_buffer = self.kv_buffer_device == "cpu" + + # reserve different cores for start_load_kv() from model_forward() + if self.device_type == "cpu": + numa_core_list = current_platform.discover_numa_topology() + # setup one last core in each numa for kv transfer. + rsv_cores_for_kv = [ + max(each_numa_core_list) for each_numa_core_list in numa_core_list + ] + + if rsv_cores_for_kv: + if not hasattr(os, "sched_setaffinity"): + raise NotImplementedError( + "os.sched_setaffinity is not available on this platform" + ) + os.sched_setaffinity(0, rsv_cores_for_kv) + + # support for oot platform which can't register nixl memory + # type based on kv_buffer_device + nixl_memory_type = current_platform.get_nixl_memory_type() + if nixl_memory_type is None: + if self.kv_buffer_device in ["cuda", "xpu"]: + nixl_memory_type = "VRAM" + elif self.kv_buffer_device == "cpu": + nixl_memory_type = "DRAM" + if nixl_memory_type is None: + raise RuntimeError( + f"{self.device_type} with {self.kv_buffer_device} kv_buffer " + "is not supported." + ) + self.nixl_memory_type = nixl_memory_type + + # Note: host xfer buffer ops when use_host_buffer is True + self.copy_blocks: CopyBlocksOp | None = None + + # Map of engine_id -> kv_caches_base_addr. For TP case, each local + self.device_id: int = 0 + # Current rank may pull from multiple remote TP workers. + # EngineId, dict[int, list[int]] -> engine_id, tp_rank, base_addr_for_layer + self.kv_caches_base_addr = defaultdict[EngineId, dict[int, list[int]]](dict) + + # Number of NIXL regions. Currently one region per cache + # (so 1 per layer for MLA, otherwise 2 per layer) + self.num_regions = 0 + + # nixl_prepped_dlist_handle. + self.src_xfer_handles_by_block_size: dict[int, int] = {} + # Populated dynamically during handshake based on remote configuration. + # Keep track of regions at different tp_ratio values. tp_ratio->handles + self.src_xfer_handles_by_tp_ratio: dict[int, list[int]] = {} + # Map of engine_id -> {tp_rank: nixl_prepped_dlist_handle (int)}. + self.dst_xfer_side_handles = defaultdict[EngineId, dict[int, int]](dict) + + # Map of engine_id -> num_blocks. All ranks in the same deployment will + # have the same number of blocks. + self.dst_num_blocks: dict[EngineId, int] = {} + self._registered_descs: list[Any] = [] + + # In progress transfers. + # [req_id -> list[handle]] + self._recving_metadata: dict[ReqId, ReqMeta] = {} + self._recving_transfers = defaultdict[ReqId, list[TransferHandle]](list) + # Track the expiration time of requests that are waiting to be sent. + self._reqs_to_send: dict[ReqId, float] = {} + # Set of requests that have been part of a batch, regardless of status. + self._reqs_to_process: set[ReqId] = set() + + # Invalid blocks from failed NIXL operations (thread-safe queue of block ids) + self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() + # requests that skipped transfer (handshake or transfer failures) + # Uses Queue for thread-safe cross-thread coordination with the + # background handshake thread, matching the _ready_requests pattern. + self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() + + # Handshake metadata of this worker for NIXL transfers. + self.xfer_handshake_metadata: NixlHandshakePayload | None = None + # Background thread for initializing new NIXL handshakes. + self._handshake_initiation_executor = ThreadPoolExecutor( + # NIXL is not guaranteed to be thread-safe, limit 1 worker. + max_workers=1, + thread_name_prefix="vllm-nixl-handshake-initiator", + ) + self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]() + self._handshake_futures: dict[EngineId, Future[dict[int, str]]] = {} + # Protects _handshake_futures and _remote_agents. + self._handshake_lock = threading.RLock() + + # TTL-based eviction of stale remote engine state. + self._engine_last_active: dict[EngineId, float] = {} + self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( + "engine_ttl", 3600.0 + ) + + self.block_size = vllm_config.cache_config.block_size + self.model_config = vllm_config.model_config + + self.use_mla = self.model_config.use_mla + + # Get the attention backend from the first layer + # NOTE (NickLucche) models with multiple backends are not supported yet + self.attn_backends = get_current_attn_backends(vllm_config) + self.backend_name = self.attn_backends[0].get_name() + + self.kv_cache_layout = get_kv_cache_layout() + self.host_buffer_kv_cache_layout = self.kv_cache_layout + logger.info( + "Detected attention backend(s) %s", + [backend.get_name() for backend in self.attn_backends], + ) + logger.info("Detected kv cache layout %s", self.kv_cache_layout) + + # lazy initialized in register_kv_caches + self.compat_hash: str | None = None + self.transfer_topo: TransferTopology | None = None + + # With heterogeneous TP, P must wait for all assigned D TP workers to + # finish reading before safely freeing the blocks. + self.consumer_notification_counts_by_req = defaultdict[ReqId, int](int) + self.xfer_stats = NixlKVConnectorStats() + + self._physical_blocks_per_logical_kv_block = 1 + self._sync_block_size_with_kernel() + + # Unwrap UniformTypeKVCacheSpecs to get the representative spec type + self._group_spec_types = tuple( + get_representative_spec_type(g.kv_cache_spec) + for g in self.kv_cache_config.kv_cache_groups + ) + + # Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE + # (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models + # combining both (e.g. GQA main + MLA Eagle-3 draft). + self._region_is_mla = list[bool]() + + # Enable different block lengths for different layers *only* when MLA is used. + # This is not used for SSM layers, which use the counterpart `mamba_ssm_size`. + self.block_len_per_layer = list[int]() + + # Per-engine TP mappings. Generated during handshake. + self.tp_mappings: dict[EngineId, TPMapping] = {} + + self.enforce_compat_hash = self.kv_transfer_config.get_from_extra_config( + "enforce_handshake_compat", True + ) + + def _sync_block_size_with_kernel(self) -> None: + backends = get_current_attn_backends(self.vllm_config) + kernel_block_size = select_common_block_size(self.block_size, backends) + # Number of blocks not accounting for kernel block mismatches + self._logical_num_blocks = self.num_blocks + if self.block_size != kernel_block_size: + logger.info_once( + "User-specified logical block size (%s) does not match" + " physical kernel block size (%s). Using the latter.", + self.block_size, + kernel_block_size, + ) + assert self.block_size > kernel_block_size + self._physical_blocks_per_logical_kv_block = ( + self.block_size // kernel_block_size + ) + self.block_size = kernel_block_size + self.num_blocks *= self._physical_blocks_per_logical_kv_block + + def _nixl_handshake( + self, + host: str, + port: int, + remote_tp_size: int, + expected_engine_id: str, + ) -> dict[int, str]: + """Do a NIXL handshake with a remote instance.""" + + # the first time we connect to a remote agent. + # be careful, the handshake happens in a background thread. + # it does not have an active cuda context until any cuda runtime + # call is made. when UCX fails to find a valid cuda context, it will + # disable any cuda ipc communication, essentially disabling any NVLink + # communication. + # when we are using device buffers, we need to set the device + # explicitly to make sure the handshake background thread has a valid + # cuda context. + if not self.use_host_buffer: + current_platform.set_device(self.device_id) + + # When target instance TP > local TP, we need to perform multiple + # handshakes. Do it in a single background job for simplicity. + # Regardless, only handshake with the remote TP rank(s) that current + # local rank will read from. Note that With homogeneous TP, + # this happens to be the same single rank_i. + assert self.transfer_topo is not None + p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size) + remote_rank_to_agent_name = {} + path = make_zmq_path("tcp", host, port) + + with zmq_ctx(zmq.REQ, path) as sock: + for remote_rank in p_remote_ranks: + logger.debug( + "Querying metadata on path: %s at remote tp rank %s", + path, + remote_rank, + ) + + start_time = time.perf_counter() + # Send query for the request. + msg = msgspec.msgpack.encode((GET_META_MSG, remote_rank)) + # Set receive timeout to 5 seconds to avoid hanging on dead server + sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds + sock.send(msg) + handshake_bytes = sock.recv() + + # Decode handshake payload to get compatibility hash + handshake_decoder = msgspec.msgpack.Decoder(NixlHandshakePayload) + try: + handshake_payload = handshake_decoder.decode(handshake_bytes) + except (msgspec.DecodeError, msgspec.ValidationError) as e: + raise RuntimeError( + f"Failed to decode NixlHandshakePayload. This likely indicates " + f"an incompatibility between connector version. Error: {e}" + ) from e + + got_metadata_time = time.perf_counter() + logger.debug( + "NIXL handshake: get metadata took: %s", + got_metadata_time - start_time, + ) + + # Check compatibility hash BEFORE decoding agent metadata + assert self.compat_hash is not None + if ( + self.enforce_compat_hash + and handshake_payload.compatibility_hash != self.compat_hash + ): + raise RuntimeError( + f"NIXL compatibility hash mismatch. " + f"Local: {self.compat_hash}, " + f"Remote: {handshake_payload.compatibility_hash}. " + f"Prefill and decode instances have incompatible " + f"configurations. This may be due to: different vLLM versions," + f" models, dtypes, KV cache layouts, attention backends, etc. " + f"Both instances must use identical configurations." + f"Disable this check using " + f'--kv-transfer-config \'{{"kv_connector_extra_config": ' + f'{{"enforce_handshake_compat": false}}}}\'' + ) + + logger.info( + "NIXL compatibility check passed (hash: %s)", + handshake_payload.compatibility_hash, + ) + + # Decode agent metadata + metadata_decoder = msgspec.msgpack.Decoder(NixlAgentMetadata) + try: + metadata = metadata_decoder.decode( + handshake_payload.agent_metadata_bytes + ) + except (msgspec.DecodeError, msgspec.ValidationError) as e: + # This should not happen if hash matched + raise RuntimeError( + f"Failed to decode NixlAgentMetadata. Error: {e}" + ) from e + + # Ensure engine id matches. + if metadata.engine_id != expected_engine_id: + raise RuntimeError( + f"Remote NIXL agent engine ID mismatch. " + f"Expected {expected_engine_id}," + f"received {metadata.engine_id}." + ) + + # Register Remote agent. + remote_agent_name = self.add_remote_agent( + metadata, remote_rank, remote_tp_size + ) + setup_agent_time = time.perf_counter() + logger.debug( + "NIXL handshake: add agent took: %s", + setup_agent_time - got_metadata_time, + ) + remote_rank_to_agent_name[remote_rank] = remote_agent_name + return remote_rank_to_agent_name + + def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None: + """ + Initialize transfer buffer in CPU mem for accelerators + NOT directly supported by NIXL (e.g., tpu) + """ + xfer_buffers: dict[str, torch.Tensor] = {} + inv_order = [0, 1, 3, 2, 4] + try: + for layer_name, kv_cache in kv_caches.items(): + kv_shape = kv_cache.shape + kv_dtype = kv_cache.dtype + permute_shape = False + if ( + self.kv_cache_layout == "NHD" + and self.vllm_config.kv_transfer_config is not None + and self.vllm_config.kv_transfer_config.enable_permute_local_kv + ): + logger.info_once( + "'enable_permute_local_kv' flag is enabled while " + "device KV Layout is NHD. Init host buffer with" + " HND to better support Decode/Prefill TP_ratio > 1." + ) + # Since NHD will not support Decode/Prefill TP_ratio > 1, + # we can leverage host_buffer for permute + self.host_buffer_kv_cache_layout = "HND" + kv_shape = ( + tuple(kv_shape[i] for i in inv_order) + if not self.use_mla + else kv_shape + ) + permute_shape = not self.use_mla + + xfer_buffers[layer_name] = torch.empty( + kv_shape, dtype=kv_dtype, device="cpu" + ) + if permute_shape: + xfer_buffers[layer_name] = xfer_buffers[layer_name].permute( + inv_order + ) + except MemoryError as e: + logger.error("NIXLConnectorWorker gets %s.", e) + raise + + self.host_xfer_buffers = xfer_buffers + + def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp): + """Assign copy (d2h, h2d) operations when host buffer is used.""" + # Set a no-op if the host buffer is not cpu. + if self.kv_buffer_device != "cpu": + return + # Set a no-op if self.device_type is 'cpu'. + if self.device_type == "cpu": + return + assert self.use_host_buffer + self.copy_blocks = copy_operation + + def _log_failure( + self, + failure_type: str, + req_id: str | None, + msg: str = "", + error: Exception | None = None, + meta: ReqMeta | None = None, + **extra_context, + ): + """Log transfer failure with structured context for easier debugging.""" + context: dict[str, Any] = { + "failure_type": failure_type, + "request_id": req_id, + "engine_id": self.engine_id, + } + if meta is None and req_id is not None: + # Try to get metadata from in progress transfers when not provided + meta = self._recving_metadata.get(req_id) + + if meta and meta.remote: + context.update( + { + "remote_engine_id": meta.remote.engine_id, + "remote_request_id": meta.remote.request_id, + "remote_host": meta.remote.host, + "remote_port": meta.remote.port, + "num_local_blocks": sum( + len(group) for group in meta.local_block_ids + ), + "num_remote_blocks": sum( + len(group) for group in meta.remote.block_ids + ), + "local_block_ids_sample": meta.local_block_ids[0][:10] + if meta.local_block_ids + else [], + } + ) + + context.update(extra_context) + if msg: + failure_type = f"{failure_type}. {msg}" + + logger.error( + "NIXL transfer failure: %s | Context: %s", + failure_type, + context, + exc_info=error is not None, + stacklevel=2, + ) + + def _ensure_handshake( + self, + engine_id: EngineId, + host: str, + port: int, + tp_size: int, + ) -> Future[dict[int, str]] | None: + """ + Ensure a handshake is in-flight (or already done) for *engine_id*. + + Returns the ``Future`` if a handshake is pending (or was just + started), or ``None`` if the handshake already completed + successfully. Callers can attach per-request callbacks to the + returned future. + Failures to handshake are logged and the request is marked as failed. + """ + self._evict_stale_engines() + with self._handshake_lock: + if engine_id in self._remote_agents: + return None + fut = self._handshake_futures.get(engine_id) + if fut is not None: + return fut + fut = self._handshake_initiation_executor.submit( + self._nixl_handshake, + host, + port, + tp_size, + engine_id, + ) + self._handshake_futures[engine_id] = fut + + def done_callback(f: Future[dict[int, str]], eid=engine_id): + with self._handshake_lock: + del self._handshake_futures[eid] + try: + self._remote_agents[eid] = f.result() + self._engine_last_active[eid] = time.perf_counter() + except Exception as e: + self._log_failure( + failure_type="handshake_setup_failed", + req_id=None, + error=e, + remote_engine_id=eid, + ) + + fut.add_done_callback(done_callback) + return fut + + def _background_nixl_handshake( + self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta + ): + # Do NIXL handshake in background and add to _ready_requests when done. + assert meta.remote is not None + fut = self._ensure_handshake( + remote_engine_id, + meta.remote.host, + meta.remote.port, + meta.tp_size, + ) + if fut is None: + # Already handshaked — only happens if caller does not pre-check. + self._ready_requests.put((req_id, meta)) + return + + # Check handshake success before proceeding with request. + def request_ready(f: Future[Any], entry=(req_id, meta)): + try: + f.result() + self._ready_requests.put(entry) + except Exception as e: + self._log_failure( + failure_type="handshake_failed", + req_id=req_id, + error=e, + meta=meta, + ) + self._handle_failed_transfer(req_id, None) + + fut.add_done_callback(request_ready) + + def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None: + """Register a cross-layers KV cache tensor with NIXL. + + `use_uniform_kv_cache()` guarantees a single KV cache group whose + layers all share the same `AttentionSpec`, so any layer name from + `_layer_specs` yields the correct per-layer spec for `page_size_bytes`. + """ + first_layer = next(iter(self._layer_specs)) + # Forwarding a real layer name rather than a synthetic key + self.register_kv_caches({first_layer: kv_cache}) + + def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): + """Register the KV Cache data in nixl.""" + self.transfer_topo = TransferTopology( + tp_rank=self.tp_rank, + tp_size=self.world_size, + block_size=self.block_size, + engine_id=self.engine_id, + is_mla=self.use_mla, + total_num_kv_heads=self.model_config.get_total_num_kv_heads(), + attn_backends=self.attn_backends, + # SSM States come in tuples (ssm, conv) + tensor_shape=next(iter(kv_caches.values())).shape + if not self._has_mamba + else None, + is_mamba=self._has_mamba, + ) + self.compat_hash = compute_nixl_compatibility_hash( + self.vllm_config, self.backend_name, self.transfer_topo.cross_layers_blocks + ) + + if self.use_host_buffer: + self.initialize_host_xfer_buffer(kv_caches=kv_caches) + assert len(self.host_xfer_buffers) == len(kv_caches), ( + f"host_buffer: {len(self.host_xfer_buffers)}, " + f"kv_caches: {len(kv_caches)}" + ) + xfer_buffers = self.host_xfer_buffers + else: + xfer_buffers = kv_caches + assert not self.host_xfer_buffers, ( + "host_xfer_buffer should not be initialized when " + f"kv_buffer_device is {self.kv_buffer_device}" + ) + + logger.info( + "Registering KV_Caches. use_mla: %s, kv_buffer_device: %s, " + "use_host_buffer: %s", + self.use_mla, + self.kv_buffer_device, + self.use_host_buffer, + ) + + caches_data = [] + # With hybrid allocator, layers can share a kv cache tensor + seen_base_addresses = [] + + # Note(tms): I modified this from the original region setup code. + # K and V are now in different regions. Advantage is that we can + # elegantly support MLA and any cases where the K and V tensors + # are non-contiguous (it's not locally guaranteed that they will be) + # Disadvantage is that the encoded NixlAgentMetadata is now larger + # (roughly 8KB vs 5KB). + # Conversely for FlashInfer, K and V are registered in the same region + # to better exploit the memory layout (ie num_blocks is the first dim). + tensor_size_bytes = None + + for layer_name, cache_or_caches in xfer_buffers.items(): + # NOTE (NickLucche) Hybrid SSM models assume a layout that is similar to + # that of FI, with block laid out as in `get_backend_aware_kv_block_len`. + # However, physical page_size may differ when kernel requires a specific + # block size. This leads to SSM and FA layers having different num_blocks. + # `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this. + layer_spec = self._layer_specs.get(layer_name) + if layer_spec is None: + logger.debug( + "Skipping layer %s as no KVCache spec is present. " + "This is likely because the layer is sharing its KV cache", + layer_name, + ) + continue + if isinstance(layer_spec, UniformTypeKVCacheSpecs): + # MLA DSv32 Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs + layer_spec = layer_spec.kv_cache_specs[layer_name] + cache_list = self.transfer_topo.get_transfer_cache_regions( + cache_or_caches, layer_spec + ) + # `layer_spec.page_size_bytes` only accounts for logical page_size, that is + # the page_size assuming constant `self._logical_num_blocks`. + physical_page_size = ( + layer_spec.page_size_bytes + if isinstance(layer_spec, MambaSpec) + else layer_spec.page_size_bytes + // self._physical_blocks_per_logical_kv_block + ) + # For when registering multiple tensors eg K/V in separate regions. + physical_page_size = physical_page_size // len(cache_list) + if self.transfer_topo._cross_layers_blocks: + # When cross-layers blocks are used, multiply by number of layers + physical_page_size = physical_page_size * len( + self.kv_cache_config.kv_cache_tensors + ) + num_blocks = ( + self._logical_num_blocks + if isinstance(layer_spec, MambaSpec) + else self.num_blocks + ) + # `page_size` accounts for physical blocks, st KVCache is always + # [`num_blocks` * `page_size`] + curr_tensor_size_bytes = num_blocks * physical_page_size + + # TODO (NickLucche) we could eventually unify how we handle FA/FI regions, + # registering a single tensor for both K/V and splitting logically like FI. + for cache in cache_list: + base_addr = cache.data_ptr() + if base_addr in seen_base_addresses: + # NOTE (NickLucche) HMA employs memory pooling to share tensors + # across groups. This results in skipping all tensors but the ones + # pointed to by group0. Also, generally we will have more blocks + # per tensor but fewer regions. + logger.debug("Skipping %s because it's already seen", layer_name) + continue + logger.debug( + "Registering layer %s with cache shape: %s", layer_name, cache.shape + ) + seen_base_addresses.append(base_addr) + # Only record non-Mamba page sizes. + if isinstance(layer_spec, MambaSpec): + self.block_len_per_layer.append( + physical_page_size // self._physical_blocks_per_logical_kv_block + ) + else: + self.block_len_per_layer.append(physical_page_size) + is_mla_region = isinstance(layer_spec, MLAAttentionSpec) + self._region_is_mla.append(is_mla_region) + + if not is_mla_region: + if tensor_size_bytes is None: + tensor_size_bytes = curr_tensor_size_bytes + assert tensor_size_bytes == curr_tensor_size_bytes, ( + "All non-MLA kv cache tensors must have the same size" + ) + + if cache.shape[0] != num_blocks: + raise AssertionError( + "All kv cache tensors must have the same number of " + f"blocks; layer={layer_name}, " + f"expected_num_blocks={num_blocks}, " + f"cache_shape={tuple(cache.shape)}, " + f"cache_stride={tuple(cache.stride())}, " + f"layer_spec={type(layer_spec).__name__}, " + f"backend={self.backend_name}, " + "all_backends=" + f"{[backend.get_name() for backend in self.attn_backends]}, " + f"kv_cache_layout={self.kv_cache_layout}, " + "blocks_first=" + f"{self.transfer_topo.is_kv_layout_blocks_first}" + ) + + # Need to make sure the device ID is non-negative for NIXL, + # Torch uses -1 to indicate CPU tensors. + self.device_id = max(cache.get_device(), 0) + caches_data.append( + (base_addr, curr_tensor_size_bytes, self.device_id, "") + ) + + logger.debug( + "Different block lengths collected: %s", set(self.block_len_per_layer) + ) + assert ( + len(self.block_len_per_layer) + == len(seen_base_addresses) + == len(self._region_is_mla) + ) + + self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses + self.num_regions = len(caches_data) + + if self.transfer_topo.virtually_split_kv_in_blocks: + # NOTE (NickLucche) When FlashInfer is used, memory is registered + # with joint KV for each block. This minimizes the overhead in + # registerMem allowing faster descs queries. In order to be able to + # split on kv_heads dim as required by heterogeneous TP, one must + # be able to index K/V separately. Hence we double the number + # of 'virtual' regions here and halve `block_len` below. + # Similarly for Mamba layers, we register SSM+Conv as a single region and + # then duplicate it logically to be able to index SSM/Conv separately. + # Exception: key-only REPLICATE regions (MLA) have no V half, so + # they contribute a single desc stream and are not doubled. + self.num_regions = sum( + 1 if self._is_region_replicated(i) else 2 + for i in range(len(self._region_is_mla)) + ) + + # Total local FA descriptors (boundary between FA and mamba descs). + self.num_descs = self.num_regions * self.num_blocks + + descs = self.nixl_wrapper.get_reg_descs(caches_data, self.nixl_memory_type) + logger.debug("Registering descs: %s", caches_data) + self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends) + logger.debug("Done registering descs") + self._registered_descs.append(descs) + + self.device_kv_caches = kv_caches + self.dst_num_blocks[self.engine_id] = self.num_blocks + + if self._has_mamba: + logger.info( + "Hybrid SSM registration: num_blocks=%s, " + "logical_num_blocks=%s, ratio=%s, num_regions=%s, " + "num_descs=%s, mamba_ssm_size=%s, block_len_per_layer=%s", + self.num_blocks, + self._logical_num_blocks, + self._physical_blocks_per_logical_kv_block, + self.num_regions, + self.num_descs, + self._mamba_ssm_size, + set(self.block_len_per_layer), + ) + + # Register local/src descr for NIXL xfer. + self.src_xfer_handles_by_block_size[self.block_size], self.src_blocks_data = ( + self.register_local_xfer_handler(self.block_size) + ) + + # After KV Caches registered, listen for new connections. + agent_metadata = NixlAgentMetadata( + engine_id=self.engine_id, + agent_metadata=self.nixl_wrapper.get_agent_metadata(), + device_id=self.device_id, + kv_caches_base_addr=self.kv_caches_base_addr[self.engine_id][self.tp_rank], + num_blocks=self.num_blocks, + block_lens=self.block_len_per_layer, + kv_cache_layout=self.kv_cache_layout + if not self.use_host_buffer + else self.host_buffer_kv_cache_layout, + block_size=self.block_size, + ssm_sizes=self._mamba_ssm_size, + attn_backend_name=self.backend_name, + physical_blocks_per_logical_kv_block=( + self._physical_blocks_per_logical_kv_block + ), + ) + # Wrap metadata in payload with hash for defensive decoding + assert self.compat_hash is not None + encoder = msgspec.msgpack.Encoder() + self.xfer_handshake_metadata = NixlHandshakePayload( + compatibility_hash=self.compat_hash, + agent_metadata_bytes=encoder.encode(agent_metadata), + ) + + def _build_mamba_local( + self, + base_addresses: list[int], + block_size_ratio: int, + ) -> list[tuple[int, int, int]]: + """Build 4 desc regions (x, B, C, ssm) per layer for local mamba + blocks, enabling the 3-read transfer with DS conv layout.""" + assert block_size_ratio == 1, ( + "Mamba 3-read transfer with block_size_ratio != 1 is not tested. " + f"Got block_size_ratio={block_size_ratio}." + ) + assert self._conv_decomp is not None + conv_offsets = self._conv_decomp.local_conv_offsets + conv_size, ssm_size = self._mamba_ssm_size + num_blocks = self._logical_num_blocks * block_size_ratio + physical_per_logical = self._physical_blocks_per_logical_kv_block + + result: list[tuple[int, int, int]] = [] + for i, base_addr in enumerate(base_addresses): + # Jump one page_size, but ssm page_size may be bigger when kernel + # locks block size to a specific value (physical_per_logical scale). + page_stride = ( + self.block_len_per_layer[i] // block_size_ratio * physical_per_logical + ) + for off, sz in conv_offsets: + for blk in range(num_blocks): + result.append( + (base_addr + blk * page_stride + off, sz, self.device_id) + ) + # SSM temporal state follows the conv state. + for blk in range(num_blocks): + result.append( + ( + base_addr + blk * page_stride + conv_size, + ssm_size, + self.device_id, + ) + ) + return result + + def _build_mamba_remote( + self, + nixl_agent_meta: NixlAgentMetadata, + tp_ratio: int, + transfer_info: EngineTransferInfo, + ) -> list[tuple[int, int, int]]: + """Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer + for the 3-read transfer. For hetero-TP, each D rank reads only its + sub-projection slice from the P rank.""" + assert self._conv_decomp is not None + effective_ratio = max(tp_ratio, 1) + # Mamba conv state is always TP-sharded, even when attention KV + # is replicated (num_kv_heads < tp_size). + local_offset = self.tp_rank % effective_ratio + conv_size_remote = nixl_agent_meta.ssm_sizes[0] + + conv_offsets = self._conv_decomp.remote_conv_offsets(local_offset, tp_ratio) + if tp_ratio >= 1: + ssm_read_size = self._mamba_ssm_size[1] + else: + ssm_read_size = nixl_agent_meta.ssm_sizes[1] + + remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical + num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical + device_id = nixl_agent_meta.device_id + + result: list[tuple[int, int, int]] = [] + # NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case + # block lengths vary across layers (e.g. MLA). + for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): + page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical + for off, sz in conv_offsets: + for blk in range(num_blocks): + result.append((base_addr + blk * page_stride + off, sz, device_id)) + # SSM temporal state is also TP-sharded on the heads dimension. + for blk in range(num_blocks): + ssm_addr = ( + base_addr + + blk * page_stride + + conv_size_remote + + local_offset * ssm_read_size + ) + result.append((ssm_addr, ssm_read_size, device_id)) + return result + + def _build_fa_local( + self, + base_addresses: list[int], + block_size_ratio: int, + ) -> list[tuple[int, int, int]]: + """Build local FA descriptors for all layers.""" + assert self.transfer_topo is not None + num_blocks = self.num_blocks * block_size_ratio + result: list[tuple[int, int, int]] = [] + for i, base_addr in enumerate(base_addresses): + kv_block_len = ( + self.get_backend_aware_kv_block_len( + layer_idx=i, first_split=True, mamba_view=False + ) + // block_size_ratio + ) + page_stride = self.block_len_per_layer[i] // block_size_ratio + for block_id in range(num_blocks): + block_offset = block_id * page_stride + addr = base_addr + block_offset + result.append((addr, kv_block_len, self.device_id)) + + if ( + self.transfer_topo.virtually_split_kv_in_blocks + and not self._is_region_replicated(i) + ): + # Separate and interleave K/V regions to maintain the same + # descs ordering. This is needed for selecting contiguous heads + # when split across TP ranks. (Skipped for key-only REPLICATE.) + second_split = self.get_backend_aware_kv_block_len( + layer_idx=i, first_split=False, mamba_view=False + ) + for block_id in range(num_blocks): + block_offset = block_id * page_stride + addr = base_addr + block_offset + v_addr = addr + kv_block_len + result.append((v_addr, second_split, self.device_id)) + return result + + def _build_fa_remote( + self, + plan: TPMapping, + nixl_agent_meta: NixlAgentMetadata, + block_size_ratio: int, + ) -> list[tuple[int, int, int]]: + """Build remote FA descriptors for all layers.""" + assert self.transfer_topo is not None + fa_group_idx = next( + i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) + ) + # SPLIT regions read their head slice from this many remote ranks at a + # per-rank offset; REPLICATE regions read the whole block once. + split_reads = len(plan.source_ranks_per_group[fa_group_idx]) + num_blocks = nixl_agent_meta.num_blocks + result: list[tuple[int, int, int]] = [] + for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): + replicated = self._is_region_replicated(i) + # Read our whole local region size from remote.. + local_block_len = self.get_backend_aware_kv_block_len( + layer_idx=i, first_split=True, mamba_view=False + ) + remote_kv_block_len = local_block_len // block_size_ratio + if block_size_ratio > 1: + # ..using remote kv_block_len as transfer unit + local_block_len = remote_kv_block_len + + # REPLICATE reads the whole block once at offset 0; SPLIT gathers + # its head slice from `split_reads` remote ranks at a per-rank offset. + num_reads = 1 if replicated else split_reads + rank_offset = ( + 0 if replicated else plan.rank_offset_factor * remote_kv_block_len + ) + local_block_len = local_block_len // num_reads + + page_size = nixl_agent_meta.block_lens[i] + for block_id in range(num_blocks): + block_offset = block_id * page_size + # For each block, grab the kv heads chunk belonging to current local + # tp rank of size local_block_len. + addr = base_addr + block_offset + rank_offset + result.append((addr, local_block_len, nixl_agent_meta.device_id)) + + emits_v = self.transfer_topo.virtually_split_kv_in_blocks and not replicated + if emits_v: + # With FlashInfer index V separately to allow head splitting. + second_split = self.get_backend_aware_kv_block_len( + layer_idx=i, first_split=False, mamba_view=False + ) + second_split = second_split // num_reads + for block_id in range(num_blocks): + block_offset = block_id * page_size + addr = base_addr + block_offset + rank_offset + # Hop over the first split of remote page, K, to read V. + v_addr = addr + nixl_agent_meta.block_lens[i] // 2 + result.append((v_addr, second_split, nixl_agent_meta.device_id)) + return result + + def register_local_xfer_handler( + self, + block_size: int, + ) -> tuple[int, list[tuple[int, int, int]]]: + """ + Function used for register local xfer handler with local block_size or + Remote block_size. + + When local block_size is same as remote block_size, we use local block_size + to register local_xfer_handler during init. + + When remote block size is less than local block size, we need to use + register another local_xfer_handler using remote block len to ensure + data copy correctness. + """ + assert self.transfer_topo is not None + block_size_ratio = self.block_size // block_size + local_base_addresses = self.kv_caches_base_addr[self.engine_id][self.tp_rank] + + blocks_data = self._build_fa_local(local_base_addresses, block_size_ratio) + logger.debug( + "Created %s blocks for src engine %s and rank %s on device id %s", + len(blocks_data), + self.engine_id, + self.tp_rank, + self.device_id, + ) + if self._has_mamba: + assert self.num_descs == len(blocks_data) + # TODO (ZhanqiuHu): For homogeneous TP (tp_ratio == 1), the 3-descs split + # is unnecessary — a single conv desc per block suffices. Consider + # adding a fast path that falls back to the standard 2-region + # registration (_build_fa_local mamba=True) when no hetero-TP + # remote has been seen. Currently we always register 4 regions + # because local descs are created before knowing the remote TP. + logger.debug("Registering local Mamba descriptors (4 regions/layer)") + blocks_data.extend( + self._build_mamba_local(local_base_addresses, block_size_ratio) + ) + + descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type) + # NIXL_INIT_AGENT to be used for preparations of local descs. + return self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs), blocks_data + + def add_remote_agent( + self, + nixl_agent_meta: NixlAgentMetadata, + remote_tp_rank: int = 0, + remote_tp_size: int = 1, + ) -> str: + """ + Add the remote NIXL agent and prepare the descriptors for reading cache + blocks from remote. + + In particular, handle both homogeneous and heterogeneous TP. The former + requires local rank_i to read from remote rank_i. + The latter, in the case of D.world_size < P.world_size, requires that a + local (D) TP worker reads from multiple remote (P) TP workers. + Conversely, assuming D.world_size > P.world_size, two or more local TP + workers will read from a single remote TP worker. + + Here's an example for the last case described above (non-MLA): + + rank_offset p_remote_tp_rank + (kv split no) + -------------------------------- + 0 0 Worker0 ---- 1st half of KV ----> Worker0 [ KV Cache ] + / + 1 0 Worker1 ---- 2nd half of KV -----/ + + 0 1 Worker2 ---- 1st half of KV ----> Worker1 [ KV Cache ] + / + 1 1 Worker3 ---- 2nd half of KV -----/ + + + Decoder TP workers Prefix TP workers + (world_size=4) (world_size=2) + tp_ratio = 4 // 2 = 2 + + Considering the KV Caches, if P-Worker_i has cache size [2, num_blocksP, kv_heads, block_size, head_dim] + then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "HND" layout format. + Assuming num_blocksD >= num_blocksP, D-Worker0 reads from P-Worker0 by preparing the kv_heads//tp_ratio + first heads from all the slots of all the blocks. D-Worker1 will do the same, but reading the second split + along the kv_heads dimension, and so forth until "tp_ratio" D TP workers have pulled from P-Worker0. + + Note that the above will also hold true for the homogeneous TP case, where tp_ratio evaluates to 1. + + Regarding MLA case, the cache is replicated across TP workers so the rank_offset will just always be 0 + so that the whole cache is shared by "tp_ratio" D TP workers. + + For Mamba hetero-TP, both tp_ratio > 0 (D_TP > P_TP) and + tp_ratio < 0 (P_TP > D_TP) are supported by the 3-read transfer. + """ # noqa: E501 + engine_id = nixl_agent_meta.engine_id + # TODO re-evaluate refreshing for scaling/recovery + if remote_tp_rank in self._remote_agents.get(engine_id, {}): + logger.debug( + "Remote agent with engine_id %s and rank" + "%s already exchanged metadata, skip handshake.", + engine_id, + remote_tp_rank, + ) + return self._remote_agents[engine_id][remote_tp_rank] + + ### Register remote engine in TransferTopology (idempotent). + assert self.transfer_topo is not None + transfer_topo = self.transfer_topo + physical_blocks_per_logical = ( + nixl_agent_meta.physical_blocks_per_logical_kv_block + ) + transfer_info = EngineTransferInfo( + remote_tp_size=remote_tp_size, + remote_block_size=nixl_agent_meta.block_size, + remote_block_len=nixl_agent_meta.block_lens[0], + remote_physical_blocks_per_logical=physical_blocks_per_logical, + ) + transfer_topo.register_remote_engine(engine_id, transfer_info) + logger.info("Transfer plan: %s", transfer_topo.describe(engine_id)) + + self.tp_mappings[engine_id] = compute_tp_mapping( + transfer_topology=transfer_topo, + remote_tp_size=remote_tp_size, + group_spec_types=self._group_spec_types, + ) + + remote_agent_name = self.nixl_wrapper.add_remote_agent( + nixl_agent_meta.agent_metadata + ) + + # Create dst descs and xfer side handles. TP workers have same #blocks + # so we only register once per engine_id. + # Example: + # block_size_ratio > 1: + # remote: | 0| 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12| + # local origin:| 0| 1| 8| 12| + # local mapped:| 0| 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15| + block_size_ratio = transfer_topo.block_size_ratio(nixl_agent_meta.block_size) + + if engine_id not in self.dst_num_blocks: + self.dst_num_blocks[engine_id] = nixl_agent_meta.num_blocks + + # Keep track of remote agent kv caches base addresses. + self.kv_caches_base_addr[engine_id][remote_tp_rank] = ( + nixl_agent_meta.kv_caches_base_addr + ) + self._validate_remote_agent_handshake(nixl_agent_meta, remote_tp_size) + + # This is 1 when P and D `--tensor-parallel-size` match. Otherwise, + # this is the ratio between the two sizes. + tp_ratio = transfer_topo.tp_ratio(remote_tp_size) + + logger.debug( + "Registering remote agent (%s, rank %s) memory regions with tp_ratio %s", + engine_id, + remote_tp_rank, + tp_ratio, + ) + + plan = self.tp_mappings[engine_id] + + ### (Optional) Register local agent memory regions. MLA is not split. + if ( + tp_ratio < 0 + and not self.use_mla + and tp_ratio not in self.src_xfer_handles_by_tp_ratio + ): + # Remote tp_size > local tp_size: read from multiple remote ranks. + # Logically "split" own regions into |tp_ratio| chunks. Mind that + # we only do this once per remote tp_size (replica-friendly). + self.src_xfer_handles_by_tp_ratio[tp_ratio] = [] + + for handle_data in self._build_local_splits_from_plan( + plan, + self.src_blocks_data, + self.num_descs, + ): + descs = self.nixl_wrapper.get_xfer_descs( + handle_data, self.nixl_memory_type + ) + handle = self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs) + self.src_xfer_handles_by_tp_ratio[tp_ratio].append(handle) + + ### Register remote agent memory regions + # With homogeneous TP, D pulls the whole kv cache from corresponding rank. With + # heterogeneous TP, prepare the descriptors by splitting the P KV cache along + # kv_head dim, of D worker's kv_head size (D>P). + # Eg. PTP1 DTP2 => P0 KV:[block0-KV_0 | block0-KV_1..]. + + # Register all remote blocks, but only the corresponding kv heads. + blocks_data = self._build_fa_remote( + plan, + nixl_agent_meta, + block_size_ratio, + ) + logger.debug( + "Created %s blocks for dst engine %s with remote rank %s and local rank %s", + len(blocks_data), + engine_id, + remote_tp_rank, + self.tp_rank, + ) + if self._has_mamba: + logger.debug( + "Registering remote Mamba blocks for engine %s rank %s", + engine_id, + remote_tp_rank, + ) + blocks_data.extend( + self._build_mamba_remote( + nixl_agent_meta, + tp_ratio, + transfer_info, + ) + ) + + # Register with NIXL. + descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type) + self.dst_xfer_side_handles[engine_id][remote_tp_rank] = ( + self.nixl_wrapper.prep_xfer_dlist(remote_agent_name, descs) + ) + + if block_size_ratio > 1: + # when prefill with smaller block_size, we need to init a + # new handler with same block_len to match + self.src_xfer_handles_by_block_size[nixl_agent_meta.block_size] = ( + self.register_local_xfer_handler(nixl_agent_meta.block_size)[0] + ) + + return remote_agent_name + + def _validate_remote_agent_handshake( + self, nixl_agent_meta: NixlAgentMetadata, remote_tp_size: int + ): + """ + Validate the remote agent handshake metadata ensuring the + invariants hold true. + """ + remote_engine_id = nixl_agent_meta.engine_id + + assert self.transfer_topo is not None + remote_info = self.transfer_topo.get_engine_info(remote_engine_id) + assert remote_info.remote_tp_size == remote_tp_size + + tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size) + block_size_ratio = self.transfer_topo.block_size_ratio( + nixl_agent_meta.block_size + ) + # num_kv_heads > tp_size with P_TP > D_TP not supported for non-mamba. + # Mamba models can have replicated FA KV with tp_ratio < 0. + # MLA models do not need to handle kv replication. + if not self.use_mla and not self._has_mamba: + assert not ( + tp_ratio < 0 and self.transfer_topo.is_kv_replicated(remote_engine_id) + ) + + remote_physical_per_logical = ( + nixl_agent_meta.physical_blocks_per_logical_kv_block + ) + if ( + self._has_mamba + and remote_physical_per_logical + != self._physical_blocks_per_logical_kv_block + and self.vllm_config.cache_config.enable_prefix_caching + ): + raise RuntimeError( + "Prefix caching with heterogeneous physical_blocks_per_logical " + "is not supported for Mamba hybrid models. " + f"Local: {self._physical_blocks_per_logical_kv_block}, " + f"Remote: {remote_physical_per_logical}. " + "Disable prefix caching with --no-enable-prefix-caching." + ) + + if self._is_hma_required: + assert block_size_ratio == 1, ( + "HMA does not support different remote block size yet" + ) + kv_cache_layout = ( + self.kv_cache_layout + if not self.use_host_buffer + else self.host_buffer_kv_cache_layout + ) + if not self.use_mla and nixl_agent_meta.kv_cache_layout != kv_cache_layout: + if ( + self.kv_transfer_config.enable_permute_local_kv + and nixl_agent_meta.kv_cache_layout == "HND" + ): + logger.info( + "Remote is HND and local is NHD, enabled additional permute " + "on local device KV." + ) + assert not self._is_hma_required, ( + "HMA does not support block size post processing" + ) + self.enable_permute_local_kv = True + else: + raise RuntimeError( + "Heterogeneous TP expects same kv_cache_layout. " + "Or enable experimental feature to use HND to NHD support by " + "setting 'enable_permute_local_kv'=True in --kv-transfer-config." + ) + # if remote_agent used attn is not same as local, + # hint heterogenuous attn post process + if ( + nixl_agent_meta.attn_backend_name != self.backend_name + and self.backend_name in ["CPU_ATTN"] + ): + if self._is_hma_required: + raise RuntimeError( + "heterogeneous attn post process is not supported with HMA" + ) + logger.info( + "[Experimental] CPU_ATTN backend is used, " + "hint heterogeneous attn post process" + ) + self.enable_heterogeneous_attn_post_process = True + + # Heterogeneous TP requires head-splitting, which only works with + # HND layout. MLA and replicated-KV cases don't split on heads. + # Mamba doesn't support heterogeneous TP. + if ( + abs(tp_ratio) != 1 + and not self.use_mla + and not self.transfer_topo.is_kv_replicated(remote_engine_id) + and kv_cache_layout != "HND" + and not self.enable_permute_local_kv + ): + raise RuntimeError( + "Heterogeneous TP head-dimension splitting requires contiguous heads. " + "Use HND layout on the prefill side." + ) + + # Per-region block_len validation enforcing the P/D invariant. + # REPLICATE regions (MLA, or a whole-model MLA / replicated-KV transfer) + # only allow the number of blocks to differ; SPLIT regions scale with + # tp_ratio. Mamba uses the ssm_sizes counterpart, so skip block_len here. + if not self._has_mamba: + assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), ( + "Number of KV layers must match between prefill and decode" + ) + model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated( + remote_engine_id + ) + for i, local_len in enumerate(self.block_len_per_layer): + replicated = model_replicated or self._is_region_replicated(i) + remote_len = nixl_agent_meta.block_lens[i] + if replicated: + assert local_len // block_size_ratio == remote_len, ( + "KV cache sizes must match between P and D when " + f"replicated (region {i}: local={local_len}, " + f"remote={remote_len}, bsr={block_size_ratio})." + ) + elif tp_ratio > 0: + assert remote_len == (local_len * tp_ratio) // block_size_ratio, ( + f"SPLIT region {i}: remote P KV block_len {remote_len} " + f"must equal local {local_len} * tp_ratio {tp_ratio} " + f"// block_size_ratio {block_size_ratio}." + ) + else: + assert block_size_ratio == 1, ( + "Different local/remote block sizes are not supported " + "when P TP > D TP." + ) + assert remote_len == local_len // (-tp_ratio), ( + f"SPLIT region {i}: remote P KV block_len " + f"{remote_len} must equal local {local_len} " + f"// |tp_ratio| {-tp_ratio}." + ) + + # TP workers that handhshake with same remote have same #blocks. + assert self.dst_num_blocks[remote_engine_id] == nixl_agent_meta.num_blocks + # Same number of regions/~layers. + assert len(nixl_agent_meta.kv_caches_base_addr) == len(self.block_len_per_layer) + + def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta): + """copy recved kv from host buffer to device.""" + assert self.use_host_buffer + assert self.copy_blocks is not None + + local_block_ids = meta.local_physical_block_ids + # TODO (NickLucche) D2H<>H2D ops could benefit from coalescing io across groups + for group_block_ids in local_block_ids: + self.copy_blocks( + self.host_xfer_buffers, + self.device_kv_caches, + group_block_ids, + group_block_ids, + "h2d", + ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "synced recved kv of request[%s] to device kv buffer," + "local_block_ids: %s. ", + req_id, + ",".join(map(str, local_block_ids)), + ) + + def save_kv_to_host(self, metadata: NixlConnectorMetadata): + """copy kv from device to host buffer.""" + assert self.use_host_buffer + assert self.copy_blocks is not None + + for req_id, meta in metadata.reqs_to_save.items(): + meta.local_physical_block_ids = self._logical_to_kernel_block_ids( + meta.local_block_ids + ) + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "save_load_kv for request[%s] to host xfer buffer." + "local_block_ids: %s. ", + req_id, + ",".join(map(str, meta.local_physical_block_ids)), + ) + # blocking + for group_block_ids in meta.local_physical_block_ids: + self.copy_blocks( + self.device_kv_caches, + self.host_xfer_buffers, + group_block_ids, + group_block_ids, + "d2h", + ) + + def post_process_device_kv_on_receive( + self, + block_size_ratio: int, + block_ids_list: list[list[int]], + ): + """ + Post process device kv cache after receiving from remote. + + 3 types of post processing supported: + * kv_cache_postprocess_layout => convert from HND to NHD + * kv_cache_postprocess_blksize => convert from small block size + to large block size + * kv_cache_postprocess_blksize_and_layout => convert from small + block size to large block size and convert from HND to NHD + + """ + if len(self.device_kv_caches) == 0: + return + assert block_size_ratio >= 1, "Only nP < nD supported currently." + assert self.transfer_topo is not None + if self.enable_permute_local_kv and block_size_ratio > 1: + logger.debug( + "Post-processing device kv cache on receive by converting " + "block_size with %sx bigger and permuting layout from HND" + " to NHD.", + block_size_ratio, + ) + elif self.enable_permute_local_kv: + logger.debug( + "Post-processing device kv cache on receive by permuting layout" + "from HND to NHD." + ) + else: + logger.debug( + "Post-processing device kv cache on receive by converting " + "block_size with %sx bigger.", + block_size_ratio, + ) + + split_k_and_v = self.transfer_topo.split_k_and_v + + for block_ids in block_ids_list: + indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) + + for _, cache_or_caches in self.device_kv_caches.items(): + cache_list = cache_or_caches if split_k_and_v else [cache_or_caches] + for cache in cache_list: + if self.enable_permute_local_kv and block_size_ratio > 1: + kv_postprocess_blksize_and_layout_on_receive( + cache, indices, block_size_ratio + ) + elif self.enable_permute_local_kv: + kv_postprocess_layout_on_receive(cache, indices) + else: + kv_postprocess_blksize_on_receive( + cache, indices, block_size_ratio + ) + + def post_process_device_kv_on_receive_heterogeneous_attn( + self, block_ids: list[int] + ): + """ + Post process device kv cache after receiving from remote + for heterogeneous attention. + """ + assert self.enable_heterogeneous_attn_post_process + + indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) + + for _, cache_or_caches in self.device_kv_caches.items(): + blocks_to_update = cache_or_caches.index_select(1, indices) + current_platform.pack_kv_cache( + key=blocks_to_update[0], + value=blocks_to_update[1], + key_cache=cache_or_caches[0], + value_cache=cache_or_caches[1], + block_ids=block_ids, + indices=indices, + ) + + def get_finished(self) -> tuple[set[str], set[str]]: + """ + Get requests that are done sending or recving on this specific worker. + The scheduler process (via the MultiprocExecutor) will use this output + to track which workers are done. + """ + assert self.transfer_topo is not None + done_sending = self._get_new_notifs() + done_recving = self._pop_done_transfers(self._recving_transfers) + + # Drain queue of requests where handshake or transfer setup failed. + failed_recv_reqs = set[ReqId]() + while not self._failed_recv_reqs.empty(): + try: + failed_recv_reqs.add(self._failed_recv_reqs.get_nowait()) + except queue.Empty: + break + + # Add failed requests to done_recving for scheduler tracking + # (blocks are already marked invalid, scheduler will handle recompute) + done_recving.update(failed_recv_reqs) + + if len(done_sending) > 0 or len(done_recving) > 0: + logger.debug( + "Rank %s, get_finished: %s requests done sending " + "and %s requests done recving (%s failed)", + self.tp_rank, + len(done_sending), + len(done_recving), + len(failed_recv_reqs), + ) + + block_ids_for_blocksize_post_process = defaultdict(list) + block_ids_for_heterogeneous_attn_post_process = list[list[int]]() + for req_id in done_recving: + # clean up metadata for completed requests + meta = self._recving_metadata.pop(req_id, None) + assert meta is not None, f"{req_id} not found in recving_metadata list" + + # Skip KV sync and post-processing for failed requests + if req_id in failed_recv_reqs: + logger.warning( + "Skipping KV post-processing for failed request %s", + req_id, + ) + continue + + assert meta.remote is not None + if self.use_host_buffer: + self.sync_recved_kv_to_device(req_id, meta) + + # post processing for heteroblocksize + remote_info = self.transfer_topo.get_engine_info(meta.remote.engine_id) + block_size_ratio = self.transfer_topo.block_size_ratio( + remote_info.remote_block_size + ) + if not self.use_mla and ( + block_size_ratio > 1 or self.enable_permute_local_kv + ): + assert not self._is_hma_required + block_ids_for_blocksize_post_process[block_size_ratio].append( + meta.local_physical_block_ids[0] + ) + # post processing for heterogeneous attention + if self.enable_heterogeneous_attn_post_process: + block_ids_for_heterogeneous_attn_post_process.append( + meta.local_physical_block_ids[0] + ) + for ( + block_size_ratio, + block_ids_list, + ) in block_ids_for_blocksize_post_process.items(): + self.post_process_device_kv_on_receive(block_size_ratio, block_ids_list) + + for block_ids in block_ids_for_heterogeneous_attn_post_process: + self.post_process_device_kv_on_receive_heterogeneous_attn(block_ids) + + # Handle timeout to avoid stranding blocks on remote. + now = time.perf_counter() + while self._reqs_to_send: + req_id, expires = next(iter(self._reqs_to_send.items())) + # Sorted dict, oldest requests are put first so we can exit early. + if now < expires: + break + count = self.consumer_notification_counts_by_req.pop(req_id, 0) + self.xfer_stats.record_kv_expired_req() + logger.warning( + "Releasing expired KV blocks for request %s which were " + "retrieved by %d remote worker(s) before lease expired.", + req_id, + count, + ) + self._reqs_to_process.remove(req_id) + del self._reqs_to_send[req_id] + done_sending.add(req_id) + + return done_sending, done_recving + + def _get_new_notifs(self) -> set[str]: + """Get req_ids which got a remote xfer notification. + + Subclasses must implement this to handle mode-specific notifications. + """ + raise NotImplementedError + + def _handle_heartbeat(self, payload: str) -> None: + """Extend leases for requests referenced in a heartbeat. + + Args: + payload: comma-separated P-side request IDs, e.g. + "req_abc,req_def". + """ + new_expiry = time.perf_counter() + self._lease_extension + for req_id in payload.split(","): + if req_id in self._reqs_to_send: + old = self._reqs_to_send[req_id] + self._reqs_to_send[req_id] = max(old, new_expiry) + logger.debug( + "Heartbeat extended lease for request %s " + "by %ds (old_expiry=%.1f, new_expiry=%.1f)", + req_id, + self._lease_extension, + old, + new_expiry, + ) + + def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]: + """ + Pop completed xfers by checking for DONE state. + Args: + transfers: dict of req_id -> list[running_xfer] + Returns: + set of req_ids that have all done xfers + """ + done_req_ids: set[str] = set() + for req_id, handles in list(transfers.items()): + in_progress = [] + for handle in handles: + try: + xfer_state = self.nixl_wrapper.check_xfer_state(handle) + if xfer_state == "DONE": + # Get telemetry from NIXL + res = self.nixl_wrapper.get_xfer_telemetry(handle) + self.xfer_stats.record_transfer(res) + self.nixl_wrapper.release_xfer_handle(handle) + elif xfer_state == "PROC": + in_progress.append(handle) + continue + else: + self._log_failure( + failure_type="transfer_failed", + msg="Marking blocks as invalid", + req_id=req_id, + xfer_state=xfer_state, + ) + self._handle_failed_transfer(req_id, handle) + except Exception as e: + self._log_failure( + failure_type="transfer_exception", + msg="Marking blocks as invalid", + req_id=req_id, + error=e, + ) + self._handle_failed_transfer(req_id, handle) + + if not in_progress: + # Only report request as completed when all transfers are done. + done_req_ids.add(req_id) + del transfers[req_id] + else: + transfers[req_id] = in_progress + return done_req_ids + + def _handle_failed_transfer(self, req_id: str, handle: int | None): + """ + Handle a failed transfer by marking all (logical) blocks as invalid and + recording the failure. + + Args: + req_id: The request ID. + handle: The transfer handle. + """ + # Use .get() here as the metadata cleanup is handled by get_finished() + # TODO (NickLucche) handle failed transfer for HMA. + if (meta := self._recving_metadata.get(req_id)) and not self._is_hma_required: + self._invalid_block_ids.put(set(meta.local_block_ids[0])) + self._failed_recv_reqs.put(req_id) + if handle is not None: + self.nixl_wrapper.release_xfer_handle(handle) + self.xfer_stats.record_failed_transfer() + + def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None: + """ + Send heartbeat notifications to remote engines, extending lease on KV blocks. + """ + for engine_id, hb_info in metadata.heartbeat_by_engine.items(): + # Proactive handshake (this request may still be in waiting queue) so + # the **next** heartbeat for this remote can go through. + if ( + self._ensure_handshake( + engine_id, hb_info.host, hb_info.port, hb_info.tp_size + ) + is not None + ): + continue # handshake is still pending + + # Build the heartbeat message: "HB:req1,req2,..." + hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode() + for agent_name in self._remote_agents[engine_id].values(): + try: + self.nixl_wrapper.send_notif(agent_name, notif_msg=hb_msg) + except Exception: + logger.debug( + "Failed to send heartbeat to engine %s", + engine_id, + exc_info=True, + ) + + def get_mapped_blocks( + self, block_ids: np.ndarray, block_size_ratio: int + ) -> np.ndarray: + """ + Calculates the new set of block IDs by mapping every element + in the (potentially sparse) input array. + Example: block_ids=[0, 2], block_size_ratio=2 + get_mapped_blocks 0 1 [2 3] 4 5 + # remote is |h0-b0|h1-b0||h0-b1|h1-b1||h0-b1|h1-b1|| + # local is |h0-b0......||h1-b0......||h2-b0........ + local_block_ids 0 [1] 2 + """ + if block_ids.size == 0: + return np.array([], dtype=np.int64) + + start_ids = block_ids * block_size_ratio + offsets = np.arange(block_size_ratio) + mapped_2d = start_ids[:, None] + offsets[None, :] + + return mapped_2d.flatten().astype(np.int64) + + def _logical_to_kernel_block_ids(self, block_ids: BlockIds) -> BlockIds: + """ + Convert logical block ids to kernel physical block ids. + This is required when the logical block size (the one set by the user) + does not match the one required by the attn backend. + """ + if self._physical_blocks_per_logical_kv_block == 1: + # Noop when physical and logical block sizes are the same + return block_ids + block_arange = np.arange(0, self._physical_blocks_per_logical_kv_block).reshape( + 1, -1 + ) + # Mamba blocks have no logical<>physical discrepancy + group_specs = self.kv_cache_config.kv_cache_groups + return [ + BlockTable.map_to_kernel_blocks( + np.array(group), + self._physical_blocks_per_logical_kv_block, + block_arange, + ).tolist() + if not isinstance(group_specs[i].kv_cache_spec, MambaSpec) + else group + for i, group in enumerate(block_ids) + ] + + def _apply_prefix_caching( + self, + local_block_ids: BlockIds, + remote_block_ids: BlockIds, + remote_physical_per_logical: int, + ) -> tuple[BlockIds, list]: + """Apply prefix caching by trimming local/remote block ID lists. + + For non-Mamba models: end-trim remote to match local count, so that + already-cached prefix blocks are skipped in the transfer. + + For Mamba hybrid (prefix caching not yet supported): front-trim both + to the minimum count to handle kernel block count discrepancies from + logical block rounding in heterogeneous TP. + """ + # Partial prefix cache hit: just read uncomputed blocks. + # Skip mamba groups — their blocks represent full state (conv+ssm), + # not per-token data, so trimming would corrupt the transfer. + remote_block_ids = list(remote_block_ids) + if not self._has_mamba: + for i, remote_group in enumerate(remote_block_ids): + num_local_blocks = len(local_block_ids[i]) + assert num_local_blocks <= len(remote_group) + if num_local_blocks < len(remote_group): + remote_block_ids[i] = remote_group[-num_local_blocks:] + else: + # (NOTE: ZhanqiuHu) Mamba hybrid: no prefix caching support so far.HeteroTP + # can cause different kernel block counts due to logical block rounding. + # Example: 640 prompt tokens, kernel_block_size=64 + # remote physical_per_logical=10, local physical_per_logical=6 + # remote logical ids from kv_transfer_params = [0] + # local logical ids allocated = [0, 1] + # remote kernel blocks: [0..9] (1*10=10) + # local kernel blocks: [0..11] (2*6=12) + # actual data blocks = ceil(640/64) = 10, trim both to 10 + # Vice versa (remote physical_per_logical=6, local=10): + # remote logical ids = [0, 1], local logical ids = [0] + # remote kernel blocks: [0..11] (2*6=12) + # local kernel blocks: [0..9] (1*10=10) + # actual data blocks = ceil(640/64) = 10, trim both to 10 + local_block_ids = list(local_block_ids) + for i, remote_group in enumerate(remote_block_ids): + num_local_blocks = len(local_block_ids[i]) + num_remote_blocks = len(remote_group) + if ( + _is_ssm_spec(self._group_spec_types[i]) + and num_local_blocks < num_remote_blocks + ): + # NOTE (NickLucche): With prefix caching on SSM, (remote) blocks + # prior to the last one are placeholders (null blocks). Mind that + # this doesn't really impact transfer, as we only still care about + # the last "block", the full in-place state. + assert num_local_blocks == 1, "SSM can only have one local block" + remote_block_ids[i] = remote_group[-num_local_blocks:] + elif ( + self._physical_blocks_per_logical_kv_block + == remote_physical_per_logical + and num_local_blocks < num_remote_blocks + ): + # Partial prefix cache hit for FA group. + remote_block_ids[i] = remote_group[-num_local_blocks:] + else: + # TODO Handle prefix caching with different block_sizes + max_padding = max( + self._physical_blocks_per_logical_kv_block, + remote_physical_per_logical, + ) + assert abs(num_local_blocks - num_remote_blocks) < max_padding, ( + f"Group {i}: |{num_local_blocks} - " + f"{num_remote_blocks}| >= {max_padding}" + ) + num_blocks = min(num_local_blocks, num_remote_blocks) + local_block_ids[i] = local_block_ids[i][:num_blocks] + remote_block_ids[i] = remote_group[:num_blocks] + return local_block_ids, remote_block_ids + + def _logical_to_remote_kernel_block_ids( + self, block_ids: BlockIds, remote_physical_per_logical: int + ) -> BlockIds: + """Map logical block IDs to physical kernel block IDs on the remote. + + Args: + block_ids: per-group lists of logical block IDs. + remote_physical_per_logical: remote engine's physical blocks + per logical block. + + Returns: + Same structure with FA groups expanded (each logical block L + becomes kernel blocks [L*remote_physical_per_logical, .. + L*remote_physical_per_logical + + remote_physical_per_logical - 1]). + Mamba groups are passed through unchanged. + """ + if remote_physical_per_logical == 1: + return block_ids + remote_arange = np.arange(remote_physical_per_logical).reshape(1, -1) + group_specs = self.kv_cache_config.kv_cache_groups + result = [ + BlockTable.map_to_kernel_blocks( + np.array(group), + remote_physical_per_logical, + remote_arange, + ).tolist() + if not isinstance(group_specs[i].kv_cache_spec, MambaSpec) + else group + for i, group in enumerate(block_ids) + ] + return result + + def get_backend_aware_kv_block_len( + self, layer_idx: int, first_split: bool = True, mamba_view: bool = False + ) -> int: + """ + Get the block length for one K/V element (K and V have the same size). + + For FA and other backends, this is equal to the length of the whole + block, as K and V are in separate regions. + For FlashInfer, this is half the length of the whole block, as K and V + share the same region. + Similarly, for SSM-based models, state and conv are interleaved, but crucially + the their size differs. + Reference diagram: + KVCacheTensor (Shared) + / \\ + / \\ + / \\ + Attention (FlashInfer) View Mamba View + | | + | | + +-------------------+ +-------------------+ + | KVCacheTensor | | KVCacheTensor | + | | | | + |<----- page ------>| |<----- page ------->| + | size | | size | + | Key 0 | Val 0 | |Conv 0 | SSM 0 | + | Key 1 | Val 1 | |Conv 1 | SSM 1 | + | ... | ... | | ... | ... | + | Key N-2 | Val N-2 | |Conv N-2| SSM N-2 | + | Key N-1 | Val N-1 | |Conv N-1| SSM N-1 | + +-------------------+ +--------------------+ + |1st_split-2nd_split| |1st_split-2nd_split | + """ + assert self.transfer_topo is not None + virtually_split = self.transfer_topo.virtually_split_kv_in_blocks + if virtually_split and mamba_view: + block_len = self._mamba_ssm_size[not first_split] + else: + half_block = virtually_split and not self._is_region_replicated(layer_idx) + block_len = self.block_len_per_layer[layer_idx] // (2 if half_block else 1) + return block_len + + def get_kv_connector_stats(self) -> KVConnectorStats | None: + """ + Get the KV transfer stats for the connector. + """ + # Clear stats for next iteration + if not self.xfer_stats.is_empty(): + return self.xfer_stats.clone_and_reset() + return None + + def get_block_ids_with_load_errors(self) -> set[int]: + """ + Return and clear the set of block IDs that failed to load. + + This is called by the scheduler to identify blocks that need + to be retried after a NIXL transfer failure. + """ + # Drain the queue (thread-safe, no lock needed). + result: set[int] = set() + while not self._invalid_block_ids.empty(): + try: + result.update(self._invalid_block_ids.get_nowait()) + except queue.Empty: + break + return result + + def _evict_stale_engines(self) -> None: + """Scan for and evict remote engines that have exceeded their TTL. + + Called from the main thread in when a new remote engine appears. + We can only go OOM as we discover and register a new remote, therefore we make + sure we clean up stale engine data structures before then. This invariant + prevents us from using background threads, though memory usage is not guaranteed + to be "optimal" until a new handshake is performed. + + Engines with active transfers or pending handshakes cannot be stale: + - Active transfers touch _engine_last_active in start_load_kv. + - Pending handshakes don't have an _engine_last_active entry yet + """ + # NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number + # of remote engines is registered all at once (adding a background cleanup + # thread wouldnt help either). + # If that scenario is plausible, we can follow up with an LRU eviction policy. + if self._engine_ttl <= 0: + return + + now = time.perf_counter() + for eid, last_active in list(self._engine_last_active.items()): + if now - last_active > self._engine_ttl: + self._cleanup_remote_engine(eid) + + def _cleanup_remote_engine( + self, engine_id: EngineId, *, log_eviction: bool = True + ) -> None: + """Remove all state for a single remote engine. + + Releases NIXL resources (dlist handles, remote agents) and clears + all per-engine data structures. Used by both TTL eviction and + shutdown. + """ + assert engine_id in self._remote_agents + + for handle in self.dst_xfer_side_handles.pop(engine_id).values(): + self.nixl_wrapper.release_dlist_handle(handle) + for agent_name in self._remote_agents.pop(engine_id).values(): + self.nixl_wrapper.remove_remote_agent(agent_name) + + del self.kv_caches_base_addr[engine_id] + del self.dst_num_blocks[engine_id] + del self.tp_mappings[engine_id] + if self.transfer_topo is not None: + self.transfer_topo.unregister_remote_engine(engine_id) + + last_active = self._engine_last_active.pop(engine_id) + if log_eviction: + logger.info( + "Evicted stale remote engine %s (inactive for %.1fs).", + engine_id, + time.perf_counter() - last_active, + ) + + def __del__(self): + self.shutdown() + + def shutdown(self): + """Shutdown the connector worker.""" + if not hasattr(self, "_handshake_initiation_executor"): + # error happens during init, no need to shutdown + return + self._handshake_initiation_executor.shutdown(wait=False) + for handles in self._recving_transfers.values(): + for handle in handles: + self.nixl_wrapper.release_xfer_handle(handle) + self._recving_transfers.clear() + for handle in self.src_xfer_handles_by_block_size.values(): + self.nixl_wrapper.release_dlist_handle(handle) + self.src_xfer_handles_by_block_size.clear() + for handles in self.src_xfer_handles_by_tp_ratio.values(): + for handle in handles: + self.nixl_wrapper.release_dlist_handle(handle) + self.src_xfer_handles_by_tp_ratio.clear() + for engine_id in list(self._remote_agents): + self._cleanup_remote_engine(engine_id, log_eviction=False) + for desc in self._registered_descs: + self.nixl_wrapper.deregister_memory(desc) + self._registered_descs.clear() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py index dad81e84c4..b321450530 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/connector.py @@ -1,6 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""NixlConnector – thin facade that delegates to scheduler / worker.""" +"""NIXL connector facades. + +This module hosts the thin facade classes that vLLM's KV-connector layer +instantiates. Almost all the real work lives in the per-mode scheduler +and worker classes; the connector classes here only forward calls. + +* :class:`NixlBaseConnector` – common logic shared by pull and push. +* :class:`NixlPullConnector` – pull-based (READ) KV transfer. +* :class:`NixlPushConnector` – push-based (WRITE) KV transfer. +* ``NixlConnector`` – backward-compatible alias for :class:`NixlPullConnector`. +""" from typing import TYPE_CHECKING, Any @@ -28,16 +38,22 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( NixlConnectorMetadata, ) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.scheduler import ( - NixlConnectorScheduler, +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import ( + NixlPullConnectorWorker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_scheduler import ( + NixlPushConnectorScheduler, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.push_worker import ( + NixlPushConnectorWorker, ) from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( NixlKVConnectorStats, NixlPromMetrics, ) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import ( - NixlConnectorWorker, -) from vllm.forward_context import ForwardContext from vllm.logger import init_logger from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata @@ -47,6 +63,12 @@ from vllm.v1.kv_cache_interface import MambaSpec from vllm.v1.outputs import KVConnectorOutput if TYPE_CHECKING: + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( + NixlBaseConnectorScheduler, + ) + from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( + NixlBaseConnectorWorker, + ) from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.request import Request @@ -54,7 +76,9 @@ if TYPE_CHECKING: logger = init_logger(__name__) -class NixlConnector(KVConnectorBase_V1, SupportsHMA): +class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA): + """Base connector with common logic shared by pull and push modes.""" + @property def prefer_cross_layer_blocks(self) -> bool: if any( @@ -106,16 +130,9 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA): self.kv_cache_config = kv_cache_config self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id self.kv_transfer_config = vllm_config.kv_transfer_config - if role == KVConnectorRole.SCHEDULER: - self.connector_scheduler: NixlConnectorScheduler | None = ( - NixlConnectorScheduler(vllm_config, self.engine_id, kv_cache_config) - ) - self.connector_worker: NixlConnectorWorker | None = None - elif role == KVConnectorRole.WORKER: - self.connector_scheduler = None - self.connector_worker = NixlConnectorWorker( - vllm_config, self.engine_id, kv_cache_config - ) + # Subclasses must set self.connector_scheduler and self.connector_worker + self.connector_scheduler: NixlBaseConnectorScheduler | None = None + self.connector_worker: NixlBaseConnectorWorker | None = None ############################################################ # Class Methods @@ -256,11 +273,6 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA): vllm_config, metric_types, labelnames, per_engine_labelvalues ) - def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: - assert self.connector_worker is not None - assert isinstance(self._connector_metadata, NixlConnectorMetadata) - self.connector_worker.start_load_kv(self._connector_metadata) - def wait_for_layer_load(self, layer_name: str) -> None: """NixlConnector does not do layerwise saving.""" pass @@ -281,6 +293,11 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA): if self.connector_worker.use_host_buffer and self.connector_worker.copy_blocks: self.connector_worker.save_kv_to_host(self._connector_metadata) + def has_pending_push_work(self) -> bool: + if self.connector_scheduler is not None: + return self.connector_scheduler.has_pending_push_work() + return False + def shutdown(self): if self.connector_worker is not None: self.connector_worker.shutdown() @@ -299,3 +316,79 @@ class NixlConnector(KVConnectorBase_V1, SupportsHMA): """ assert self.connector_worker is not None return self.connector_worker.xfer_handshake_metadata + + +class NixlPullConnector(NixlBaseConnector): + """Pull-based (READ) NIXL KV transfer connector.""" + + def __init__( + self, + vllm_config: VllmConfig, + role: KVConnectorRole, + kv_cache_config: "KVCacheConfig", + ): + super().__init__(vllm_config, role, kv_cache_config) + if role == KVConnectorRole.SCHEDULER: + self.connector_scheduler = NixlPullConnectorScheduler( + vllm_config, self.engine_id, kv_cache_config + ) + self.connector_worker = None + elif role == KVConnectorRole.WORKER: + self.connector_scheduler = None + self.connector_worker = NixlPullConnectorWorker( + vllm_config, self.engine_id, kv_cache_config + ) + + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: + assert self.connector_worker is not None + assert isinstance(self.connector_worker, NixlPullConnectorWorker) + assert isinstance(self._connector_metadata, NixlConnectorMetadata) + self.connector_worker.start_load_kv(self._connector_metadata) + + +class NixlPushConnector(NixlBaseConnector): + """Push-based (WRITE) NIXL KV transfer connector.""" + + def __init__( + self, + vllm_config: VllmConfig, + role: KVConnectorRole, + kv_cache_config: "KVCacheConfig", + ): + super().__init__(vllm_config, role, kv_cache_config) + self.connector_scheduler: NixlPushConnectorScheduler | None = None + self.connector_worker: NixlPushConnectorWorker | None = None + if role == KVConnectorRole.SCHEDULER: + self.connector_scheduler = NixlPushConnectorScheduler( + vllm_config, self.engine_id, kv_cache_config + ) + elif role == KVConnectorRole.WORKER: + self.connector_worker = NixlPushConnectorWorker( + vllm_config, self.engine_id, kv_cache_config + ) + else: + raise ValueError(f"Unsupported KVConnectorRole: {role}") + + def start_load_kv(self, forward_context: "ForwardContext", **kwargs) -> None: + """Drive push processing on the worker. + + The worker enqueues registrations / finished blocks for the + background ``nixl-push-writer`` thread; the writer issues the + WRITE transfers and polls NIXL notifs without further + engine-thread involvement. + """ + assert self.connector_worker is not None + assert isinstance(self._connector_metadata, NixlConnectorMetadata) + self.connector_worker.start_load_kv(self._connector_metadata) + + +# Backward compatibility: NixlConnector is the pull-based connector. +NixlConnector = NixlPullConnector + + +__all__ = [ + "NixlBaseConnector", + "NixlConnector", + "NixlPullConnector", + "NixlPushConnector", +] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py index b9e3436f50..c120f939af 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/metadata.py @@ -19,6 +19,11 @@ TransferHandle = int ReqId = str GET_META_MSG = b"get_meta_msg" + +# Push-mode (WRITE-based) registration notification. +# Sent worker-to-worker over NIXL: D worker -> P worker, encoded as +# PUSH_REG_NOTIF_PREFIX + msgpack(registration_data). +PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:" # # NIXL Connector Version # @@ -160,6 +165,8 @@ class ReqMeta: local_physical_block_ids: BlockIds tp_size: int remote: RemoteMeta | None = None + # Remote block size, discovered during NIXL handshake (push mode). + remote_block_size: int | None = None class NixlConnectorMetadata(KVConnectorMetadata): @@ -171,6 +178,12 @@ class NixlConnectorMetadata(KVConnectorMetadata): self.reqs_not_processed: set[ReqId] = set() # Heartbeat data grouped by remote engine, sent by D worker to P. self.heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {} + # Push mode (D side): registration data the D worker should send to + # P workers via NIXL notification on this step. + self.push_registrations: dict[ReqId, dict[str, Any]] = {} + # Push mode (P side): newly finished request blocks to be matched + # against pending D registrations on the P worker. + self.push_finished_blocks: dict[ReqId, BlockIds] = {} def _add_new_req( self, @@ -182,6 +195,7 @@ class NixlConnectorMetadata(KVConnectorMetadata): local_physical_block_ids=local_block_ids, # P workers don't need to receive tp_size from proxy here. tp_size=kv_transfer_params.get("tp_size", 1), + remote_block_size=kv_transfer_params.get("remote_block_size"), ) def add_new_req_to_save( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py new file mode 100644 index 0000000000..f13e216056 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_scheduler.py @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pull-specific scheduler-side logic for the NIXL connector.""" + +import time +from typing import TYPE_CHECKING, Any + +from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( + NixlBaseConnectorScheduler, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + from vllm.v1.kv_cache_interface import KVCacheConfig + from vllm.v1.request import Request + +logger = init_logger(__name__) + + +class NixlPullConnectorScheduler(NixlBaseConnectorScheduler): + """Pull-specific scheduler logic (READ-based KV transfer).""" + + def __init__( + self, + vllm_config: "VllmConfig", + engine_id: str, + kv_cache_config: "KVCacheConfig", + ): + super().__init__(vllm_config, engine_id, kv_cache_config) + + def get_num_new_matched_tokens( + self, request: "Request", num_computed_tokens: int + ) -> tuple[int, bool]: + """ + For remote prefill, pull all prompt blocks from remote + asynchronously relative to engine execution. + + Args: + request (Request): the request object. + num_computed_tokens (int): the number of locally + computed tokens for this request + Returns: + * the number of tokens that can be loaded from the + external KV cache beyond what is already computed. + * true if the external KV cache tokens will be loaded + asynchronously (between scheduler steps). + """ + + params = request.kv_transfer_params + logger.debug( + "NIXLConnector get_num_new_matched_tokens: " + "num_computed_tokens=%s, kv_transfer_params=%s", + num_computed_tokens, + params, + ) + + if params is not None and params.get("do_remote_prefill"): + # Remote prefill: get all prompt blocks from remote. + token_ids = request.prompt_token_ids or [] + actual = self._mamba_prefill_token_count(len(token_ids)) + count = actual - num_computed_tokens + if count > 0: + return count, True + + if params is not None and params.get("do_remote_decode") and self._has_mamba: + self._truncate_mamba_request_for_prefill(request) + + if ( + params is not None + and params.get("do_remote_decode") + and params.get("remote_block_ids") + and all( + p in params + for p in ( + "remote_engine_id", + "remote_request_id", + "remote_host", + "remote_port", + ) + ) + ): + # Decode node has kv blocks for part of prefill request, so, provide them + # as an external token count to scheduler. + # The tokens will be loaded if not already present + # in the prefill node local cache + remote_num_tokens = params.get("remote_num_tokens") or 0 + count = ( + min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens + ) + if count > 0: + # Check kv_recompute_threshold: skip pull if + # remote tokens are below the threshold. + if ( + self.kv_recompute_threshold > 0 + and count < self.kv_recompute_threshold + ): + logger.debug( + "Skipping remote pull for %s: %d remote tokens < threshold %d", + request.request_id, + count, + self.kv_recompute_threshold, + ) + return 0, False + return count, True + + # No remote prefill for this request. + return 0, False + + def update_state_after_alloc( + self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int + ): + params = request.kv_transfer_params + logger.debug( + "NIXLConnector update_state_after_alloc: " + "num_external_tokens=%s, kv_transfer_params=%s", + num_external_tokens, + params, + ) + + if not params: + return + + if params.get("do_remote_decode") or ( + params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled + ): + self._reqs_in_batch.add(request.request_id) + if self.use_host_buffer and params.get("do_remote_decode"): + # NOTE: when accelerator is not directly supported by Nixl, + # prefilled blocks need to be saved to host memory before transfer. + self._reqs_need_save[request.request_id] = request + elif params.get("do_remote_prefill") or ( + params.get("do_remote_decode") + and self.is_bidirectional_kv_xfer_enabled + and not params.get("_remote_blocks_processed") + ): + if params.get("remote_block_ids"): + if all( + p in params + for p in ( + "remote_engine_id", + "remote_request_id", + "remote_host", + "remote_port", + ) + ): + # If remote_blocks and num_external_tokens = 0, we have + # a full prefix cache hit on the local node. We need to call + # send_notif in _read_blocks to free the memory on the remote node. + + unhashed_local_block_ids: BlockIds = ( + blocks.get_unhashed_block_ids_all_groups() + if num_external_tokens > 0 + else () + ) + local_block_ids = self.get_sw_clipped_blocks( + unhashed_local_block_ids + ) + + # Get unhashed blocks to pull from remote. Mind that a full prefix + # cache hit is indicated with an empty list. + self._reqs_need_recv[request.request_id] = ( + request, + local_block_ids, + ) + + else: + logger.warning( + "Got invalid KVTransferParams: %s. This " + "request will not utilize KVTransfer", + params, + ) + else: + assert num_external_tokens == 0 + # Only trigger 1 KV transfer per request. + params["do_remote_prefill"] = False + params["_remote_blocks_processed"] = True + + def request_finished( + self, + request: "Request", + block_ids: "BlockIds", + ) -> tuple[bool, dict[str, Any] | None]: + """ + Once a request is finished, determine whether request blocks + should be freed now or will be sent asynchronously and freed later. + """ + from vllm.v1.request import RequestStatus + + params = request.kv_transfer_params + logger.debug( + "NIXLConnector request_finished(%s), request_status=%s, " + "kv_transfer_params=%s", + request.request_id, + request.status, + params, + ) + if not params: + return False, None + + is_p_node = bool(params.get("do_remote_decode")) + is_d_node = not is_p_node + + # Stop heartbeating for aborted requests that never reached finished_recving: + # normal path cleans up in update_connector_output. + self._stop_heartbeat(request.request_id) + + if params.get("do_remote_prefill"): + # If do_remote_prefill is still True when the request is finished, + # update_state_after_alloc must not have been called (the request + # must have been aborted before it was scheduled, e.g. via the + # abort_immediately path used to clean up KV-transfer requests + # rejected at the D-side serving layer). + # To avoid stranding the prefill blocks in the prefill instance, + # we must add empty block_ids to _reqs_need_recv so that our + # worker side will notify and free blocks in the prefill instance. + self._reqs_need_recv[request.request_id] = (request, []) + params["do_remote_prefill"] = False + return False, None + + if is_d_node and not self.is_bidirectional_kv_xfer_enabled: + return False, None + + if request.status not in ( + RequestStatus.FINISHED_LENGTH_CAPPED, + RequestStatus.FINISHED_STOPPED, + ): + # Also include the case of a P/D Prefill request with immediate + # block free (eg abort). Stop tracking this request. + self._reqs_not_processed.add(request.request_id) + # Clear _reqs_need_save if a request is aborted as partial prefill. + self._reqs_need_save.pop(request.request_id, None) + return False, None + + # TODO: check whether block_ids actually ever be 0. If not we could + # remove the conditional below + delay_free_blocks = any(len(group) > 0 for group in block_ids) + remote_num_tokens = 0 + if delay_free_blocks: + # Prefill request on remote. It will be read from D upon completion + request_kv_blocks_ttl = self._kv_lease_duration + if is_d_node: + # For blocks pinned on D, use a simpler timeout for now instead of a + # lease mechanism as turn2 request is client-driven. + request_kv_blocks_ttl = self.decoder_kv_blocks_ttl + logger.debug( + "NIXLConnector request_finished(%s) waiting for %d seconds " + "before releasing blocks", + request.request_id, + request_kv_blocks_ttl, + ) + self._reqs_need_send[request.request_id] = ( + time.perf_counter() + request_kv_blocks_ttl + ) + # NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones), + # trimming down after allocating for the whole sequence length. Empty + # blocks are always at the start of the list. + # Here we "unpad" blocks to send the actual remote blocks to be read. + block_ids = self.get_sw_clipped_blocks(block_ids) + + remote_num_tokens = request.num_computed_tokens + + return delay_free_blocks, dict( + do_remote_prefill=is_p_node, + do_remote_decode=is_d_node, + remote_block_ids=block_ids, + remote_engine_id=self.engine_id, + remote_request_id=request.request_id, + remote_host=self.side_channel_host, + remote_port=self.side_channel_port, + tp_size=self.vllm_config.parallel_config.tensor_parallel_size, + remote_num_tokens=remote_num_tokens, + ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py new file mode 100644 index 0000000000..26f5fde24d --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/pull_worker.py @@ -0,0 +1,382 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Pull-specific (READ) worker-side logic for the NIXL connector.""" + +import time +from typing import TYPE_CHECKING + +import numpy as np + +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( + NixlBaseConnectorWorker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + NixlConnectorMetadata, + ReqMeta, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( + ReadSpec, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.kv_cache_interface import KVCacheConfig + +logger = init_logger(__name__) + + +class NixlPullConnectorWorker(NixlBaseConnectorWorker): + """Pull-specific (READ) worker logic.""" + + def __init__( + self, + vllm_config: "VllmConfig", + engine_id: str, + kv_cache_config: "KVCacheConfig", + ): + super().__init__(vllm_config, engine_id, kv_cache_config) + + def start_load_kv(self, metadata: NixlConnectorMetadata): + """ + Start loading by triggering non-blocking nixl_xfer. + We check for these trnxs to complete in each step(). + """ + for req_id, meta in metadata.reqs_to_recv.items(): + meta.local_physical_block_ids = self._logical_to_kernel_block_ids( + meta.local_block_ids + ) + assert meta.remote is not None + # Remote block IDs are kept logical here; expanded in + # _read_blocks_for_req using the remote engine's phys ratio. + remote_engine_id = meta.remote.engine_id + logger.debug( + "start_load_kv for request %s from remote engine %s. " + "Num local_block_ids: %s. Num remote_block_ids: %s. ", + req_id, + remote_engine_id, + len(meta.local_physical_block_ids), + len(meta.remote.block_ids), + ) + # always store metadata for failure recovery + self._recving_metadata[req_id] = meta + if remote_engine_id not in self._remote_agents: + # Initiate handshake with remote engine to exchange metadata. + with self._handshake_lock: + if remote_engine_id not in self._remote_agents: + self._background_nixl_handshake(req_id, remote_engine_id, meta) + continue + + # Handshake already completed, start async read xfer. + self._read_blocks_for_req(req_id, meta) + + # Start transfers for requests whose handshakes have now finished. + while not self._ready_requests.empty(): + self._read_blocks_for_req(*self._ready_requests.get_nowait()) + + # Keep around the requests that have been part of a batch. This is + # needed because async scheduling pushes the misalignment between the + # moment in which requests expiration is set (P side) and the moment in + # which blocks are read from D. As P can now more easily lag behind D + # while processing the next batch, we make sure to only set an + # expiration for requests that have not been read from D yet. + for req_id in metadata.reqs_in_batch: + self._reqs_to_process.add(req_id) + + # Remove all requests that are not to be processed (eg aborted). + for req_id in metadata.reqs_not_processed: + self._reqs_to_process.discard(req_id) + # We should never get an abort after setting an expiry timer + assert req_id not in self._reqs_to_send + + # Add to requests that are waiting to be read and track expiration. + for req_id, expiration_time in metadata.reqs_to_send.items(): + if req_id in self._reqs_to_process: + self._reqs_to_send[req_id] = expiration_time + + # Send heartbeats to P-side engines to keep KV blocks alive while + # requests sit in the D scheduler WAITING queue. + self._send_heartbeats(metadata) + + def _read_blocks_for_req(self, req_id: str, meta: ReqMeta): + assert meta.remote is not None and self.transfer_topo is not None + engine_id = meta.remote.engine_id + # Update last activity from this remote. Mind that cleanup is done on main + # thread (this one), so we don't race on this structure. + self._engine_last_active[engine_id] = time.perf_counter() + plan = self.tp_mappings[engine_id] + remote_info = self.transfer_topo.get_engine_info(engine_id) + tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) + + meta.remote.block_ids = self._logical_to_remote_kernel_block_ids( + meta.remote.block_ids, + remote_info.remote_physical_blocks_per_logical, + ) + remote_block_ids = meta.remote.block_ids + local_block_ids = meta.local_physical_block_ids + num_groups = len(local_block_ids) + read_specs = [ + ReadSpec( + remote_rank=rank, + local_block_ids=[ + list(local_block_ids[g]) + if rank in plan.source_ranks_per_group[g] + else [] + for g in range(num_groups) + ], + remote_block_ids=[ + list(remote_block_ids[g]) + if rank in plan.source_ranks_per_group[g] + else [] + for g in range(num_groups) + ], + ) + for rank in plan.all_source_ranks + ] + + # D may have to perform multiple reads from different remote ranks. + # MLA opt: when P TP > D TP, only a single read is executed for + # the first remote rank (cache is duplicated).. + if self.use_mla and tp_ratio < 0: + assert len(read_specs) == 1 + + for i, spec in enumerate(read_specs): + remote_block_size = remote_info.remote_block_size + logger.debug( + "Remote agent %s available, calling _read_blocks" + " on remote rank %s with remote block size %s for req %s", + meta.remote.engine_id, + spec.remote_rank, + remote_block_size, + req_id, + ) + # Get side handles. + if tp_ratio < 0 and not self.use_mla: + assert remote_block_size == self.block_size + # Remote tp_size > local tp_size: we must perform multiple + # reads. Get the memory chunk onto which we will write to. + local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[tp_ratio][i] + else: + # Single read from remote, we write to the whole memory region. + # Also handle remote block size different from local block size. + local_xfer_side_handle = self.src_xfer_handles_by_block_size[ + remote_block_size + ] + + # Destination handle: remote_engine_id -> remote_rank -> handle. + remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][ + spec.remote_rank + ] + + self._read_blocks( + read_spec=spec, + request_id=req_id, + dst_engine_id=meta.remote.engine_id, + remote_request_id=meta.remote.request_id, + local_xfer_side_handle=local_xfer_side_handle, + remote_xfer_side_handle=remote_xfer_side_handle, + ) + + if self.use_mla and tp_ratio < 0 and read_specs: + # ..but we still need to notify the other remote ranks that we + # have the blocks we need so they can update the request state. + notif_id = f"{meta.remote.request_id}:{self.world_size}".encode() + remote_agents = self._remote_agents[meta.remote.engine_id] + for rank_to_notify, agent in remote_agents.items(): + if rank_to_notify != read_specs[0].remote_rank: + self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + + def _read_blocks( + self, + read_spec: ReadSpec, + dst_engine_id: str, + request_id: str, + remote_request_id: str, + local_xfer_side_handle: int, + remote_xfer_side_handle: int, + ): + """ + Post a READ point-to-point xfer request from a single local worker to + a single remote worker. + """ + assert self.transfer_topo is not None + remote_rank = read_spec.remote_rank + local_block_ids = read_spec.local_block_ids + remote_block_ids = read_spec.remote_block_ids + + remote_info = self.transfer_topo.get_engine_info(dst_engine_id) + block_size_ratio = self.transfer_topo.block_size_ratio( + remote_info.remote_block_size + ) + if block_size_ratio > 1: + # TODO (NickLucche) assume HMA is off. Change to handle multiple KV groups. + assert not self._is_hma_required + local_block_ids0 = local_block_ids[0] if local_block_ids else [] + remote_block_ids0 = remote_block_ids[0] + local_block_ids_mapped = self.get_mapped_blocks( + np.asarray(local_block_ids0), block_size_ratio + ).tolist() + if len(local_block_ids_mapped) > len(remote_block_ids0): + # NOTE: + # get_mapped_blocks will always expand block_ids for n times. + # ex: + # prefill block_ids with block_size as 4: + # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + # Local decode block_ids with block_size as 16: [1, 2, 3] + # expanded decode block_ids with get_mapped_blocks from [1, 2, 3] to + # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] + # Then we clip local to align with prefill + # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] to + # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + local_block_ids_mapped = local_block_ids_mapped[ + : len(remote_block_ids0) + ] + local_block_ids = [local_block_ids_mapped] if local_block_ids_mapped else [] + remote_block_ids = [remote_block_ids0] + # NOTE(rob): having the staging blocks be on the READER side is + # not going to work well (since we will have to call rearrange tensors). + # after we detect the txn is complete (which means we cannot make the + # read trxn async easily). If we want to make "READ" happen cleanly, + # then we will need to have the staging blocks on the remote side. + + # NOTE(rob): according to nvidia the staging blocks are used to + # saturate IB with heterogeneous TP sizes. + + # Number of D TP workers that will read from dst P. Propagate info + # on notification so that dst worker can wait before freeing blocks. + notif_id = f"{remote_request_id}:{self.world_size}".encode() + + # Full prefix cache hit: do not need to read remote blocks, + # just notify P worker that we have the blocks we need. + if len(local_block_ids) == 0: + # A full prefix cache hit is indicated with an empty list. + agent_name = self._remote_agents[dst_engine_id][remote_rank] + try: + self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id) + except Exception as e: + self._log_failure( + failure_type="notification_failed", + msg="P worker blocks will be freed after timeout. " + "This may indicate network issues.", + req_id=request_id, + error=e, + dst_engine_id=dst_engine_id, + remote_rank=remote_rank, + remote_agent_name=agent_name, + ) + self.xfer_stats.record_failed_notification() + return + + assert ( + len(remote_block_ids) + == len(local_block_ids) + == len(self.kv_cache_config.kv_cache_groups) + ) + remote_physical_per_logical = remote_info.remote_physical_blocks_per_logical + local_block_ids, remote_block_ids = self._apply_prefix_caching( + local_block_ids, remote_block_ids, remote_physical_per_logical + ) + + # NOTE (nicolo) With homogeneous TP, each TP worker loads KV from + # corresponding rank. With heterogeneous TP, fixing D>P, the D tp + # workers will issue xfers to parts of the P worker remote kv caches. + + # Get descs ids. + remote_block_descs_ids = self._compute_desc_ids( + block_ids=remote_block_ids, + dst_num_blocks=self.dst_num_blocks[dst_engine_id], + block_size_ratio=None, + physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical, + ) + local_block_descs_ids = self._compute_desc_ids( + block_ids=local_block_ids, + dst_num_blocks=self.dst_num_blocks[self.engine_id], + block_size_ratio=block_size_ratio, + physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block, + ) + + assert len(local_block_descs_ids) == len(remote_block_descs_ids) + + # Prepare transfer with Nixl. + handle = None + try: + handle = self.nixl_wrapper.make_prepped_xfer( + "READ", + local_xfer_side_handle, + local_block_descs_ids, + remote_xfer_side_handle, + remote_block_descs_ids, + notif_msg=notif_id, + ) + + # Begin async xfer. + self.nixl_wrapper.transfer(handle) + + # Use handle to check completion in future step(). + self._recving_transfers[request_id].append(handle) + except Exception as e: + # mark all (logical) blocks for this request as invalid + self._log_failure( + failure_type="transfer_setup_failed", + req_id=request_id, + msg="Marking blocks as invalid", + error=e, + dst_engine_id=dst_engine_id, + remote_rank=remote_rank, + ) + self._handle_failed_transfer(request_id, handle) + + def _get_new_notifs(self) -> set[str]: + """ + Get req_ids which got a remote xfer message. When multiple consumers + are reading from the same producer (heterogeneous TP scenario), wait + for all consumers to be done pulling. + + Also handles heartbeat notifications ("HB:req1,req2,...") by + extending the lease on the referenced requests. + """ + assert self.transfer_topo is not None + notified_req_ids: set[str] = set() + for notifs in self.nixl_wrapper.get_new_notifs().values(): + for notif in notifs: + msg = notif.decode("utf-8") + + # Handle heartbeat messages from D-side. + if msg.startswith("HB:"): + self._handle_heartbeat(msg[3:]) + continue + + req_id, tp_size = msg.rsplit(":", 1) + if ( + req_id not in self._reqs_to_send + and req_id not in self._reqs_to_process + ): + logger.error( + "Potentially invalid KV blocks for " + "unrecognized request %s were retrieved by " + "a decode worker. They may have expired.", + req_id, + ) + continue + + # NOTE: `tp_ratio` is the opposite when swapping local<>remote + n_consumers = int(tp_size) + tp_ratio = self.transfer_topo.tp_ratio(n_consumers) + + # Number of reads *per producer* to wait for. + # When remote D TP > local P TP we expect `tp_ratio` reads. + consumers_per_producer = ( + -tp_ratio if n_consumers > self.world_size else 1 + ) + + self.consumer_notification_counts_by_req[req_id] += 1 + # Wait all consumers (D) to be done reading before freeing. + if ( + self.consumer_notification_counts_by_req[req_id] + == consumers_per_producer + ): + notified_req_ids.add(req_id) + del self.consumer_notification_counts_by_req[req_id] + self._reqs_to_process.remove(req_id) + self._reqs_to_send.pop(req_id, None) + return notified_req_ids diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py new file mode 100644 index 0000000000..dc976ae3a3 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_scheduler.py @@ -0,0 +1,348 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Push-specific scheduler-side logic for the NIXL connector. + +In push mode, scheduler-side responsibilities are: + +* D side (decode): on ``update_state_after_alloc``, stash registration data + (D's identity + locally allocated block IDs) into + ``_push_pending_registrations``. The D worker drains it from + ``meta.push_registrations`` next step and sends a NIXL notification to the + P worker (no scheduler-level networking). +* P side (prefill): on ``request_finished``, stash the finished block IDs + into ``_finished_request_blocks`` for the lease, and into + ``_newly_finished_push_blocks`` so the P worker picks them up via + ``meta.push_finished_blocks`` and matches against any D registrations + it already received via NIXL notifications. +* Both sides: ``has_pending_push_work`` keeps the engine main loop stepping + while pushes are in flight. ``update_connector_output`` cleans up + ``_finished_request_blocks`` once the WRITE completes. + +A soft per-registration watchdog on the D scheduler fails requests that have +been registered but not fulfilled within a configurable timeout. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, Any + +from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds +from vllm.distributed.kv_transfer.kv_connector.v1.base import ( + KVConnectorMetadata, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_scheduler import ( + NixlBaseConnectorScheduler, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + NixlConnectorMetadata, + ReqId, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config import VllmConfig + from vllm.v1.core.kv_cache_manager import KVCacheBlocks + from vllm.v1.core.sched.output import SchedulerOutput + from vllm.v1.kv_cache_interface import KVCacheConfig + from vllm.v1.outputs import KVConnectorOutput + from vllm.v1.request import Request + +logger = init_logger(__name__) + + +class NixlPushConnectorScheduler(NixlBaseConnectorScheduler): + """Push-specific scheduler logic (WRITE-based KV transfer). + + All P2P communication is deferred to the worker level via NIXL + notifications. The scheduler communicates with workers only through + the standard ``build_connector_meta`` / ``update_connector_output`` + hooks. + """ + + def __init__( + self, + vllm_config: VllmConfig, + engine_id: str, + kv_cache_config: KVCacheConfig, + ): + super().__init__(vllm_config, engine_id, kv_cache_config) + + # D-side: registration data to pass to D workers via metadata on + # the next ``build_connector_meta`` call. + self._push_pending_registrations: dict[ReqId, dict[str, Any]] = {} + + # D-side: track the wall-clock deadline for each registered request + # to detect "registered but never fulfilled" failures (e.g. the P + # node disappeared after registration). Keyed by D request_id. + self._push_registration_deadlines: dict[ReqId, float] = {} + + # P-side: block IDs for finished requests, kept for the lease and + # used to drive ``has_pending_push_work``. + self._finished_request_blocks: dict[ReqId, BlockIds] = {} + # P-side: newly finished blocks to ship to P workers on next step. + self._newly_finished_push_blocks: dict[ReqId, BlockIds] = {} + + # Soft watchdog timeout (seconds) for D-side registrations that + # never receive a push completion. Defaults to the existing + # decoder KV blocks TTL so behaviour matches the lease. + assert vllm_config.kv_transfer_config is not None + self._push_registration_timeout: float = float( + vllm_config.kv_transfer_config.get_from_extra_config( + "push_registration_timeout", + self.decoder_kv_blocks_ttl, + ) + ) + + def get_num_new_matched_tokens( + self, request: Request, num_computed_tokens: int + ) -> tuple[int, bool]: + """In push mode, D doesn't pull — it registers blocks and waits. + + However, we still need to handle the do_remote_prefill case where D + needs to know how many tokens will be pushed. + """ + params = request.kv_transfer_params + logger.debug( + "NixlPushConnector get_num_new_matched_tokens: " + "num_computed_tokens=%s, kv_transfer_params=%s", + num_computed_tokens, + params, + ) + + if params is not None and params.get("do_remote_prefill"): + token_ids = request.prompt_token_ids or [] + actual = self._mamba_prefill_token_count(len(token_ids)) + count = actual - num_computed_tokens + if count > 0: + return count, True + + if params is not None and params.get("do_remote_decode") and self._has_mamba: + self._truncate_mamba_request_for_prefill(request) + + return 0, False + + def update_state_after_alloc( + self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int + ): + """In push mode, D stores registration data for the worker to send + to P via NIXL notification (deferred to ``build_connector_meta``). + """ + params = request.kv_transfer_params + logger.debug( + "NixlPushConnector update_state_after_alloc: " + "num_external_tokens=%s, kv_transfer_params=%s", + num_external_tokens, + params, + ) + + if not params: + return + + # P side: track the request as in-batch so the lease accounting + # matches what the worker expects on the next step. + if params.get("do_remote_decode"): + self._reqs_in_batch.add(request.request_id) + + # P side with host-buffer offload: defer save to the worker. + if self.use_host_buffer and params.get("do_remote_decode"): + self._reqs_need_save[request.request_id] = request + return + + # D side: only act on the first call (``do_remote_prefill`` is + # unset on re-entry by the marker below). + if not params.get("do_remote_prefill"): + return + + if num_external_tokens <= 0: + # Nothing to receive: full prefix-cache hit on D, no + # registration to stage. + return + + # First-pass D path: stash registration data the worker will + # ship to P on the next ``build_connector_meta`` cycle. + logger.debug( + "KV PUSH mode: D node storing registration for request %s", + request.request_id, + ) + local_block_ids: BlockIds = blocks.get_unhashed_block_ids_all_groups() + local_block_ids = self.get_sw_clipped_blocks(local_block_ids) + + # ``remote_*`` fields are P's coordinates (from D's perspective). + # ``decode_*`` fields are D's own info that P needs for the + # reverse handshake before WRITE-ing. + self._push_pending_registrations[request.request_id] = { + "request_id": request.request_id, + "decode_engine_id": self.engine_id, + "decode_host": self.side_channel_host, + "decode_port": self.side_channel_port, + "decode_tp_size": (self.vllm_config.parallel_config.tensor_parallel_size), + "local_block_ids": local_block_ids, + "remote_engine_id": params["remote_engine_id"], + "remote_host": params["remote_host"], + "remote_port": params["remote_port"], + "remote_tp_size": params["tp_size"], + } + self._push_registration_deadlines[request.request_id] = ( + time.perf_counter() + self._push_registration_timeout + ) + # In push mode D doesn't know P's blocks; P determines them + # from the registration. We still track the request as + # needing recv so the engine waits for P's WRITE completion. + # ``remote_block_ids`` is also seeded to an empty tuple so the + # base scheduler's ``add_new_req_to_recv`` can build the + # ReqMeta without a KeyError — the actual remote block IDs are + # learned by P over the NIXL handshake at WRITE time. + params["remote_block_ids"] = () + self._reqs_need_recv[request.request_id] = (request, local_block_ids) + + # Mark as processed so a re-entry (e.g. preemption + reschedule) + # doesn't re-stage the registration. + params["do_remote_prefill"] = False + + def request_finished( + self, + request: Request, + block_ids: BlockIds, + ) -> tuple[bool, dict[str, Any] | None]: + """Push-mode request_finished: stores blocks for workers.""" + from vllm.v1.request import RequestStatus + + params = request.kv_transfer_params + logger.debug( + "NixlPushConnector request_finished(%s), request_status=%s, " + "kv_transfer_params=%s", + request.request_id, + request.status, + params, + ) + if not params: + return False, None + + is_p_node = bool(params.get("do_remote_decode")) + + self._stop_heartbeat(request.request_id) + # Drop any pending registration deadline; the request either + # completed or was cancelled. + self._push_registration_deadlines.pop(request.request_id, None) + + if params.get("do_remote_prefill"): + # ``do_remote_prefill`` is still set, which means + # ``update_state_after_alloc`` never ran (it would have + # flipped this flag to False). The request was aborted + # before it could be scheduled — e.g. rejected at the D + # serving layer via abort_immediately. To keep P from + # stranding the prefill blocks, we still register an empty + # recv so the worker emits a notif that lets P free them. + self._reqs_need_recv[request.request_id] = (request, []) + params["do_remote_prefill"] = False + return False, None + + # Push connector only acts on the P-side terminal path; D-side + # finishing without a remote prefill is a no-op. + if not is_p_node: + return False, None + + if request.status not in ( + RequestStatus.FINISHED_LENGTH_CAPPED, + RequestStatus.FINISHED_STOPPED, + ): + self._reqs_not_processed.add(request.request_id) + self._reqs_need_save.pop(request.request_id, None) + return False, None + + delay_free_blocks = any(len(group) > 0 for group in block_ids) + remote_num_tokens = 0 + if delay_free_blocks: + logger.debug( + "NixlPushConnector request_finished(%s) waiting for %d seconds " + "before releasing blocks", + request.request_id, + self._kv_lease_duration, + ) + self._reqs_need_send[request.request_id] = ( + time.perf_counter() + self._kv_lease_duration + ) + + block_ids = self.get_sw_clipped_blocks(block_ids) + remote_num_tokens = request.num_computed_tokens + + # Store finished blocks for worker-level matching with D + # registrations (via NIXL notifications). + self._finished_request_blocks[request.request_id] = block_ids + self._newly_finished_push_blocks[request.request_id] = block_ids + + return delay_free_blocks, dict( + do_remote_prefill=True, + do_remote_decode=False, + remote_block_ids=block_ids, + remote_engine_id=self.engine_id, + remote_request_id=request.request_id, + remote_host=self.side_channel_host, + remote_port=self.side_channel_port, + tp_size=self.vllm_config.parallel_config.tensor_parallel_size, + remote_num_tokens=remote_num_tokens, + ) + + def build_connector_meta( + self, + scheduler_output: SchedulerOutput, + ) -> KVConnectorMetadata: + meta = super().build_connector_meta(scheduler_output) + assert isinstance(meta, NixlConnectorMetadata) + + # Watchdog: any D-side registration whose deadline has passed without + # a corresponding push completion is treated as failed and cleaned up. + # The corresponding request is already tracked via _reqs_need_recv; + # the engine layer will eventually time it out via the lease, but we + # at least drop the stale registration so we don't keep retrying. + now = time.perf_counter() + # Deadlines are inserted in non-decreasing order (monotonic clock + + # constant timeout, armed once per request), and dict insertion order + # is preserved across key deletions, so we can stop at the first + # not-yet-expired entry instead of scanning the whole dict. + expired = [] + for rid, deadline in self._push_registration_deadlines.items(): + if deadline > now: + break + expired.append(rid) + for rid in expired: + self._push_registration_deadlines.pop(rid, None) + # Avoid resending a registration that already timed out. + self._push_pending_registrations.pop(rid, None) + logger.warning( + "NixlPushConnector: registration for request %s timed out " + "after %.1fs without a push completion", + rid, + self._push_registration_timeout, + ) + + # D side: package pending registrations for D workers to send out. + if self._push_pending_registrations: + meta.push_registrations = dict(self._push_pending_registrations) + self._push_pending_registrations.clear() + + # P side: package newly finished blocks for P workers to match against + # any D registrations they have received via NIXL notifications. + if self._newly_finished_push_blocks: + meta.push_finished_blocks = dict(self._newly_finished_push_blocks) + self._newly_finished_push_blocks.clear() + + return meta + + def has_pending_push_work(self) -> bool: + # Keep the engine main loop alive while we have: + # - finished P blocks awaiting WRITE completion, or + # - pending D registrations the worker has not yet shipped, or + # - newly finished blocks not yet shipped to P workers. + return bool(self._finished_request_blocks or self._push_pending_registrations) + + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: + """Clean up finished request blocks after push completes.""" + super().update_connector_output(connector_output) + for req_id in connector_output.finished_sending or (): + self._finished_request_blocks.pop(req_id, None) + # On D side, finished_recving means the push completed; clear the + # watchdog so we don't trip an expiration on a fulfilled request. + for req_id in connector_output.finished_recving or (): + self._push_registration_deadlines.pop(req_id, None) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py new file mode 100644 index 0000000000..a15fc204d2 --- /dev/null +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/push_worker.py @@ -0,0 +1,742 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Push-specific (WRITE) worker-side logic for the NIXL connector. + +A dedicated ``nixl-push-writer`` thread owns all push-related NIXL ops: +calls ``get_new_notifs`` (routing PUSH_REG internally; HB / completion +notifs are forwarded to the engine main thread), sends PUSH_REG via +``send_notif``, matches D registrations with P finished blocks, and +issues WRITE transfers via ``make_prepped_xfer`` / ``transfer``. + +The engine main thread feeds the writer through three queues: +``_reg_send_inbox`` (D-side regs to send), ``_finished_blocks_inbox`` +(P-side blocks from metadata) and ``_pending_completion_notifs`` +(non-PUSH_REG notifs forwarded back for HB / completion accounting). + +Wake model: the writer self-polls every +``_PUSH_WRITER_POLL_INTERVAL_MS`` only while it has unmatched +``_push_finished_blocks`` (i.e. P-side blocks waiting for a D PUSH_REG +notif that has no other wake source). All other progress is +event-driven: the engine main thread sets ``_push_writer_wake`` from +``start_load_kv`` (when handing it new work) and from ``get_finished`` +(so each engine step gives the writer a chance to drain NIXL notifs); +the handshake-completion callback sets the same event after a deferred +PUSH_REG send has been queued. When a request's lease expires (the base +worker reports it via ``done_sending``) or the WRITE completes, +``get_finished`` enqueues an eviction onto ``_evict_finished_inbox`` so +the writer drops any leftover ``_push_finished_blocks`` / +``_pending_d_registrations`` and stops self-polling. +""" + +import queue +import threading +import time +from collections import defaultdict +from concurrent.futures import Future +from typing import TYPE_CHECKING, Any + +import msgspec +import numpy as np + +from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import ( + NixlBaseConnectorWorker, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( + PUSH_REG_NOTIF_PREFIX, + NixlConnectorMetadata, + RemoteMeta, + ReqId, + ReqMeta, + TransferHandle, +) +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ReadSpec +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import get_base_request_id +from vllm.logger import init_logger + +if TYPE_CHECKING: + import torch + + from vllm.config import VllmConfig + from vllm.v1.kv_cache_interface import KVCacheConfig + +logger = init_logger(__name__) + +# Writer-thread poll cadence while there is in-flight push state. When +# fully idle, the writer blocks on a wake event signalled by the engine +# main thread (start_load_kv / get_finished). Smaller -> lower latency +# while active, slightly more CPU. +_PUSH_WRITER_POLL_INTERVAL_MS = 1.0 + + +class NixlPushConnectorWorker(NixlBaseConnectorWorker): + """Push-specific (WRITE) worker logic. See module docstring.""" + + def __init__( + self, + vllm_config: "VllmConfig", + engine_id: str, + kv_cache_config: "KVCacheConfig", + ): + super().__init__(vllm_config, engine_id, kv_cache_config) + + # Push-specific state. + # P-side: outgoing WRITE handles awaiting completion, keyed by + # request_id. Mutated by writer (submit) and main thread + # (``_pop_done_transfers``); guarded by + # ``_sending_transfers_lock``. + self._sending_transfers = defaultdict[ReqId, list[TransferHandle]](list) + self._sending_transfers_lock = threading.Lock() + + # Writer-thread owned matching state. + # P-side: finished request blocks received from scheduler metadata + # that have not yet been matched with an incoming D registration. + self._push_finished_blocks: dict[ReqId, BlockIds] = {} + # P-side: D registrations received via NIXL notification that have + # not yet been matched with a finished P request. + self._pending_d_registrations: dict[ReqId, dict[str, Any]] = {} + + # Cross-thread channels. + self._reg_send_inbox: queue.Queue[tuple[str, dict[str, Any]]] = queue.Queue() + self._finished_blocks_inbox: queue.Queue[tuple[str, BlockIds]] = queue.Queue() + self._pending_completion_notifs: queue.Queue[bytes] = queue.Queue() + # Main thread → writer: req_ids whose lease has expired or whose + # WRITE has completed. Writer drops them from + # ``_push_finished_blocks`` so an unmatched entry doesn't keep the + # writer busy-polling forever. + self._evict_finished_inbox: queue.Queue[str] = queue.Queue() + + # Wake signal from engine main thread (start_load_kv / get_finished). + # Writer self-polls at _PUSH_WRITER_POLL_INTERVAL_MS while it has + # active in-flight state; otherwise it blocks until signalled. + self._push_writer_wake = threading.Event() + + self._push_writer_stop = threading.Event() + self._push_writer_thread: threading.Thread | None = None + + # --- Lifecycle ----------------------------------------------------- # + + def register_kv_caches(self, kv_caches: dict[str, "torch.Tensor"]): + super().register_kv_caches(kv_caches) + if self._push_writer_thread is None: + self._push_writer_thread = threading.Thread( + target=self._push_writer_loop, + daemon=True, + name="nixl-push-writer", + ) + self._push_writer_thread.start() + logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank) + + def shutdown(self): + self._push_writer_stop.set() + # Unblock the writer if it's waiting in the no-active-state branch. + self._push_writer_wake.set() + if self._push_writer_thread is not None: + self._push_writer_thread.join(timeout=2) + self._push_writer_thread = None + with self._sending_transfers_lock: + for handles in self._sending_transfers.values(): + for handle in handles: + self.nixl_wrapper.release_xfer_handle(handle) + self._sending_transfers.clear() + super().shutdown() + + # --- Engine-main-thread entry point -------------------------------- # + + def start_load_kv(self, metadata: NixlConnectorMetadata): + """Pre-process metadata; defer NIXL ops to the writer thread.""" + # D-side: track reqs waiting for P to push. + for req_id, meta in metadata.reqs_to_recv.items(): + meta.local_physical_block_ids = self._logical_to_kernel_block_ids( + meta.local_block_ids + ) + assert meta.remote is not None + remote_engine_id = meta.remote.engine_id + logger.debug( + "start_load_kv (push) for request %s from remote engine %s. " + "Num local_block_ids: %s. Num remote_block_ids: %s. ", + req_id, + remote_engine_id, + len(meta.local_physical_block_ids), + len(meta.remote.block_ids), + ) + self._recving_metadata[req_id] = meta + + # --- D-side: registrations to send to P via NIXL --- + if metadata.push_registrations: + for req_id, reg_data in metadata.push_registrations.items(): + self._reg_send_inbox.put((req_id, reg_data)) + self._push_writer_wake.set() + + # --- P-side: newly finished blocks awaiting a D registration match --- + if metadata.push_finished_blocks: + for req_id, block_ids in metadata.push_finished_blocks.items(): + self._finished_blocks_inbox.put((req_id, block_ids)) + self._push_writer_wake.set() + + # Batch + lease tracking (same as pull). + for req_id in metadata.reqs_in_batch: + self._reqs_to_process.add(req_id) + for req_id in metadata.reqs_not_processed: + self._reqs_to_process.discard(req_id) + assert req_id not in self._reqs_to_send + for req_id, expiration_time in metadata.reqs_to_send.items(): + if req_id in self._reqs_to_process: + self._reqs_to_send[req_id] = expiration_time + + # Heartbeats still leave from the main thread (base worker behaviour). + self._send_heartbeats(metadata) + + # --- Writer thread ------------------------------------------------- # + + def _push_writer_loop(self) -> None: + sleep_s = _PUSH_WRITER_POLL_INTERVAL_MS / 1000.0 + + while not self._push_writer_stop.is_set(): + try: + # 1. D registrations to send. + while True: + try: + rid, rd = self._reg_send_inbox.get_nowait() + except queue.Empty: + break + self._send_registration_to_p(rid, rd) + + # 2. P-side finished blocks; match against pending regs. + while True: + try: + rid, blocks = self._finished_blocks_inbox.get_nowait() + except queue.Empty: + break + matched = self._pop_matching_registration(rid) + if matched is not None: + self._do_start_push_kv(rid, blocks, matched) + else: + self._push_finished_blocks[rid] = blocks + + # 2b. Evict finished blocks for requests that have either + # completed (WRITE acknowledged) or whose lease expired + # without a D registration. Drop pending registrations + # for the same reason so we don't leak state. + while True: + try: + rid = self._evict_finished_inbox.get_nowait() + except queue.Empty: + break + self._push_finished_blocks.pop(rid, None) + self._pending_d_registrations.pop(rid, None) + + # 3. NIXL notifs: route PUSH_REG; forward the rest. + for notifs in self.nixl_wrapper.get_new_notifs().values(): + for notif in notifs: + if notif.startswith(PUSH_REG_NOTIF_PREFIX): + self._handle_push_reg_notif(notif) + else: + self._pending_completion_notifs.put(notif) + except Exception: + logger.exception("nixl-push-writer error; continuing") + + # Self-poll only while there is no other wake source: P-side + # finished blocks waiting for a D PUSH_REG match. All other + # progress is event-driven (see module docstring). + if self._push_finished_blocks: + self._push_writer_stop.wait(timeout=sleep_s) + else: + self._push_writer_wake.wait() + self._push_writer_wake.clear() + + def _handle_push_reg_notif(self, notif: bytes) -> None: + try: + reg_data = msgspec.msgpack.decode(notif[len(PUSH_REG_NOTIF_PREFIX) :]) + except Exception: + logger.exception("Failed to decode PUSH_REG notification payload") + return + rid = reg_data.get("request_id") if isinstance(reg_data, dict) else None + if not isinstance(rid, str): + logger.warning("PUSH_REG notif missing request_id; dropping") + return + + match = self._pop_matching_finished_blocks(rid) + if match is not None: + fin_id, blocks = match + self._do_start_push_kv(fin_id, blocks, reg_data) + else: + self._pending_d_registrations[rid] = reg_data + + # --- D-side registration send (writer thread) ---------------------- # + + def _send_registration_to_p( + self, + req_id: str, + reg_data: dict[str, Any], + ) -> None: + """Handshake (if needed) then send PUSH_REG. ``send_notif`` always + executes on the writer; the handshake runs on the background executor + and the request is re-queued onto ``_reg_send_inbox`` once it + completes (at which point ``_ensure_handshake`` returns ``None`` and we + send directly).""" + fut = self._ensure_handshake( + reg_data["remote_engine_id"], + reg_data["remote_host"], + reg_data["remote_port"], + reg_data["remote_tp_size"], + ) + if fut is None: + self._do_send_reg_notif(req_id, reg_data) + return + + def _on_handshake( + f: Future[dict[int, str]], + rid: str = req_id, + rd: dict[str, Any] = reg_data, + ) -> None: + try: + f.result() + except Exception as e: + self._log_failure( + failure_type="push_reg_handshake_failed", req_id=rid, error=e + ) + self._handle_failed_transfer(rid, None) + return + # Re-queue for the writer to send now that the handshake is done. + self._reg_send_inbox.put((rid, rd)) + # Wake the writer so it sends the PUSH_REG promptly even if + # otherwise parked. + self._push_writer_wake.set() + + fut.add_done_callback(_on_handshake) + + def _do_send_reg_notif(self, req_id: str, reg_data: dict[str, Any]) -> None: + engine_id = reg_data["remote_engine_id"] + notif_msg = PUSH_REG_NOTIF_PREFIX + msgspec.msgpack.encode(reg_data) + agents = self._remote_agents.get(engine_id) + if not agents: + logger.error( + "No remote agents for engine %s; cannot send registration for %s", + engine_id, + req_id, + ) + self._handle_failed_transfer(req_id, None) + return + for rank, agent_name in agents.items(): + try: + self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_msg) + except Exception as e: + self._log_failure( + failure_type="push_reg_notif_failed", + req_id=req_id, + error=e, + remote_rank=rank, + ) + logger.debug( + "Sent PUSH_REG for %s to engine %s (%dB)", req_id, engine_id, len(notif_msg) + ) + + # --- Matching helpers --------------------------------------------- # + + def _pop_matching_registration(self, request_id: str) -> dict[str, Any] | None: + """Pop the D-side registration matching *request_id*. + + Exact key first, then a match after stripping the random suffix from + both sides. No match leaves the request unmatched (push not started). + """ + data = self._pending_d_registrations.pop(request_id, None) + if data is not None: + return data + base_id = get_base_request_id(request_id) + for reg_id in list(self._pending_d_registrations): + if get_base_request_id(reg_id) == base_id: + return self._pending_d_registrations.pop(reg_id) + return None + + def _pop_matching_finished_blocks( + self, request_id: str + ) -> tuple[str, BlockIds] | None: + """Pop the P-side finished blocks matching *request_id*. + + Same lookup as ``_pop_matching_registration``: exact key, then a + match after stripping the random suffix from both sides. + """ + blocks = self._push_finished_blocks.pop(request_id, None) + if blocks is not None: + return request_id, blocks + base_id = get_base_request_id(request_id) + for fin_id in list(self._push_finished_blocks): + if get_base_request_id(fin_id) == base_id: + return fin_id, self._push_finished_blocks.pop(fin_id) + return None + + # --- WRITE transfer logic (writer thread) ------------------------- # + + def _do_start_push_kv( + self, + request_id: str, + local_block_ids: BlockIds, + registration_data: dict[str, Any], + ) -> None: + """Start push-based KV transfer from P worker to D node. + + ``local_block_ids`` are P's *logical* block IDs (from the P + scheduler's metadata). ``registration_data["local_block_ids"]`` + are D's *logical* block IDs (from D's scheduler, sent over the + PUSH_REG notif). All conversion to physical block IDs is + deferred to ``_xfer_blocks_for_req`` so each side uses its own + physical-blocks-per-logical ratio (P uses + ``self._physical_blocks_per_logical_kv_block``; D's ratio is + learned during the NIXL handshake).""" + decode_engine_id = registration_data["decode_engine_id"] + remote_block_ids = registration_data["local_block_ids"] + decode_host = registration_data["decode_host"] + decode_port = registration_data["decode_port"] + decode_request_id = registration_data["request_id"] + if not local_block_ids: + logger.warning("No local blocks to push for request %s", request_id) + return + + if not self._ensure_d_handshake( + decode_engine_id, + decode_host, + decode_port, + registration_data["decode_tp_size"], + request_id, + ): + return + + # Both sides are kept in logical form here; ``_xfer_blocks_for_req`` + # expands each side using the appropriate ratio. + logical_local = self._as_grouped_block_ids(local_block_ids) + logical_remote = self._as_grouped_block_ids(remote_block_ids) + physical_local = self._logical_to_kernel_block_ids(logical_local) + + push_meta = ReqMeta( + local_block_ids=logical_local, + local_physical_block_ids=physical_local, + tp_size=self.world_size, + remote=RemoteMeta( + block_ids=logical_remote, + host="", + port=0, + engine_id=decode_engine_id, + request_id=decode_request_id, + ), + ) + + t0 = time.perf_counter() + self._xfer_blocks_for_req(req_id=request_id, meta=push_meta) + elapsed_ms = (time.perf_counter() - t0) * 1000.0 + if elapsed_ms > 200.0: + logger.warning( + "_do_start_push_kv for %s took %.1fms (slow NIXL submission)", + request_id, + elapsed_ms, + ) + + def _ensure_d_handshake( + self, + decode_engine_id: str, + decode_host: str, + decode_port: int, + decode_tp_size: int, + request_id: str, + ) -> bool: + """First-time P→D handshake. Blocking call on the writer thread. + + Returns True iff the handshake succeeded (or had already been + completed). Returns False if the handshake raised; the request is + skipped in that case (the engine layer will reschedule or fail it + via the standard lease/timeout path).""" + if decode_engine_id in self._remote_agents: + return True + try: + remote_agents = self._nixl_handshake( + decode_host, + decode_port, + decode_tp_size, + decode_engine_id, + ) + except Exception: + logger.exception( + "Failed handshake to D %s for push %s", + decode_engine_id, + request_id, + ) + return False + with self._handshake_lock: + self._remote_agents[decode_engine_id] = remote_agents + logger.info( + "Push handshake to D %s done (%d agents)", + decode_engine_id, + len(remote_agents), + ) + return True + + @staticmethod + def _as_grouped_block_ids(block_ids: BlockIds) -> BlockIds: + """Normalise a sequence of block IDs to a tuple-of-groups shape. + + ``BlockIds`` is canonically a tuple of per-group lists, but some + registration payloads collapse a single-group case to a flat + list. Re-wrap that case so downstream group-aware helpers see a + consistent shape.""" + if block_ids and not isinstance(block_ids[0], (list, tuple)): + return (list(block_ids),) + return block_ids + + def _xfer_blocks_for_req(self, req_id: str, meta: ReqMeta): + """Issue WRITE transfers to one or more remote TP ranks.""" + assert meta.remote is not None and self.transfer_topo is not None + engine_id = meta.remote.engine_id + plan = self.tp_mappings[engine_id] + remote_info = self.transfer_topo.get_engine_info(engine_id) + tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) + + # Expand D's logical IDs using the ratio learned during the + # NIXL handshake. ``meta`` is freshly built by + # ``_do_start_push_kv`` so mutating it here is safe. + meta.remote.block_ids = self._logical_to_remote_kernel_block_ids( + meta.remote.block_ids, + remote_info.remote_physical_blocks_per_logical, + ) + remote_block_ids = meta.remote.block_ids + local_block_ids = meta.local_physical_block_ids + num_groups = len(local_block_ids) + read_specs = [ + ReadSpec( + remote_rank=rank, + local_block_ids=[ + list(local_block_ids[g]) + if rank in plan.source_ranks_per_group[g] + else [] + for g in range(num_groups) + ], + remote_block_ids=[ + list(remote_block_ids[g]) + if rank in plan.source_ranks_per_group[g] + else [] + for g in range(num_groups) + ], + ) + for rank in plan.all_source_ranks + ] + + if self.use_mla and tp_ratio < 0: + assert len(read_specs) == 1 + + for i, spec in enumerate(read_specs): + remote_block_size = remote_info.remote_block_size + logger.debug( + "Remote agent %s available, calling _xfer_blocks" + " on remote rank %s with remote block size %s for req %s", + meta.remote.engine_id, + spec.remote_rank, + remote_block_size, + req_id, + ) + if tp_ratio < 0 and not self.use_mla: + assert remote_block_size == self.block_size + local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[tp_ratio][i] + else: + local_xfer_side_handle = self.src_xfer_handles_by_block_size[ + remote_block_size + ] + + remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][ + spec.remote_rank + ] + + self._xfer_blocks( + read_spec=spec, + request_id=req_id, + dst_engine_id=meta.remote.engine_id, + remote_request_id=meta.remote.request_id, + local_xfer_side_handle=local_xfer_side_handle, + remote_xfer_side_handle=remote_xfer_side_handle, + ) + + if self.use_mla and tp_ratio < 0 and read_specs: + notif_id = f"{meta.remote.request_id}:{self.world_size}".encode() + remote_agents = self._remote_agents[meta.remote.engine_id] + for rank_to_notify, agent in remote_agents.items(): + if rank_to_notify != read_specs[0].remote_rank: + self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) + + def _xfer_blocks( + self, + read_spec: ReadSpec, + dst_engine_id: str, + request_id: str, + remote_request_id: str, + local_xfer_side_handle: int, + remote_xfer_side_handle: int, + ): + """Post a WRITE point-to-point xfer request.""" + assert self.transfer_topo is not None + remote_rank = read_spec.remote_rank + local_block_ids = read_spec.local_block_ids + remote_block_ids = read_spec.remote_block_ids + + remote_info = self.transfer_topo.get_engine_info(dst_engine_id) + block_size_ratio = self.transfer_topo.block_size_ratio( + remote_info.remote_block_size + ) + if block_size_ratio > 1: + assert not self._is_hma_required + local_block_ids0 = local_block_ids[0] if local_block_ids else [] + remote_block_ids0 = remote_block_ids[0] + local_block_ids_mapped = self.get_mapped_blocks( + np.asarray(local_block_ids0), block_size_ratio + ).tolist() + if len(local_block_ids_mapped) > len(remote_block_ids0): + local_block_ids_mapped = local_block_ids_mapped[ + : len(remote_block_ids0) + ] + local_block_ids = [local_block_ids_mapped] if local_block_ids_mapped else [] + remote_block_ids = [remote_block_ids0] + + notif_id = f"{remote_request_id}:{self.world_size}".encode() + + if len(local_block_ids) == 0: + logger.warning("No blocks to push for request %s", request_id) + return + + # Align per-group block counts for push. + local_block_ids = list(local_block_ids) + remote_block_ids = list(remote_block_ids) + for i in range(min(len(local_block_ids), len(remote_block_ids))): + num_local = len(local_block_ids[i]) + num_remote = len(remote_block_ids[i]) + if num_local > num_remote: + local_block_ids[i] = local_block_ids[i][:num_remote] + elif num_local < num_remote: + remote_block_ids[i] = remote_block_ids[i][:num_local] + + # Get descs ids. + remote_block_descs_ids = self._compute_desc_ids( + block_ids=remote_block_ids, + dst_num_blocks=self.dst_num_blocks[dst_engine_id], + block_size_ratio=None, + physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical, + ) + local_block_descs_ids = self._compute_desc_ids( + block_ids=local_block_ids, + dst_num_blocks=self.dst_num_blocks[self.engine_id], + block_size_ratio=block_size_ratio, + physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block, + ) + + assert len(local_block_descs_ids) == len(remote_block_descs_ids) + + handle = None + try: + handle = self.nixl_wrapper.make_prepped_xfer( + "WRITE", + local_xfer_side_handle, + local_block_descs_ids, + remote_xfer_side_handle, + remote_block_descs_ids, + notif_msg=notif_id, + ) + self.nixl_wrapper.transfer(handle) + # Track push WRITE handles so P can free blocks once done. + with self._sending_transfers_lock: + self._sending_transfers[request_id].append(handle) + except Exception as e: + self._log_failure( + failure_type="transfer_setup_failed", + req_id=request_id, + msg="Push WRITE submission failed; releasing handle", + error=e, + dst_engine_id=dst_engine_id, + remote_rank=remote_rank, + ) + # On the P side this WRITE failure is purely outbound; we + # don't have a ``_recving_metadata`` entry to invalidate, so + # we just release the handle and let the engine reschedule + # via the lease / watchdog. + if handle is not None: + self.nixl_wrapper.release_xfer_handle(handle) + self.xfer_stats.record_failed_transfer() + + # --- Notification handling on engine main thread ------------------ # + + def _get_new_notifs(self) -> set[str]: + """Drain HB / completion notifs forwarded by the writer thread. + + The writer owns ``nixl_wrapper.get_new_notifs`` for push; PUSH_REG + notifs are handled there. Everything else is forwarded here for + existing accounting. + """ + assert self.transfer_topo is not None + notified_req_ids: set[str] = set() + while True: + try: + notif = self._pending_completion_notifs.get_nowait() + except queue.Empty: + break + + msg = notif.decode("utf-8") + if msg.startswith("HB:"): + self._handle_heartbeat(msg[3:]) + continue + + req_id, tp_size = msg.rsplit(":", 1) + + # Not tracked as a P-side send/process for this notif. + if req_id not in self._reqs_to_send and req_id not in self._reqs_to_process: + if req_id in self._recving_metadata: + # D-side: P signalled push completion. The transfer was + # driven entirely by P (we don't own a NIXL handle here), + # so materialise an empty entry in ``_recving_transfers`` + # and let ``_pop_done_transfers`` report it done on the + # next ``get_finished``. + self._recving_transfers.setdefault(req_id, []) + else: + # Not tracked on either side (lease may have expired + # before the notif arrived). Log and skip. + logger.error( + "Unrecognized request %s notif (may have expired).", + req_id, + ) + continue + + n_consumers = int(tp_size) + tp_ratio = self.transfer_topo.tp_ratio(n_consumers) + consumers_per_producer = -tp_ratio if n_consumers > self.world_size else 1 + self.consumer_notification_counts_by_req[req_id] += 1 + if ( + self.consumer_notification_counts_by_req[req_id] + == consumers_per_producer + ): + notified_req_ids.add(req_id) + del self.consumer_notification_counts_by_req[req_id] + self._reqs_to_process.remove(req_id) + self._reqs_to_send.pop(req_id, None) + return notified_req_ids + + def get_finished(self) -> tuple[set[str], set[str]]: + # Engine main thread asking for completions: also wake the writer + # so it gets a chance to drain NIXL notifs (heartbeats, completion + # notifs, late PUSH_REGs) even if it had been parked. + self._push_writer_wake.set() + + done_sending, done_recving = super().get_finished() + + # ``_pop_done_transfers`` mutates ``_sending_transfers``; the + # writer thread also appends to it, so guard the pop. + with self._sending_transfers_lock: + done_pushing = self._pop_done_transfers(self._sending_transfers) + for req_id in done_pushing: + self._reqs_to_send.pop(req_id, None) + self._reqs_to_process.discard(req_id) + self.consumer_notification_counts_by_req.pop(req_id, None) + done_sending.add(req_id) + + # Tell the writer to drop any state it still holds for any + # request that just finished (push completed) or expired + # (lease ran out without a D registration ever arriving). + for req_id in done_sending: + self._evict_finished_inbox.put(req_id) + if done_sending: + self._push_writer_wake.set() + + return done_sending, done_recving diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py index b2122ed0d3..3da8e28a74 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/scheduler.py @@ -1,674 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Scheduler-side logic for the NIXL connector.""" +"""Backward-compatible re-export of NixlPullConnectorScheduler.""" -import threading -import time -from typing import TYPE_CHECKING, Any - -import msgspec -import zmq - -from vllm import envs -from vllm.distributed.kv_transfer.kv_connector.utils import ( - BlockIds, - EngineId, - yield_req_data, -) -from vllm.distributed.kv_transfer.kv_connector.v1.base import ( - KVConnectorHandshakeMetadata, - KVConnectorMetadata, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - GET_META_MSG, - HeartbeatInfo, - NixlConnectorMetadata, - NixlHandshakePayload, - ReqId, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import zmq_ctx -from vllm.logger import init_logger -from vllm.platforms import current_platform -from vllm.utils.math_utils import cdiv -from vllm.utils.network_utils import make_zmq_path -from vllm.v1.core.sched.output import SchedulerOutput -from vllm.v1.kv_cache_interface import ( - FullAttentionSpec, - MambaSpec, - SlidingWindowSpec, +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import ( + NixlPullConnectorScheduler, ) -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.core.kv_cache_manager import KVCacheBlocks - from vllm.v1.kv_cache_interface import KVCacheConfig - from vllm.v1.outputs import KVConnectorOutput - from vllm.v1.request import Request +# Backward compatibility: NixlConnectorScheduler is the pull-based scheduler. +NixlConnectorScheduler = NixlPullConnectorScheduler -logger = init_logger(__name__) - - -class NixlConnectorScheduler: - """Implementation of Scheduler side methods""" - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - self.vllm_config = vllm_config - self.block_size = vllm_config.cache_config.block_size - self.engine_id: EngineId = engine_id - self.kv_cache_config = kv_cache_config - self.side_channel_host = envs.VLLM_NIXL_SIDE_CHANNEL_HOST - self.side_channel_port = ( - envs.VLLM_NIXL_SIDE_CHANNEL_PORT - + vllm_config.parallel_config.data_parallel_index - ) - assert vllm_config.kv_transfer_config is not None - self._kv_lease_duration: int = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "kv_lease_duration", 30 - ) - ) - # NOTE (NickLucche): For now we use a hardcoded value for a simpler interface. - self._heartbeat_interval = self._kv_lease_duration // 6 - if current_platform.device_type == "cpu": - self.use_host_buffer = False - else: - self.use_host_buffer = ( - vllm_config.kv_transfer_config.kv_buffer_device == "cpu" - ) - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - # Also handle unlikely SW-only model case instead of checking num_groups>1. - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.kv_cache_groups - ) - ) - self._has_mamba = any( - isinstance(g.kv_cache_spec, MambaSpec) - for g in kv_cache_config.kv_cache_groups - ) - - logger.info("Initializing NIXL Scheduler %s", engine_id) - if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager: - logger.info("Hybrid Memory Allocator is enabled with NIXL") - - # Background thread for handling new handshake requests. - self._nixl_handshake_listener_t: threading.Thread | None = None - self._stop_event = threading.Event() - - # Requests that need to start recv/send. - # New requests are added by update_state_after_alloc in - # the scheduler. Used to make metadata passed to Worker. - self._reqs_need_recv: dict[ReqId, tuple[Request, BlockIds]] = {} - self._reqs_need_save: dict[ReqId, Request] = {} - # Reqs to send and their expiration time - self._reqs_need_send: dict[ReqId, float] = {} - self._reqs_in_batch: set[ReqId] = set() - # Reqs to remove from processed set because they're not to send after - # remote prefill or aborted. - self._reqs_not_processed: set[ReqId] = set() - - # Heartbeat tracking: requests needing periodic lease-renewal heartbeats to - # remote P-side, stored as ready-to-send HeartbeatInfo grouped by remote engine - self._heartbeat_by_engine: dict[EngineId, HeartbeatInfo] = {} - # Reverse lookup: local req_id -> (engine_id, remote_req_id) for O(1) removal - self._heartbeat_req_engine: dict[ReqId, tuple[EngineId, ReqId]] = {} - self._last_heartbeat_time: float = 0.0 - - # Gather Sliding Window sizes for each kv cache group (if any) in number of - # blocks per KV cache group. This is used to clip the local attention window. - sw_sizes_tokens: list[tuple[int, int]] = [ - (g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size) - if isinstance(g.kv_cache_spec, SlidingWindowSpec) - else (0, self.block_size) - for g in kv_cache_config.kv_cache_groups - ] - # cdiv(n_tokens, block_size) gives blocks/window; add 1 to conservatively - # account for boundary overlap eg window isn't fully aligned with blocks. - self.blocks_per_sw = [ - cdiv(n_tokens, block_size) + 1 if n_tokens else 0 - for n_tokens, block_size in sw_sizes_tokens - ] - - # Threshold to decide whether to compute kv cache locally - # or pull from a remote node: minimum number of remote - # tokens to amortize the xfer latencies - self.kv_recompute_threshold: int = int( - vllm_config.kv_transfer_config.get_from_extra_config( - "kv_recompute_threshold", 64 - ) - ) - - # Bi-directional KV transfer feature supports KV block - # transfers from D node to P node - self.is_bidirectional_kv_xfer_enabled = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "bidirectional_kv_xfer", False - ) - ) - self.decoder_kv_blocks_ttl = ( - vllm_config.kv_transfer_config.get_from_extra_config( - "decoder_kv_blocks_ttl", 480 - ) - ) - - if self.is_bidirectional_kv_xfer_enabled and self.kv_recompute_threshold > 0: - logger.info( - "Bidirectional KV transfer is enabled and the kv " - "recompute threshold is set to %d tokens." - "KV blocks on D are released after a TTL of %d seconds.", - self.kv_recompute_threshold, - self.decoder_kv_blocks_ttl, - ) - - def shutdown(self): - self._stop_event.set() - if self._nixl_handshake_listener_t is not None: - self._nixl_handshake_listener_t.join() - self._nixl_handshake_listener_t = None - - def on_new_request(self, request: "Request") -> None: - """Track a request that may need heartbeats.""" - params = request.kv_transfer_params - # NOTE (NickLucche) This excludes request meant for P, ie heartbeats are - # effectively disabled for Bidirectional KV transfer. - if params is None or not params.get("do_remote_prefill"): - return - # Only track if all required remote fields are present. - remote_engine_id = params.get("remote_engine_id") - remote_request_id = params.get("remote_request_id") - host = params.get("remote_host") - port = params.get("remote_port") - tp_size = params.get("tp_size") - if ( - remote_engine_id is None - or remote_request_id is None - or host is None - or port is None - or tp_size is None - ): - return - if remote_engine_id not in self._heartbeat_by_engine: - self._heartbeat_by_engine[remote_engine_id] = HeartbeatInfo( - req_ids=set(), - host=host, - port=port, - tp_size=tp_size, - ) - self._heartbeat_by_engine[remote_engine_id].req_ids.add(remote_request_id) - self._heartbeat_req_engine[request.request_id] = ( - remote_engine_id, - remote_request_id, - ) - - def _stop_heartbeat(self, req_id: ReqId) -> None: - """Remove *req_id* from heartbeat tracking (if tracked).""" - if key := self._heartbeat_req_engine.pop(req_id, None): - engine_id, remote_id = key - if info := self._heartbeat_by_engine.get(engine_id): - info.req_ids.discard(remote_id) - if not info.req_ids: - # Clean up empty engines so we don't leak a key when remote dies. - del self._heartbeat_by_engine[engine_id] - - def get_sw_clipped_blocks(self, block_ids: BlockIds) -> BlockIds: - """ - Clip the number of blocks to the sliding window size for each kv cache group - that employs SWA. - This is necessary because the KV Cache manager initially allocates blocks for - the entire sequence length, and successively cleans up blocks that are outside - the window prior to the `request_finished_all_groups` hook. - """ - if len(block_ids) == 0 or not self._is_hma_required: - # No blocks to clip eg Full prefix cache hit or not a hybrid model. - return block_ids - # NOTE (NickLucche) This logic is currently handled at the connector level - # because offloading connectors might want to receive the whole sequence even - # for SWA groups. We will abstract this logic once the interface is more stable - assert len(block_ids) == len(self.blocks_per_sw), ( - "Number of KV cache groups must match" - ) - # For non-SWA groups, blocks_per_sw is 0 so we return all block_ids unchanged - return tuple( - [ - blocks[-self.blocks_per_sw[i] :] - if self.blocks_per_sw[i] > 0 - else blocks - for i, blocks in enumerate(block_ids) - ] - ) - - def set_xfer_handshake_metadata( - self, metadata: dict[int, KVConnectorHandshakeMetadata] - ) -> None: - """ - Set the KV connector handshake metadata for this connector. - - Args: - metadata (dict): the handshake metadata to set. - """ - encoded_data: dict[int, bytes] = {} - encoder = msgspec.msgpack.Encoder() - for tp_rank, rank_metadata in metadata.items(): - if not isinstance(rank_metadata, NixlHandshakePayload): - raise ValueError( - "NixlConnectorScheduler expects NixlHandshakePayload for " - "handshake metadata." - ) - encoded_data[tp_rank] = encoder.encode(rank_metadata) - logger.debug( - "Tp rank %d: encoded NixlHandshakePayload size: %s bytes", - tp_rank, - str(len(encoded_data[tp_rank])), - ) - - # Only start the listener when we have metadata to serve. - if self._nixl_handshake_listener_t is None: - ready_event = threading.Event() - self._nixl_handshake_listener_t = threading.Thread( - target=self._nixl_handshake_listener, - args=( - encoded_data, - ready_event, - self._stop_event, - self.side_channel_host, - self.side_channel_port, - ), - daemon=True, - name="nixl_handshake_listener", - ) - self._nixl_handshake_listener_t.start() - ready_event.wait() # Wait for listener ZMQ socket to be ready. - - @staticmethod - def _nixl_handshake_listener( - encoded_data: dict[int, Any], - ready_event: threading.Event, - stop_event: threading.Event, - host: str, - port: int, - ): - """Background thread for getting new NIXL handshakes.""" - # NOTE(rob): this is a simple implementation. We will move - # to a better approach via HTTP endpoint soon. - - # Listen for new requests for metadata. - path = make_zmq_path("tcp", host, port) - logger.debug("Starting listening on path: %s", path) - with zmq_ctx(zmq.ROUTER, path) as sock: - sock.setsockopt(zmq.RCVTIMEO, 1000) - ready_event.set() - while True: - try: - identity, _, msg = sock.recv_multipart() - except zmq.Again: - if stop_event.is_set(): - break - continue - # Decode the message which contains (GET_META_MSG, rank) - msg, target_tp_rank = msgspec.msgpack.decode(msg) - logger.debug( - "Received message for tp rank %s", - target_tp_rank, - ) - if msg != GET_META_MSG: - logger.warning("Connection listener got unexpected message %s", msg) - sock.send_multipart((identity, b"", encoded_data[target_tp_rank])) - - def _mamba_prefill_token_count(self, num_prompt_tokens: int) -> int: - """D-side only. Returns N-1 for Mamba models since the decoder - always recomputes the last token and must start from h(N-1).""" - if self._has_mamba and num_prompt_tokens > 1: - return num_prompt_tokens - 1 - return num_prompt_tokens - - def _truncate_mamba_request_for_prefill(self, request: "Request") -> None: - """P-side only: drop the last prompt token so the prefiller computes - h(N-1) instead of h(N). The decoder recomputes the last token to - derive h(N) correctly. - - Guarded by ``_p_side_truncated`` to avoid repeated truncation if the - request is preempted and rescheduled.""" - params = request.kv_transfer_params - if ( - params is not None - # Guard against repeated truncation after preemption/reschedule. - and not params.get("_p_side_truncated") - and request.num_prompt_tokens > 1 - ): - if request.prompt_token_ids is not None: - request.prompt_token_ids.pop() - elif request.prompt_embeds is not None: - request.prompt_embeds = request.prompt_embeds[:-1] - else: - return - - request._all_token_ids.pop() - request.num_prompt_tokens -= 1 - request.max_tokens = 1 - params["_p_side_truncated"] = True - - def get_num_new_matched_tokens( - self, request: "Request", num_computed_tokens: int - ) -> tuple[int, bool]: - """ - For remote prefill, pull all prompt blocks from remote - asynchronously relative to engine execution. - - Args: - request (Request): the request object. - num_computed_tokens (int): the number of locally - computed tokens for this request - Returns: - * the number of tokens that can be loaded from the - external KV cache beyond what is already computed. - * true if the external KV cache tokens will be loaded - asynchronously (between scheduler steps). - """ - - params = request.kv_transfer_params - logger.debug( - "NIXLConnector get_num_new_matched_tokens: " - "num_computed_tokens=%s, kv_transfer_params=%s", - num_computed_tokens, - params, - ) - - if params is not None and params.get("do_remote_prefill"): - # Remote prefill: get all prompt blocks from remote. - token_ids = request.prompt_token_ids or [] - actual = self._mamba_prefill_token_count(len(token_ids)) - count = actual - num_computed_tokens - if count > 0: - return count, True - - if params is not None and params.get("do_remote_decode") and self._has_mamba: - self._truncate_mamba_request_for_prefill(request) - - if ( - params is not None - and params.get("do_remote_decode") - and params.get("remote_block_ids") - and all( - p in params - for p in ( - "remote_engine_id", - "remote_request_id", - "remote_host", - "remote_port", - ) - ) - ): - # Decode node has kv blocks for part of prefill request, so, provide them - # as an external token count to scheduler. - # The tokens will be loaded if not already present - # in the prefill node local cache - remote_num_tokens = params.get("remote_num_tokens") or 0 - count = ( - min(remote_num_tokens, request.num_prompt_tokens) - num_computed_tokens - ) - if count > 0: - # Check kv_recompute_threshold: skip pull if - # remote tokens are below the threshold. - if ( - self.kv_recompute_threshold > 0 - and count < self.kv_recompute_threshold - ): - logger.debug( - "Skipping remote pull for %s: %d remote tokens < threshold %d", - request.request_id, - count, - self.kv_recompute_threshold, - ) - return 0, False - return count, True - - # No remote prefill for this request. - return 0, False - - def update_state_after_alloc( - self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int - ): - params = request.kv_transfer_params - logger.debug( - "NIXLConnector update_state_after_alloc: " - "num_external_tokens=%s, kv_transfer_params=%s", - num_external_tokens, - params, - ) - - if not params: - return - - if params.get("do_remote_decode") or ( - params.get("do_remote_prefill") and self.is_bidirectional_kv_xfer_enabled - ): - self._reqs_in_batch.add(request.request_id) - if self.use_host_buffer and params.get("do_remote_decode"): - # NOTE: when accelerator is not directly supported by Nixl, - # prefilled blocks need to be saved to host memory before transfer. - self._reqs_need_save[request.request_id] = request - elif params.get("do_remote_prefill") or ( - params.get("do_remote_decode") - and self.is_bidirectional_kv_xfer_enabled - and not params.get("_remote_blocks_processed") - ): - if params.get("remote_block_ids"): - if all( - p in params - for p in ( - "remote_engine_id", - "remote_request_id", - "remote_host", - "remote_port", - ) - ): - # If remote_blocks and num_external_tokens = 0, we have - # a full prefix cache hit on the local node. We need to call - # send_notif in _read_blocks to free the memory on the remote node. - - unhashed_local_block_ids: BlockIds = ( - blocks.get_unhashed_block_ids_all_groups() - if num_external_tokens > 0 - else () - ) - local_block_ids = self.get_sw_clipped_blocks( - unhashed_local_block_ids - ) - - # Get unhashed blocks to pull from remote. Mind that a full prefix - # cache hit is indicated with an empty list. - self._reqs_need_recv[request.request_id] = ( - request, - local_block_ids, - ) - - else: - logger.warning( - "Got invalid KVTransferParams: %s. This " - "request will not utilize KVTransfer", - params, - ) - else: - assert num_external_tokens == 0 - # Only trigger 1 KV transfer per request. - params["do_remote_prefill"] = False - params["_remote_blocks_processed"] = True - - def _build_save_meta( - self, - meta: NixlConnectorMetadata, - scheduler_output: SchedulerOutput, - ) -> None: - # only called when use_host_buffer is True to build the save metadata - - # NOTE: For the prefill side, there might be a chance that an early added - # request is a chunked prefill, so we need to check if new blocks are added - for req_id, new_block_id_groups, _ in yield_req_data(scheduler_output): - req_to_save = self._reqs_need_save.get(req_id) - if req_to_save is None or new_block_id_groups is None: - continue - req = req_to_save - - assert req.kv_transfer_params is not None - clipped_block_id_groups = self.get_sw_clipped_blocks(new_block_id_groups) - meta.add_new_req_to_save( - request_id=req_id, - local_block_ids=clipped_block_id_groups, - kv_transfer_params=req.kv_transfer_params, - ) - assert scheduler_output.num_scheduled_tokens is not None - num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] - is_partial = ( - req.num_computed_tokens + num_scheduled_tokens - ) < req.num_prompt_tokens - if not is_partial: - # For non-partial prefills, once new req_meta is scheduled, it - # can be removed from _reqs_need_save. - # For partial prefill case, we will retain the request in - # _reqs_need_save until all blocks are scheduled with req_meta. - # Therefore, only pop if `not is_partial`. - self._reqs_need_save.pop(req_id) - - def build_connector_meta( - self, - scheduler_output: SchedulerOutput, - ) -> KVConnectorMetadata: - meta = NixlConnectorMetadata() - - # Loop through scheduled reqs and convert to ReqMeta. - for req_id, (req, block_ids) in self._reqs_need_recv.items(): - assert req.kv_transfer_params is not None - meta.add_new_req_to_recv( - request_id=req_id, - local_block_ids=block_ids, - kv_transfer_params=req.kv_transfer_params, - ) - - if self.use_host_buffer: - self._build_save_meta(meta, scheduler_output) - - meta.reqs_to_send = self._reqs_need_send - meta.reqs_in_batch = self._reqs_in_batch - meta.reqs_not_processed = self._reqs_not_processed - - # Package heartbeats, throttled by heartbeat_interval. - if self._heartbeat_by_engine: - now = time.perf_counter() - if now - self._last_heartbeat_time >= self._heartbeat_interval: - self._last_heartbeat_time = now - meta.heartbeat_by_engine = self._heartbeat_by_engine - - # Clear the list once workers start the transfers - self._reqs_need_recv.clear() - self._reqs_in_batch = set() - self._reqs_not_processed = set() - self._reqs_need_send = {} - - return meta - - def update_connector_output(self, connector_output: "KVConnectorOutput") -> None: - """Stop heartbeating for requests whose KV transfer completed.""" - for req_id in connector_output.finished_recving or (): - self._stop_heartbeat(req_id) - - def request_finished( - self, - request: "Request", - block_ids: BlockIds, - ) -> tuple[bool, dict[str, Any] | None]: - """ - Once a request is finished, determine whether request blocks - should be freed now or will be sent asynchronously and freed later. - """ - from vllm.v1.request import RequestStatus - - params = request.kv_transfer_params - logger.debug( - "NIXLConnector request_finished(%s), request_status=%s, " - "kv_transfer_params=%s", - request.request_id, - request.status, - params, - ) - if not params: - return False, None - - is_p_node = bool(params.get("do_remote_decode")) - is_d_node = not is_p_node - - # Stop heartbeating for aborted requests that never reached finished_recving: - # normal path cleans up in update_connector_output. - self._stop_heartbeat(request.request_id) - - if params.get("do_remote_prefill"): - # If do_remote_prefill is still True when the request is finished, - # update_state_after_alloc must not have been called (the request - # must have been aborted before it was scheduled, e.g. via the - # abort_immediately path used to clean up KV-transfer requests - # rejected at the D-side serving layer). - # To avoid stranding the prefill blocks in the prefill instance, - # we must add empty block_ids to _reqs_need_recv so that our - # worker side will notify and free blocks in the prefill instance. - self._reqs_need_recv[request.request_id] = (request, []) - params["do_remote_prefill"] = False - return False, None - - if is_d_node and not self.is_bidirectional_kv_xfer_enabled: - return False, None - - if request.status not in ( - RequestStatus.FINISHED_LENGTH_CAPPED, - RequestStatus.FINISHED_STOPPED, - ): - # Also include the case of a P/D Prefill request with immediate - # block free (eg abort). Stop tracking this request. - self._reqs_not_processed.add(request.request_id) - # Clear _reqs_need_save if a request is aborted as partial prefill. - self._reqs_need_save.pop(request.request_id, None) - return False, None - - # TODO: check whether block_ids actually ever be 0. If not we could - # remove the conditional below - delay_free_blocks = any(len(group) > 0 for group in block_ids) - remote_num_tokens = 0 - if delay_free_blocks: - # Prefill request on remote. It will be read from D upon completion - request_kv_blocks_ttl = self._kv_lease_duration - if is_d_node: - # For blocks pinned on D, use a simpler timeout for now instead of a - # lease mechanism as turn2 request is client-driven. - request_kv_blocks_ttl = self.decoder_kv_blocks_ttl - logger.debug( - "NIXLConnector request_finished(%s) waiting for %d seconds " - "before releasing blocks", - request.request_id, - request_kv_blocks_ttl, - ) - self._reqs_need_send[request.request_id] = ( - time.perf_counter() + request_kv_blocks_ttl - ) - # NOTE HMA will "mark" empty/null blocks in groups with 0s (eg SWA ones), - # trimming down after allocating for the whole sequence length. Empty - # blocks are always at the start of the list. - # Here we "unpad" blocks to send the actual remote blocks to be read. - block_ids = self.get_sw_clipped_blocks(block_ids) - - remote_num_tokens = request.num_computed_tokens - - return delay_free_blocks, dict( - do_remote_prefill=is_p_node, - do_remote_decode=is_d_node, - remote_block_ids=block_ids, - remote_engine_id=self.engine_id, - remote_request_id=request.request_id, - remote_host=self.side_channel_host, - remote_port=self.side_channel_port, - tp_size=self.vllm_config.parallel_config.tensor_parallel_size, - remote_num_tokens=remote_num_tokens, - ) +__all__ = ["NixlConnectorScheduler", "NixlPullConnectorScheduler"] diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py index 2fa3829eae..b860616734 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/utils.py @@ -6,6 +6,7 @@ import contextlib from collections.abc import Iterator from typing import Any +import regex as re import zmq from vllm.platforms import current_platform @@ -55,3 +56,13 @@ def get_representative_spec_type(spec: KVCacheSpec) -> type[KVCacheSpec]: inner = next(iter(spec.kv_cache_specs.values())) return type(inner) return type(spec) + + +# Trailing 8-hex randomization suffix appended by +# ``input_processor.assign_request_id`` as ``-{random_uuid():.8}``. +_RANDOM_SUFFIX_RE = re.compile(r"-[0-9a-f]{8}$", re.IGNORECASE) + + +def get_base_request_id(request_id: str) -> str: + """Strip the per-request ``-<8 hex>`` randomization suffix, if present.""" + return _RANDOM_SUFFIX_RE.sub("", request_id) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index 213a3b0314..66ad155bda 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -1,2641 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Worker-side logic for the NIXL connector.""" +"""Backward-compatible re-export of NixlPullConnectorWorker.""" -import logging -import os -import queue -import threading -import time -import uuid -from collections import defaultdict -from collections.abc import Iterator -from concurrent.futures import Future, ThreadPoolExecutor -from typing import TYPE_CHECKING, Any, cast - -import msgspec -import numpy as np -import torch -import zmq - -from vllm.distributed.kv_transfer.kv_connector.utils import ( - BlockIds, - EngineId, - EngineTransferInfo, - TransferTopology, - get_current_attn_backends, - kv_postprocess_blksize_and_layout_on_receive, - kv_postprocess_blksize_on_receive, - kv_postprocess_layout_on_receive, +from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_worker import ( + NixlPullConnectorWorker, ) -from vllm.distributed.kv_transfer.kv_connector.v1.base import CopyBlocksOp -from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import ( - GET_META_MSG, - NixlAgentMetadata, - NixlConnectorMetadata, - NixlHandshakePayload, - ReqId, - ReqMeta, - TransferHandle, - compute_nixl_compatibility_hash, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import ( - NixlKVConnectorStats, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import ( - ReadSpec, - TPMapping, - _is_attention_spec, - _is_ssm_spec, - compute_tp_mapping, -) -from vllm.distributed.kv_transfer.kv_connector.v1.nixl.utils import ( - _NIXL_SUPPORTED_DEVICE, - get_representative_spec_type, - zmq_ctx, -) -from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( - MambaConvSplitInfo, - derive_mamba_conv_split, -) -from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config -from vllm.distributed.parallel_state import ( - get_tensor_model_parallel_rank, - get_tensor_model_parallel_world_size, -) -from vllm.logger import init_logger -from vllm.platforms import current_platform -from vllm.utils.network_utils import make_zmq_path -from vllm.v1.attention.backends.utils import get_kv_cache_layout -from vllm.v1.kv_cache_interface import ( - FullAttentionSpec, - MambaSpec, - MLAAttentionSpec, - UniformTypeKVCacheSpecs, -) -from vllm.v1.worker.block_table import BlockTable -from vllm.v1.worker.utils import select_common_block_size -if TYPE_CHECKING: - from vllm.config import VllmConfig - from vllm.v1.kv_cache_interface import KVCacheConfig +# Backward compatibility: NixlConnectorWorker is the pull-based worker. +NixlConnectorWorker = NixlPullConnectorWorker -logger = init_logger(__name__) - -class NixlConnectorWorker: - """Implementation of Worker side methods""" - - def _compute_desc_ids( - self, - block_ids: BlockIds, - dst_num_blocks: int, - block_size_ratio: float | None, - physical_blocks_per_logical: int, - ) -> np.ndarray: - """Compute NIXL descriptor IDs for given block IDs.""" - num_fa_regions = self.num_regions - num_ssm_regions = len(self.block_len_per_layer) * 4 if self._has_mamba else 0 - - num_blocks = dst_num_blocks - if block_size_ratio is not None: - num_blocks = int(num_blocks * block_size_ratio) - num_fa_descs = num_fa_regions * num_blocks - - # All-attention fast path: single vectorized broadcast. - if num_ssm_regions == 0: - # NOTE (NickLucche) With HMA, every kv group has the same number of layers - # and layers from different groups share the same kv tensor. - # eg block_ids=[[1, 2], [3]]->blocks [1, 2] need to be - # read across all regions, same for [3], but group0-group1 blocks will - # always differ (different areas). Therefore we can just flatten the - # block_ids and compute the descs ids for all groups at once. - block_arr = np.concatenate(block_ids)[None, :] - region_ids = np.arange(num_fa_regions)[:, None] - return (region_ids * num_blocks + block_arr).flatten() - - # Compute desc ids per group using the right stride: FA descs have - # num_blocks entries per region (kernel granularity), SSM descs have - # logical_blocks entries per region (no kernel splitting). - logical_blocks = num_blocks // physical_blocks_per_logical - all_descs: list[np.ndarray] = [] - for i, group in enumerate(block_ids): - group_arr = np.asarray(group) - if _is_attention_spec(self._group_spec_types[i]): - fa_region_ids = np.arange(num_fa_regions)[:, None] - all_descs.append( - (fa_region_ids * num_blocks + group_arr[None, :]).flatten() - ) - elif _is_ssm_spec(self._group_spec_types[i]): - # NOTE (NickLucche) SSM and Attention block regions can - # be exchanged arbitrarily by manager. Therefore, descs - # are laid out as: - # [descs_fa (all regions) | descs_ssm (all regions)]. - # num_fa_descs offset must be computed per-engine since - # P and D can have different num_blocks (and thus - # different FA desc counts). - ssm_region_ids = np.arange(num_ssm_regions)[:, None] - all_descs.append( - ( - ssm_region_ids * logical_blocks - + group_arr[None, :] - + num_fa_descs - ).flatten() - ) - else: - raise ValueError( - f"Unknown spec type {self._group_spec_types[i]} at index {i}" - ) - - return np.concatenate(all_descs) - - def _build_local_splits_from_plan( - self, - plan: TPMapping, - src_blocks_data: list[tuple[int, int, int]], - num_fa_descs: int, - ) -> Iterator[list[tuple[int, int, int]]]: - """Build split handle data for P_TP > D_TP scenario. - - num_fa_descs is the boundary between FA and SSM descriptors. - Split counts are derived from source_ranks_per_group lengths. - FA uses rank_to_attention_slot for the slot offset; - SSM uses the rank's positional index. - """ - fa_idx = next( - i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) - ) - fa_num_splits = len(plan.source_ranks_per_group[fa_idx]) - - has_ssm_descs = num_fa_descs < len(src_blocks_data) - ssm_idx = next( - (i for i, t in enumerate(self._group_spec_types) if _is_ssm_spec(t)), - None, - ) - ssm_num_splits = ( - len(plan.source_ranks_per_group[ssm_idx]) - if has_ssm_descs and ssm_idx is not None - else 0 - ) - - # Per-FA-descriptor replicate flag, in _build_fa_local emission order. - fa_desc_replicated = self._fa_desc_replicated(num_fa_descs) - - for p_idx, p_rank in enumerate(plan.all_source_ranks): - fa_slot = plan.rank_to_attention_slot.get(p_rank, 0) - - handle: list[tuple[int, int, int]] = [] - for j, (addr, local_len, dev) in enumerate(src_blocks_data): - if j < num_fa_descs: - if fa_desc_replicated[j]: - # REPLICATE (MLA): whole block written on every rank. - handle.append((addr, local_len, dev)) - else: - # SPLIT (full-attn): this rank's head slice. - chunk = local_len // fa_num_splits - handle.append((addr + fa_slot * chunk, chunk, dev)) - else: - chunk = local_len // ssm_num_splits - handle.append((addr + p_idx * chunk, chunk, dev)) - yield handle - - def _fa_desc_replicated(self, num_fa_descs: int) -> list[bool]: - """Per-FA-descriptor replicate flag, in _build_fa_local emission order - (region-major; K then optional V per region). Length ``num_fa_descs``. - """ - assert self.transfer_topo is not None - n_regions = len(self.block_len_per_layer) - # Unset only when the worker is built directly in unit tests; a real - # model always registers regions (no-KV-cache crashes long before here). - # Fall back to all-SPLIT to preserve the pre-per-region behavior. - if n_regions == 0 or self.num_regions == 0: - return [False] * num_fa_descs - # Descriptors (blocks) per stream; all streams share the same count. - nblk = num_fa_descs // self.num_regions - virtually_split = self.transfer_topo.virtually_split_kv_in_blocks - flags: list[bool] = [] - for i in range(n_regions): - replicated = self._is_region_replicated(i) - # REPLICATE (MLA) is key-only -> 1 stream; SPLIT emits K and V - # (2 streams) under the virtually-split layout. - num_streams = 1 if replicated or not virtually_split else 2 - flags.extend([replicated] * (num_streams * nblk)) - assert len(flags) == num_fa_descs, ( - f"FA desc flags {len(flags)} != num_fa_descs {num_fa_descs}" - ) - return flags - - def _is_region_replicated(self, region_idx: int) -> bool: - """Whether region ``region_idx`` is transferred REPLICATE vs SPLIT. - - REPLICATE (MLA): identical on every rank, whole block read from one - rank at offset 0, key-only. SPLIT (full-attn): head-sharded across TP. - Defaults to SPLIT when the per-region map is unset (e.g. tests that set - block_len_per_layer without register_kv_caches). - """ - return region_idx < len(self._region_is_mla) and self._region_is_mla[region_idx] - - def __init__( - self, - vllm_config: "VllmConfig", - engine_id: str, - kv_cache_config: "KVCacheConfig", - ): - nixl_wrapper_cls = NixlWrapper - if nixl_wrapper_cls is None: - logger.error("NIXL is not available") - raise RuntimeError("NIXL is not available") - logger.info("Initializing NIXL wrapper") - logger.info("Initializing NIXL worker %s", engine_id) - - # Config. - self.vllm_config = vllm_config - # mypy will complain on re-assignment otherwise. - self.block_size: int = cast(int, vllm_config.cache_config.block_size) - - if vllm_config.kv_transfer_config is None: - raise ValueError("kv_transfer_config must be set for NixlConnector") - self.kv_transfer_config = vllm_config.kv_transfer_config - - self.nixl_backends = vllm_config.kv_transfer_config.get_from_extra_config( - "backends", ["UCX"] - ) - kv_lease_duration: int = vllm_config.kv_transfer_config.get_from_extra_config( - "kv_lease_duration", 30 - ) - # NOTE (NickLucche): For now we use a hardcoded value for a simpler interface. - self._lease_extension = kv_lease_duration * 2 // 3 - - self._is_hma_required = ( - not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager - and any( - not isinstance(g.kv_cache_spec, FullAttentionSpec) - for g in kv_cache_config.kv_cache_groups - ) - ) - self.kv_cache_config = kv_cache_config - self._layer_specs = { - layer: group.kv_cache_spec - for group in kv_cache_config.kv_cache_groups - for layer in group.layer_names - } - self.hma_group_size = len(kv_cache_config.kv_cache_tensors) - - # ---- Model state (derived from model config) ---- - mamba_ssm_size = (0, 0) - # Conv state sub-projection decomposition (None when no Mamba). - # The 3-read transfer requires DS (dim, state_len) conv layout so - # that x/B/C sub-projections are contiguous in memory. - self._conv_decomp: MambaConvSplitInfo | None = None - self._has_mamba = any( - isinstance(g.kv_cache_spec, MambaSpec) - for g in kv_cache_config.kv_cache_groups - ) - if self._has_mamba: - assert self._is_hma_required - from vllm.model_executor.layers.mamba.mamba_utils import ( - is_conv_state_dim_first, - ) - - assert is_conv_state_dim_first(), ( - "3-read Mamba conv transfer requires DS conv state layout. " - "Set VLLM_SSM_CONV_STATE_LAYOUT=DS" - ) - mamba_spec = next( - spec - for spec in self._layer_specs.values() - if isinstance(spec, MambaSpec) - ) - self._conv_decomp = derive_mamba_conv_split( - mamba_spec, - vllm_config.parallel_config.tensor_parallel_size, - ) - mamba_ssm_size = self._conv_decomp.ssm_sizes - self._mamba_ssm_size = mamba_ssm_size - - # Agent. - non_ucx_backends = [b for b in self.nixl_backends if b != "UCX"] - # Configure NIXL num_threads to avoid UAR exhaustion on Mellanox NICs. - # Each UCX thread allocates UARs (doorbell pages) via DevX, and - # excessive NIXL UAR usage can exhaust NIC UAR space. This can cause - # components like NVSHMEM (used by DeepEP kernels) to fail during RDMA - # initialization with "mlx5dv_devx_alloc_uar" errors. - # Ref: https://network.nvidia.com/files/doc-2020/ethernet-adapters-programming-manual.pdf#page=63 - num_threads = vllm_config.kv_transfer_config.get_from_extra_config( - "num_threads", 4 - ) - if nixl_agent_config is None: - config = None - else: - # Enable telemetry by default for NIXL 0.7.1 and above. - config = ( - nixl_agent_config(backends=self.nixl_backends, capture_telemetry=True) - if len(non_ucx_backends) > 0 - else nixl_agent_config(num_threads=num_threads, capture_telemetry=True) - ) - - self.nixl_wrapper = nixl_wrapper_cls(str(uuid.uuid4()), config) - # Map of engine_id -> {rank0: agent_name0, rank1: agent_name1..}. - self._remote_agents: dict[EngineId, dict[int, str]] = defaultdict(dict) - - # Metadata. - self.engine_id: EngineId = engine_id - self.tp_rank = get_tensor_model_parallel_rank() - self.world_size = get_tensor_model_parallel_world_size() - - self.num_blocks = kv_cache_config.num_blocks - self.enable_permute_local_kv = False - self.enable_heterogeneous_attn_post_process = False - - # KV Caches and nixl tracking data. - self.device_type = current_platform.device_type - self.kv_buffer_device: str = vllm_config.kv_transfer_config.kv_buffer_device - if self.device_type not in _NIXL_SUPPORTED_DEVICE: - raise RuntimeError(f"{self.device_type} is not supported.") - elif self.kv_buffer_device not in _NIXL_SUPPORTED_DEVICE[self.device_type]: - raise RuntimeError( - f"{self.device_type} with {self.kv_buffer_device} kv_buffer " - "is not supported." - ) - self.device_kv_caches: dict[str, torch.Tensor] = {} - - # cpu kv buffer for xfer - # used when device memory can not be registered under nixl - self.host_xfer_buffers: dict[str, torch.Tensor] = {} - if self.device_type == "cpu": - self.use_host_buffer = False - else: - self.use_host_buffer = self.kv_buffer_device == "cpu" - - # reserve different cores for start_load_kv() from model_forward() - if self.device_type == "cpu": - numa_core_list = current_platform.discover_numa_topology() - # setup one last core in each numa for kv transfer. - rsv_cores_for_kv = [ - max(each_numa_core_list) for each_numa_core_list in numa_core_list - ] - - if rsv_cores_for_kv: - if not hasattr(os, "sched_setaffinity"): - raise NotImplementedError( - "os.sched_setaffinity is not available on this platform" - ) - os.sched_setaffinity(0, rsv_cores_for_kv) - - # support for oot platform which can't register nixl memory - # type based on kv_buffer_device - nixl_memory_type = current_platform.get_nixl_memory_type() - if nixl_memory_type is None: - if self.kv_buffer_device in ["cuda", "xpu"]: - nixl_memory_type = "VRAM" - elif self.kv_buffer_device == "cpu": - nixl_memory_type = "DRAM" - if nixl_memory_type is None: - raise RuntimeError( - f"{self.device_type} with {self.kv_buffer_device} kv_buffer " - "is not supported." - ) - self.nixl_memory_type = nixl_memory_type - - # Note: host xfer buffer ops when use_host_buffer is True - self.copy_blocks: CopyBlocksOp | None = None - - # Map of engine_id -> kv_caches_base_addr. For TP case, each local - self.device_id: int = 0 - # Current rank may pull from multiple remote TP workers. - # EngineId, dict[int, list[int]] -> engine_id, tp_rank, base_addr_for_layer - self.kv_caches_base_addr = defaultdict[EngineId, dict[int, list[int]]](dict) - - # Number of NIXL regions. Currently one region per cache - # (so 1 per layer for MLA, otherwise 2 per layer) - self.num_regions = 0 - - # nixl_prepped_dlist_handle. - self.src_xfer_handles_by_block_size: dict[int, int] = {} - # Populated dynamically during handshake based on remote configuration. - # Keep track of regions at different tp_ratio values. tp_ratio->handles - self.src_xfer_handles_by_tp_ratio: dict[int, list[int]] = {} - # Map of engine_id -> {tp_rank: nixl_prepped_dlist_handle (int)}. - self.dst_xfer_side_handles = defaultdict[EngineId, dict[int, int]](dict) - - # Map of engine_id -> num_blocks. All ranks in the same deployment will - # have the same number of blocks. - self.dst_num_blocks: dict[EngineId, int] = {} - self._registered_descs: list[Any] = [] - - # In progress transfers. - # [req_id -> list[handle]] - self._recving_metadata: dict[ReqId, ReqMeta] = {} - self._recving_transfers = defaultdict[ReqId, list[TransferHandle]](list) - # Track the expiration time of requests that are waiting to be sent. - self._reqs_to_send: dict[ReqId, float] = {} - # Set of requests that have been part of a batch, regardless of status. - self._reqs_to_process: set[ReqId] = set() - - # Invalid blocks from failed NIXL operations (thread-safe queue of block ids) - self._invalid_block_ids: queue.Queue[set[int]] = queue.Queue() - # requests that skipped transfer (handshake or transfer failures) - # Uses Queue for thread-safe cross-thread coordination with the - # background handshake thread, matching the _ready_requests pattern. - self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue() - - # Handshake metadata of this worker for NIXL transfers. - self.xfer_handshake_metadata: NixlHandshakePayload | None = None - # Background thread for initializing new NIXL handshakes. - self._handshake_initiation_executor = ThreadPoolExecutor( - # NIXL is not guaranteed to be thread-safe, limit 1 worker. - max_workers=1, - thread_name_prefix="vllm-nixl-handshake-initiator", - ) - self._ready_requests = queue.Queue[tuple[ReqId, ReqMeta]]() - self._handshake_futures: dict[EngineId, Future[dict[int, str]]] = {} - # Protects _handshake_futures and _remote_agents. - self._handshake_lock = threading.RLock() - - # TTL-based eviction of stale remote engine state. - self._engine_last_active: dict[EngineId, float] = {} - self._engine_ttl: float = vllm_config.kv_transfer_config.get_from_extra_config( - "engine_ttl", 3600.0 - ) - - self.block_size = vllm_config.cache_config.block_size - self.model_config = vllm_config.model_config - - self.use_mla = self.model_config.use_mla - - # Get the attention backend from the first layer - # NOTE (NickLucche) models with multiple backends are not supported yet - self.attn_backends = get_current_attn_backends(vllm_config) - self.backend_name = self.attn_backends[0].get_name() - - self.kv_cache_layout = get_kv_cache_layout() - self.host_buffer_kv_cache_layout = self.kv_cache_layout - logger.info( - "Detected attention backend(s) %s", - [backend.get_name() for backend in self.attn_backends], - ) - logger.info("Detected kv cache layout %s", self.kv_cache_layout) - - # lazy initialized in register_kv_caches - self.compat_hash: str | None = None - self.transfer_topo: TransferTopology | None = None - - # With heterogeneous TP, P must wait for all assigned D TP workers to - # finish reading before safely freeing the blocks. - self.consumer_notification_counts_by_req = defaultdict[ReqId, int](int) - self.xfer_stats = NixlKVConnectorStats() - - self._physical_blocks_per_logical_kv_block = 1 - self._sync_block_size_with_kernel() - - # Unwrap UniformTypeKVCacheSpecs to get the representative spec type - self._group_spec_types = tuple( - get_representative_spec_type(g.kv_cache_spec) - for g in self.kv_cache_config.kv_cache_groups - ) - - # Per-region MLA flag, 1:1 with block_len_per_layer. True -> REPLICATE - # (MLA), False -> SPLIT (head-sharded full-attn). Mixed only for models - # combining both (e.g. GQA main + MLA Eagle-3 draft). - self._region_is_mla = list[bool]() - - # Enable different block lengths for different layers *only* when MLA is used. - # This is not used for SSM layers, which use the counterpart `mamba_ssm_size`. - self.block_len_per_layer = list[int]() - - # Per-engine TP mappings. Generated during handshake. - self.tp_mappings: dict[EngineId, TPMapping] = {} - - self.enforce_compat_hash = self.kv_transfer_config.get_from_extra_config( - "enforce_handshake_compat", True - ) - - def _sync_block_size_with_kernel(self) -> None: - backends = get_current_attn_backends(self.vllm_config) - kernel_block_size = select_common_block_size(self.block_size, backends) - # Number of blocks not accounting for kernel block mismatches - self._logical_num_blocks = self.num_blocks - if self.block_size != kernel_block_size: - logger.info_once( - "User-specified logical block size (%s) does not match" - " physical kernel block size (%s). Using the latter.", - self.block_size, - kernel_block_size, - ) - assert self.block_size > kernel_block_size - self._physical_blocks_per_logical_kv_block = ( - self.block_size // kernel_block_size - ) - self.block_size = kernel_block_size - self.num_blocks *= self._physical_blocks_per_logical_kv_block - - def _nixl_handshake( - self, - host: str, - port: int, - remote_tp_size: int, - expected_engine_id: str, - ) -> dict[int, str]: - """Do a NIXL handshake with a remote instance.""" - - # the first time we connect to a remote agent. - # be careful, the handshake happens in a background thread. - # it does not have an active cuda context until any cuda runtime - # call is made. when UCX fails to find a valid cuda context, it will - # disable any cuda ipc communication, essentially disabling any NVLink - # communication. - # when we are using device buffers, we need to set the device - # explicitly to make sure the handshake background thread has a valid - # cuda context. - if not self.use_host_buffer: - current_platform.set_device(self.device_id) - - # When target instance TP > local TP, we need to perform multiple - # handshakes. Do it in a single background job for simplicity. - # Regardless, only handshake with the remote TP rank(s) that current - # local rank will read from. Note that With homogeneous TP, - # this happens to be the same single rank_i. - assert self.transfer_topo is not None - p_remote_ranks = self.transfer_topo.handshake_target_ranks(remote_tp_size) - remote_rank_to_agent_name = {} - path = make_zmq_path("tcp", host, port) - - with zmq_ctx(zmq.REQ, path) as sock: - for remote_rank in p_remote_ranks: - logger.debug( - "Querying metadata on path: %s at remote tp rank %s", - path, - remote_rank, - ) - - start_time = time.perf_counter() - # Send query for the request. - msg = msgspec.msgpack.encode((GET_META_MSG, remote_rank)) - # Set receive timeout to 5 seconds to avoid hanging on dead server - sock.setsockopt(zmq.RCVTIMEO, 5000) # milliseconds - sock.send(msg) - handshake_bytes = sock.recv() - - # Decode handshake payload to get compatibility hash - handshake_decoder = msgspec.msgpack.Decoder(NixlHandshakePayload) - try: - handshake_payload = handshake_decoder.decode(handshake_bytes) - except (msgspec.DecodeError, msgspec.ValidationError) as e: - raise RuntimeError( - f"Failed to decode NixlHandshakePayload. This likely indicates " - f"an incompatibility between connector version. Error: {e}" - ) from e - - got_metadata_time = time.perf_counter() - logger.debug( - "NIXL handshake: get metadata took: %s", - got_metadata_time - start_time, - ) - - # Check compatibility hash BEFORE decoding agent metadata - assert self.compat_hash is not None - if ( - self.enforce_compat_hash - and handshake_payload.compatibility_hash != self.compat_hash - ): - raise RuntimeError( - f"NIXL compatibility hash mismatch. " - f"Local: {self.compat_hash}, " - f"Remote: {handshake_payload.compatibility_hash}. " - f"Prefill and decode instances have incompatible " - f"configurations. This may be due to: different vLLM versions," - f" models, dtypes, KV cache layouts, attention backends, etc. " - f"Both instances must use identical configurations." - f"Disable this check using " - f'--kv-transfer-config \'{{"kv_connector_extra_config": ' - f'{{"enforce_handshake_compat": false}}}}\'' - ) - - logger.info( - "NIXL compatibility check passed (hash: %s)", - handshake_payload.compatibility_hash, - ) - - # Decode agent metadata - metadata_decoder = msgspec.msgpack.Decoder(NixlAgentMetadata) - try: - metadata = metadata_decoder.decode( - handshake_payload.agent_metadata_bytes - ) - except (msgspec.DecodeError, msgspec.ValidationError) as e: - # This should not happen if hash matched - raise RuntimeError( - f"Failed to decode NixlAgentMetadata. Error: {e}" - ) from e - - # Ensure engine id matches. - if metadata.engine_id != expected_engine_id: - raise RuntimeError( - f"Remote NIXL agent engine ID mismatch. " - f"Expected {expected_engine_id}," - f"received {metadata.engine_id}." - ) - - # Register Remote agent. - remote_agent_name = self.add_remote_agent( - metadata, remote_rank, remote_tp_size - ) - setup_agent_time = time.perf_counter() - logger.debug( - "NIXL handshake: add agent took: %s", - setup_agent_time - got_metadata_time, - ) - remote_rank_to_agent_name[remote_rank] = remote_agent_name - return remote_rank_to_agent_name - - def initialize_host_xfer_buffer(self, kv_caches: dict[str, torch.Tensor]) -> None: - """ - Initialize transfer buffer in CPU mem for accelerators - NOT directly supported by NIXL (e.g., tpu) - """ - xfer_buffers: dict[str, torch.Tensor] = {} - inv_order = [0, 1, 3, 2, 4] - try: - for layer_name, kv_cache in kv_caches.items(): - kv_shape = kv_cache.shape - kv_dtype = kv_cache.dtype - permute_shape = False - if ( - self.kv_cache_layout == "NHD" - and self.vllm_config.kv_transfer_config is not None - and self.vllm_config.kv_transfer_config.enable_permute_local_kv - ): - logger.info_once( - "'enable_permute_local_kv' flag is enabled while " - "device KV Layout is NHD. Init host buffer with" - " HND to better support Decode/Prefill TP_ratio > 1." - ) - # Since NHD will not support Decode/Prefill TP_ratio > 1, - # we can leverage host_buffer for permute - self.host_buffer_kv_cache_layout = "HND" - kv_shape = ( - tuple(kv_shape[i] for i in inv_order) - if not self.use_mla - else kv_shape - ) - permute_shape = not self.use_mla - - xfer_buffers[layer_name] = torch.empty( - kv_shape, dtype=kv_dtype, device="cpu" - ) - if permute_shape: - xfer_buffers[layer_name] = xfer_buffers[layer_name].permute( - inv_order - ) - except MemoryError as e: - logger.error("NIXLConnectorWorker gets %s.", e) - raise - - self.host_xfer_buffers = xfer_buffers - - def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp): - """Assign copy (d2h, h2d) operations when host buffer is used.""" - # Set a no-op if the host buffer is not cpu. - if self.kv_buffer_device != "cpu": - return - # Set a no-op if self.device_type is 'cpu'. - if self.device_type == "cpu": - return - assert self.use_host_buffer - self.copy_blocks = copy_operation - - def _log_failure( - self, - failure_type: str, - req_id: str | None, - msg: str = "", - error: Exception | None = None, - meta: ReqMeta | None = None, - **extra_context, - ): - """Log transfer failure with structured context for easier debugging.""" - context: dict[str, Any] = { - "failure_type": failure_type, - "request_id": req_id, - "engine_id": self.engine_id, - } - if meta is None and req_id is not None: - # Try to get metadata from in progress transfers when not provided - meta = self._recving_metadata.get(req_id) - - if meta and meta.remote: - context.update( - { - "remote_engine_id": meta.remote.engine_id, - "remote_request_id": meta.remote.request_id, - "remote_host": meta.remote.host, - "remote_port": meta.remote.port, - "num_local_blocks": sum( - len(group) for group in meta.local_block_ids - ), - "num_remote_blocks": sum( - len(group) for group in meta.remote.block_ids - ), - "local_block_ids_sample": meta.local_block_ids[0][:10] - if meta.local_block_ids - else [], - } - ) - - context.update(extra_context) - if msg: - failure_type = f"{failure_type}. {msg}" - - logger.error( - "NIXL transfer failure: %s | Context: %s", - failure_type, - context, - exc_info=error is not None, - stacklevel=2, - ) - - def _ensure_handshake( - self, - engine_id: EngineId, - host: str, - port: int, - tp_size: int, - ) -> Future[dict[int, str]] | None: - """ - Ensure a handshake is in-flight (or already done) for *engine_id*. - - Returns the ``Future`` if a handshake is pending (or was just - started), or ``None`` if the handshake already completed - successfully. Callers can attach per-request callbacks to the - returned future. - Failures to handshake are logged and the request is marked as failed. - """ - self._evict_stale_engines() - with self._handshake_lock: - if engine_id in self._remote_agents: - return None - fut = self._handshake_futures.get(engine_id) - if fut is not None: - return fut - fut = self._handshake_initiation_executor.submit( - self._nixl_handshake, - host, - port, - tp_size, - engine_id, - ) - self._handshake_futures[engine_id] = fut - - def done_callback(f: Future[dict[int, str]], eid=engine_id): - with self._handshake_lock: - del self._handshake_futures[eid] - try: - self._remote_agents[eid] = f.result() - self._engine_last_active[eid] = time.perf_counter() - except Exception as e: - self._log_failure( - failure_type="handshake_setup_failed", - req_id=None, - error=e, - remote_engine_id=eid, - ) - - fut.add_done_callback(done_callback) - return fut - - def _background_nixl_handshake( - self, req_id: str, remote_engine_id: EngineId, meta: ReqMeta - ): - # Do NIXL handshake in background and add to _ready_requests when done. - assert meta.remote is not None - fut = self._ensure_handshake( - remote_engine_id, - meta.remote.host, - meta.remote.port, - meta.tp_size, - ) - if fut is None: - # Already handshaked — only happens if caller does not pre-check. - self._ready_requests.put((req_id, meta)) - return - - # Check handshake success before proceeding with request. - def request_ready(f: Future[Any], entry=(req_id, meta)): - try: - f.result() - self._ready_requests.put(entry) - except Exception as e: - self._log_failure( - failure_type="handshake_failed", - req_id=req_id, - error=e, - meta=meta, - ) - self._handle_failed_transfer(req_id, None) - - fut.add_done_callback(request_ready) - - def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None: - """Register a cross-layers KV cache tensor with NIXL. - - `use_uniform_kv_cache()` guarantees a single KV cache group whose - layers all share the same `AttentionSpec`, so any layer name from - `_layer_specs` yields the correct per-layer spec for `page_size_bytes`. - """ - first_layer = next(iter(self._layer_specs)) - # Forwarding a real layer name rather than a synthetic key - self.register_kv_caches({first_layer: kv_cache}) - - def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]): - """Register the KV Cache data in nixl.""" - self.transfer_topo = TransferTopology( - tp_rank=self.tp_rank, - tp_size=self.world_size, - block_size=self.block_size, - engine_id=self.engine_id, - is_mla=self.use_mla, - total_num_kv_heads=self.model_config.get_total_num_kv_heads(), - attn_backends=self.attn_backends, - # SSM States come in tuples (ssm, conv) - tensor_shape=next(iter(kv_caches.values())).shape - if not self._has_mamba - else None, - is_mamba=self._has_mamba, - ) - self.compat_hash = compute_nixl_compatibility_hash( - self.vllm_config, self.backend_name, self.transfer_topo.cross_layers_blocks - ) - - if self.use_host_buffer: - self.initialize_host_xfer_buffer(kv_caches=kv_caches) - assert len(self.host_xfer_buffers) == len(kv_caches), ( - f"host_buffer: {len(self.host_xfer_buffers)}, " - f"kv_caches: {len(kv_caches)}" - ) - xfer_buffers = self.host_xfer_buffers - else: - xfer_buffers = kv_caches - assert not self.host_xfer_buffers, ( - "host_xfer_buffer should not be initialized when " - f"kv_buffer_device is {self.kv_buffer_device}" - ) - - logger.info( - "Registering KV_Caches. use_mla: %s, kv_buffer_device: %s, " - "use_host_buffer: %s", - self.use_mla, - self.kv_buffer_device, - self.use_host_buffer, - ) - - caches_data = [] - # With hybrid allocator, layers can share a kv cache tensor - seen_base_addresses = [] - - # Note(tms): I modified this from the original region setup code. - # K and V are now in different regions. Advantage is that we can - # elegantly support MLA and any cases where the K and V tensors - # are non-contiguous (it's not locally guaranteed that they will be) - # Disadvantage is that the encoded NixlAgentMetadata is now larger - # (roughly 8KB vs 5KB). - # Conversely for FlashInfer, K and V are registered in the same region - # to better exploit the memory layout (ie num_blocks is the first dim). - tensor_size_bytes = None - - for layer_name, cache_or_caches in xfer_buffers.items(): - # NOTE (NickLucche) Hybrid SSM models assume a layout that is similar to - # that of FI, with block laid out as in `get_backend_aware_kv_block_len`. - # However, physical page_size may differ when kernel requires a specific - # block size. This leads to SSM and FA layers having different num_blocks. - # `_physical_blocks_per_logical_kv_block` ratio is used to adjust for this. - layer_spec = self._layer_specs.get(layer_name) - if layer_spec is None: - logger.debug( - "Skipping layer %s as no KVCache spec is present. " - "This is likely because the layer is sharing its KV cache", - layer_name, - ) - continue - if isinstance(layer_spec, UniformTypeKVCacheSpecs): - # MLA DSv32 Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs - layer_spec = layer_spec.kv_cache_specs[layer_name] - cache_list = self.transfer_topo.get_transfer_cache_regions( - cache_or_caches, layer_spec - ) - # `layer_spec.page_size_bytes` only accounts for logical page_size, that is - # the page_size assuming constant `self._logical_num_blocks`. - physical_page_size = ( - layer_spec.page_size_bytes - if isinstance(layer_spec, MambaSpec) - else layer_spec.page_size_bytes - // self._physical_blocks_per_logical_kv_block - ) - # For when registering multiple tensors eg K/V in separate regions. - physical_page_size = physical_page_size // len(cache_list) - if self.transfer_topo._cross_layers_blocks: - # When cross-layers blocks are used, multiply by number of layers - physical_page_size = physical_page_size * len( - self.kv_cache_config.kv_cache_tensors - ) - num_blocks = ( - self._logical_num_blocks - if isinstance(layer_spec, MambaSpec) - else self.num_blocks - ) - # `page_size` accounts for physical blocks, st KVCache is always - # [`num_blocks` * `page_size`] - curr_tensor_size_bytes = num_blocks * physical_page_size - - # TODO (NickLucche) we could eventually unify how we handle FA/FI regions, - # registering a single tensor for both K/V and splitting logically like FI. - for cache in cache_list: - base_addr = cache.data_ptr() - if base_addr in seen_base_addresses: - # NOTE (NickLucche) HMA employs memory pooling to share tensors - # across groups. This results in skipping all tensors but the ones - # pointed to by group0. Also, generally we will have more blocks - # per tensor but fewer regions. - logger.debug("Skipping %s because it's already seen", layer_name) - continue - logger.debug( - "Registering layer %s with cache shape: %s", layer_name, cache.shape - ) - seen_base_addresses.append(base_addr) - # Only record non-Mamba page sizes. - if isinstance(layer_spec, MambaSpec): - self.block_len_per_layer.append( - physical_page_size // self._physical_blocks_per_logical_kv_block - ) - else: - self.block_len_per_layer.append(physical_page_size) - is_mla_region = isinstance(layer_spec, MLAAttentionSpec) - self._region_is_mla.append(is_mla_region) - - # HeteroTP cannot transfer differently-sized regions, so every - # non-MLA region in a group must share one tensor size (this also - # holds for Mamba-like models). The sole exception is the DeepSeek - # MLA indexer, which sits in a UniformTypeKVCacheSpecs group at a - # different size; MLA regions are therefore exempt. - if not is_mla_region: - if tensor_size_bytes is None: - tensor_size_bytes = curr_tensor_size_bytes - assert tensor_size_bytes == curr_tensor_size_bytes, ( - "All non-MLA kv cache tensors must have the same size" - ) - - if cache.shape[0] != num_blocks: - raise AssertionError( - "All kv cache tensors must have the same number of " - f"blocks; layer={layer_name}, " - f"expected_num_blocks={num_blocks}, " - f"cache_shape={tuple(cache.shape)}, " - f"cache_stride={tuple(cache.stride())}, " - f"layer_spec={type(layer_spec).__name__}, " - f"backend={self.backend_name}, " - "all_backends=" - f"{[backend.get_name() for backend in self.attn_backends]}, " - f"kv_cache_layout={self.kv_cache_layout}, " - "blocks_first=" - f"{self.transfer_topo.is_kv_layout_blocks_first}" - ) - - # Need to make sure the device ID is non-negative for NIXL, - # Torch uses -1 to indicate CPU tensors. - self.device_id = max(cache.get_device(), 0) - caches_data.append( - (base_addr, curr_tensor_size_bytes, self.device_id, "") - ) - - logger.debug( - "Different block lengths collected: %s", set(self.block_len_per_layer) - ) - assert ( - len(self.block_len_per_layer) - == len(seen_base_addresses) - == len(self._region_is_mla) - ) - - self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses - self.num_regions = len(caches_data) - - if self.transfer_topo.virtually_split_kv_in_blocks: - # NOTE (NickLucche) When FlashInfer is used, memory is registered - # with joint KV for each block. This minimizes the overhead in - # registerMem allowing faster descs queries. In order to be able to - # split on kv_heads dim as required by heterogeneous TP, one must - # be able to index K/V separately. Hence we double the number - # of 'virtual' regions here and halve `block_len` below. - # Similarly for Mamba layers, we register SSM+Conv as a single region and - # then duplicate it logically to be able to index SSM/Conv separately. - # Exception: key-only REPLICATE regions (MLA) have no V half, so - # they contribute a single desc stream and are not doubled. - self.num_regions = sum( - 1 if self._is_region_replicated(i) else 2 - for i in range(len(self._region_is_mla)) - ) - - # Total local FA descriptors (boundary between FA and mamba descs). - self.num_descs = self.num_regions * self.num_blocks - - descs = self.nixl_wrapper.get_reg_descs(caches_data, self.nixl_memory_type) - logger.debug("Registering descs: %s", caches_data) - self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends) - logger.debug("Done registering descs") - self._registered_descs.append(descs) - - self.device_kv_caches = kv_caches - self.dst_num_blocks[self.engine_id] = self.num_blocks - - if self._has_mamba: - logger.info( - "Hybrid SSM registration: num_blocks=%s, " - "logical_num_blocks=%s, ratio=%s, num_regions=%s, " - "num_descs=%s, mamba_ssm_size=%s, block_len_per_layer=%s", - self.num_blocks, - self._logical_num_blocks, - self._physical_blocks_per_logical_kv_block, - self.num_regions, - self.num_descs, - self._mamba_ssm_size, - set(self.block_len_per_layer), - ) - - # Register local/src descr for NIXL xfer. - self.src_xfer_handles_by_block_size[self.block_size], self.src_blocks_data = ( - self.register_local_xfer_handler(self.block_size) - ) - - # After KV Caches registered, listen for new connections. - agent_metadata = NixlAgentMetadata( - engine_id=self.engine_id, - agent_metadata=self.nixl_wrapper.get_agent_metadata(), - device_id=self.device_id, - kv_caches_base_addr=self.kv_caches_base_addr[self.engine_id][self.tp_rank], - num_blocks=self.num_blocks, - block_lens=self.block_len_per_layer, - kv_cache_layout=self.kv_cache_layout - if not self.use_host_buffer - else self.host_buffer_kv_cache_layout, - block_size=self.block_size, - ssm_sizes=self._mamba_ssm_size, - attn_backend_name=self.backend_name, - physical_blocks_per_logical_kv_block=( - self._physical_blocks_per_logical_kv_block - ), - ) - # Wrap metadata in payload with hash for defensive decoding - assert self.compat_hash is not None - encoder = msgspec.msgpack.Encoder() - self.xfer_handshake_metadata = NixlHandshakePayload( - compatibility_hash=self.compat_hash, - agent_metadata_bytes=encoder.encode(agent_metadata), - ) - - def _build_mamba_local( - self, - base_addresses: list[int], - block_size_ratio: int, - ) -> list[tuple[int, int, int]]: - """Build 4 desc regions (x, B, C, ssm) per layer for local mamba - blocks, enabling the 3-read transfer with DS conv layout.""" - assert block_size_ratio == 1, ( - "Mamba 3-read transfer with block_size_ratio != 1 is not tested. " - f"Got block_size_ratio={block_size_ratio}." - ) - assert self._conv_decomp is not None - conv_offsets = self._conv_decomp.local_conv_offsets - conv_size, ssm_size = self._mamba_ssm_size - num_blocks = self._logical_num_blocks * block_size_ratio - physical_per_logical = self._physical_blocks_per_logical_kv_block - - result: list[tuple[int, int, int]] = [] - for i, base_addr in enumerate(base_addresses): - # Jump one page_size, but ssm page_size may be bigger when kernel - # locks block size to a specific value (physical_per_logical scale). - page_stride = ( - self.block_len_per_layer[i] // block_size_ratio * physical_per_logical - ) - for off, sz in conv_offsets: - for blk in range(num_blocks): - result.append( - (base_addr + blk * page_stride + off, sz, self.device_id) - ) - # SSM temporal state follows the conv state. - for blk in range(num_blocks): - result.append( - ( - base_addr + blk * page_stride + conv_size, - ssm_size, - self.device_id, - ) - ) - return result - - def _build_mamba_remote( - self, - nixl_agent_meta: NixlAgentMetadata, - tp_ratio: int, - transfer_info: EngineTransferInfo, - ) -> list[tuple[int, int, int]]: - """Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer - for the 3-read transfer. For hetero-TP, each D rank reads only its - sub-projection slice from the P rank.""" - assert self._conv_decomp is not None - effective_ratio = max(tp_ratio, 1) - # Mamba conv state is always TP-sharded, even when attention KV - # is replicated (num_kv_heads < tp_size). - local_offset = self.tp_rank % effective_ratio - conv_size_remote = nixl_agent_meta.ssm_sizes[0] - - conv_offsets = self._conv_decomp.remote_conv_offsets(local_offset, tp_ratio) - if tp_ratio >= 1: - ssm_read_size = self._mamba_ssm_size[1] - else: - ssm_read_size = nixl_agent_meta.ssm_sizes[1] - - remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical - num_blocks = nixl_agent_meta.num_blocks // remote_physical_per_logical - device_id = nixl_agent_meta.device_id - - result: list[tuple[int, int, int]] = [] - # NOTE (ZhanqiuHu): use per-layer block_lens[i], not [0], in case - # block lengths vary across layers (e.g. MLA). - for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): - page_stride = nixl_agent_meta.block_lens[i] * remote_physical_per_logical - for off, sz in conv_offsets: - for blk in range(num_blocks): - result.append((base_addr + blk * page_stride + off, sz, device_id)) - # SSM temporal state is also TP-sharded on the heads dimension. - for blk in range(num_blocks): - ssm_addr = ( - base_addr - + blk * page_stride - + conv_size_remote - + local_offset * ssm_read_size - ) - result.append((ssm_addr, ssm_read_size, device_id)) - return result - - def _build_fa_local( - self, - base_addresses: list[int], - block_size_ratio: int, - ) -> list[tuple[int, int, int]]: - """Build local FA descriptors for all layers.""" - assert self.transfer_topo is not None - num_blocks = self.num_blocks * block_size_ratio - result: list[tuple[int, int, int]] = [] - for i, base_addr in enumerate(base_addresses): - kv_block_len = ( - self.get_backend_aware_kv_block_len( - layer_idx=i, first_split=True, mamba_view=False - ) - // block_size_ratio - ) - page_stride = self.block_len_per_layer[i] // block_size_ratio - for block_id in range(num_blocks): - block_offset = block_id * page_stride - addr = base_addr + block_offset - result.append((addr, kv_block_len, self.device_id)) - - if ( - self.transfer_topo.virtually_split_kv_in_blocks - and not self._is_region_replicated(i) - ): - # Separate and interleave K/V regions to maintain the same - # descs ordering. This is needed for selecting contiguous heads - # when split across TP ranks. (Skipped for key-only REPLICATE.) - second_split = self.get_backend_aware_kv_block_len( - layer_idx=i, first_split=False, mamba_view=False - ) - for block_id in range(num_blocks): - block_offset = block_id * page_stride - addr = base_addr + block_offset - v_addr = addr + kv_block_len - result.append((v_addr, second_split, self.device_id)) - return result - - def _build_fa_remote( - self, - plan: TPMapping, - nixl_agent_meta: NixlAgentMetadata, - block_size_ratio: int, - ) -> list[tuple[int, int, int]]: - """Build remote FA descriptors for all layers.""" - assert self.transfer_topo is not None - fa_group_idx = next( - i for i, t in enumerate(self._group_spec_types) if _is_attention_spec(t) - ) - # SPLIT regions read their head slice from this many remote ranks at a - # per-rank offset; REPLICATE regions read the whole block once. - split_reads = len(plan.source_ranks_per_group[fa_group_idx]) - num_blocks = nixl_agent_meta.num_blocks - result: list[tuple[int, int, int]] = [] - for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr): - replicated = self._is_region_replicated(i) - # Read our whole local region size from remote.. - local_block_len = self.get_backend_aware_kv_block_len( - layer_idx=i, first_split=True, mamba_view=False - ) - remote_kv_block_len = local_block_len // block_size_ratio - if block_size_ratio > 1: - # ..using remote kv_block_len as transfer unit - local_block_len = remote_kv_block_len - - # REPLICATE reads the whole block once at offset 0; SPLIT gathers - # its head slice from `split_reads` remote ranks at a per-rank offset. - num_reads = 1 if replicated else split_reads - rank_offset = ( - 0 if replicated else plan.rank_offset_factor * remote_kv_block_len - ) - local_block_len = local_block_len // num_reads - - page_size = nixl_agent_meta.block_lens[i] - for block_id in range(num_blocks): - block_offset = block_id * page_size - # For each block, grab the kv heads chunk belonging to current local - # tp rank of size local_block_len. - addr = base_addr + block_offset + rank_offset - result.append((addr, local_block_len, nixl_agent_meta.device_id)) - - emits_v = self.transfer_topo.virtually_split_kv_in_blocks and not replicated - if emits_v: - # With FlashInfer index V separately to allow head splitting. - second_split = self.get_backend_aware_kv_block_len( - layer_idx=i, first_split=False, mamba_view=False - ) - second_split = second_split // num_reads - for block_id in range(num_blocks): - block_offset = block_id * page_size - addr = base_addr + block_offset + rank_offset - # Hop over the first split of remote page, K, to read V. - v_addr = addr + nixl_agent_meta.block_lens[i] // 2 - result.append((v_addr, second_split, nixl_agent_meta.device_id)) - return result - - def register_local_xfer_handler( - self, - block_size: int, - ) -> tuple[int, list[tuple[int, int, int]]]: - """ - Function used for register local xfer handler with local block_size or - Remote block_size. - - When local block_size is same as remote block_size, we use local block_size - to register local_xfer_handler during init. - - When remote block size is less than local block size, we need to use - register another local_xfer_handler using remote block len to ensure - data copy correctness. - """ - assert self.transfer_topo is not None - block_size_ratio = self.block_size // block_size - local_base_addresses = self.kv_caches_base_addr[self.engine_id][self.tp_rank] - - blocks_data = self._build_fa_local(local_base_addresses, block_size_ratio) - logger.debug( - "Created %s blocks for src engine %s and rank %s on device id %s", - len(blocks_data), - self.engine_id, - self.tp_rank, - self.device_id, - ) - if self._has_mamba: - assert self.num_descs == len(blocks_data) - # TODO (ZhanqiuHu): For homogeneous TP (tp_ratio == 1), the 3-descs split - # is unnecessary — a single conv desc per block suffices. Consider - # adding a fast path that falls back to the standard 2-region - # registration (_build_fa_local mamba=True) when no hetero-TP - # remote has been seen. Currently we always register 4 regions - # because local descs are created before knowing the remote TP. - logger.debug("Registering local Mamba descriptors (4 regions/layer)") - blocks_data.extend( - self._build_mamba_local(local_base_addresses, block_size_ratio) - ) - - descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type) - # NIXL_INIT_AGENT to be used for preparations of local descs. - return self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs), blocks_data - - def add_remote_agent( - self, - nixl_agent_meta: NixlAgentMetadata, - remote_tp_rank: int = 0, - remote_tp_size: int = 1, - ) -> str: - """ - Add the remote NIXL agent and prepare the descriptors for reading cache - blocks from remote. - - In particular, handle both homogeneous and heterogeneous TP. The former - requires local rank_i to read from remote rank_i. - The latter, in the case of D.world_size < P.world_size, requires that a - local (D) TP worker reads from multiple remote (P) TP workers. - Conversely, assuming D.world_size > P.world_size, two or more local TP - workers will read from a single remote TP worker. - - Here's an example for the last case described above (non-MLA): - - rank_offset p_remote_tp_rank - (kv split no) - -------------------------------- - 0 0 Worker0 ---- 1st half of KV ----> Worker0 [ KV Cache ] - / - 1 0 Worker1 ---- 2nd half of KV -----/ - - 0 1 Worker2 ---- 1st half of KV ----> Worker1 [ KV Cache ] - / - 1 1 Worker3 ---- 2nd half of KV -----/ - - - Decoder TP workers Prefix TP workers - (world_size=4) (world_size=2) - tp_ratio = 4 // 2 = 2 - - Considering the KV Caches, if P-Worker_i has cache size [2, num_blocksP, kv_heads, block_size, head_dim] - then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "HND" layout format. - Assuming num_blocksD >= num_blocksP, D-Worker0 reads from P-Worker0 by preparing the kv_heads//tp_ratio - first heads from all the slots of all the blocks. D-Worker1 will do the same, but reading the second split - along the kv_heads dimension, and so forth until "tp_ratio" D TP workers have pulled from P-Worker0. - - Note that the above will also hold true for the homogeneous TP case, where tp_ratio evaluates to 1. - - Regarding MLA case, the cache is replicated across TP workers so the rank_offset will just always be 0 - so that the whole cache is shared by "tp_ratio" D TP workers. - - For Mamba hetero-TP, both tp_ratio > 0 (D_TP > P_TP) and - tp_ratio < 0 (P_TP > D_TP) are supported by the 3-read transfer. - """ # noqa: E501 - engine_id = nixl_agent_meta.engine_id - # TODO re-evaluate refreshing for scaling/recovery - if remote_tp_rank in self._remote_agents.get(engine_id, {}): - logger.debug( - "Remote agent with engine_id %s and rank" - "%s already exchanged metadata, skip handshake.", - engine_id, - remote_tp_rank, - ) - return self._remote_agents[engine_id][remote_tp_rank] - - ### Register remote engine in TransferTopology (idempotent). - assert self.transfer_topo is not None - transfer_topo = self.transfer_topo - physical_blocks_per_logical = ( - nixl_agent_meta.physical_blocks_per_logical_kv_block - ) - transfer_info = EngineTransferInfo( - remote_tp_size=remote_tp_size, - remote_block_size=nixl_agent_meta.block_size, - remote_block_len=nixl_agent_meta.block_lens[0], - remote_physical_blocks_per_logical=physical_blocks_per_logical, - ) - transfer_topo.register_remote_engine(engine_id, transfer_info) - logger.info("Transfer plan: %s", transfer_topo.describe(engine_id)) - - self.tp_mappings[engine_id] = compute_tp_mapping( - transfer_topology=transfer_topo, - remote_tp_size=remote_tp_size, - group_spec_types=self._group_spec_types, - ) - - remote_agent_name = self.nixl_wrapper.add_remote_agent( - nixl_agent_meta.agent_metadata - ) - - # Create dst descs and xfer side handles. TP workers have same #blocks - # so we only register once per engine_id. - # Example: - # block_size_ratio > 1: - # remote: | 0| 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12| - # local origin:| 0| 1| 8| 12| - # local mapped:| 0| 1| 2| 3| 4| 5| 6| 7| 8| 9|10|11|12|13|14|15| - block_size_ratio = transfer_topo.block_size_ratio(nixl_agent_meta.block_size) - - if engine_id not in self.dst_num_blocks: - self.dst_num_blocks[engine_id] = nixl_agent_meta.num_blocks - - # Keep track of remote agent kv caches base addresses. - self.kv_caches_base_addr[engine_id][remote_tp_rank] = ( - nixl_agent_meta.kv_caches_base_addr - ) - self._validate_remote_agent_handshake(nixl_agent_meta, remote_tp_size) - - # This is 1 when P and D `--tensor-parallel-size` match. Otherwise, - # this is the ratio between the two sizes. - tp_ratio = transfer_topo.tp_ratio(remote_tp_size) - - logger.debug( - "Registering remote agent (%s, rank %s) memory regions with tp_ratio %s", - engine_id, - remote_tp_rank, - tp_ratio, - ) - - plan = self.tp_mappings[engine_id] - - ### (Optional) Register local agent memory regions. MLA is not split. - if ( - tp_ratio < 0 - and not self.use_mla - and tp_ratio not in self.src_xfer_handles_by_tp_ratio - ): - # Remote tp_size > local tp_size: read from multiple remote ranks. - # Logically "split" own regions into |tp_ratio| chunks. Mind that - # we only do this once per remote tp_size (replica-friendly). - self.src_xfer_handles_by_tp_ratio[tp_ratio] = [] - - for handle_data in self._build_local_splits_from_plan( - plan, - self.src_blocks_data, - self.num_descs, - ): - descs = self.nixl_wrapper.get_xfer_descs( - handle_data, self.nixl_memory_type - ) - handle = self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs) - self.src_xfer_handles_by_tp_ratio[tp_ratio].append(handle) - - ### Register remote agent memory regions - # With homogeneous TP, D pulls the whole kv cache from corresponding rank. With - # heterogeneous TP, prepare the descriptors by splitting the P KV cache along - # kv_head dim, of D worker's kv_head size (D>P). - # Eg. PTP1 DTP2 => P0 KV:[block0-KV_0 | block0-KV_1..]. - - # Register all remote blocks, but only the corresponding kv heads. - blocks_data = self._build_fa_remote( - plan, - nixl_agent_meta, - block_size_ratio, - ) - logger.debug( - "Created %s blocks for dst engine %s with remote rank %s and local rank %s", - len(blocks_data), - engine_id, - remote_tp_rank, - self.tp_rank, - ) - if self._has_mamba: - logger.debug( - "Registering remote Mamba blocks for engine %s rank %s", - engine_id, - remote_tp_rank, - ) - blocks_data.extend( - self._build_mamba_remote( - nixl_agent_meta, - tp_ratio, - transfer_info, - ) - ) - - # Register with NIXL. - descs = self.nixl_wrapper.get_xfer_descs(blocks_data, self.nixl_memory_type) - self.dst_xfer_side_handles[engine_id][remote_tp_rank] = ( - self.nixl_wrapper.prep_xfer_dlist(remote_agent_name, descs) - ) - - if block_size_ratio > 1: - # when prefill with smaller block_size, we need to init a - # new handler with same block_len to match - self.src_xfer_handles_by_block_size[nixl_agent_meta.block_size] = ( - self.register_local_xfer_handler(nixl_agent_meta.block_size)[0] - ) - - return remote_agent_name - - def _validate_remote_agent_handshake( - self, nixl_agent_meta: NixlAgentMetadata, remote_tp_size: int - ): - """ - Validate the remote agent handshake metadata ensuring the - invariants hold true. - """ - remote_engine_id = nixl_agent_meta.engine_id - - assert self.transfer_topo is not None - remote_info = self.transfer_topo.get_engine_info(remote_engine_id) - assert remote_info.remote_tp_size == remote_tp_size - - tp_ratio = self.transfer_topo.tp_ratio(remote_tp_size) - block_size_ratio = self.transfer_topo.block_size_ratio( - nixl_agent_meta.block_size - ) - # num_kv_heads > tp_size with P_TP > D_TP not supported for non-mamba. - # Mamba models can have replicated FA KV with tp_ratio < 0. - # MLA models do not need to handle kv replication. - if not self.use_mla and not self._has_mamba: - assert not ( - tp_ratio < 0 and self.transfer_topo.is_kv_replicated(remote_engine_id) - ) - - remote_physical_per_logical = ( - nixl_agent_meta.physical_blocks_per_logical_kv_block - ) - if ( - self._has_mamba - and remote_physical_per_logical - != self._physical_blocks_per_logical_kv_block - and self.vllm_config.cache_config.enable_prefix_caching - ): - raise RuntimeError( - "Prefix caching with heterogeneous physical_blocks_per_logical " - "is not supported for Mamba hybrid models. " - f"Local: {self._physical_blocks_per_logical_kv_block}, " - f"Remote: {remote_physical_per_logical}. " - "Disable prefix caching with --no-enable-prefix-caching." - ) - - if self._is_hma_required: - assert block_size_ratio == 1, ( - "HMA does not support different remote block size yet" - ) - kv_cache_layout = ( - self.kv_cache_layout - if not self.use_host_buffer - else self.host_buffer_kv_cache_layout - ) - if not self.use_mla and nixl_agent_meta.kv_cache_layout != kv_cache_layout: - if ( - self.kv_transfer_config.enable_permute_local_kv - and nixl_agent_meta.kv_cache_layout == "HND" - ): - logger.info( - "Remote is HND and local is NHD, enabled additional permute " - "on local device KV." - ) - assert not self._is_hma_required, ( - "HMA does not support block size post processing" - ) - self.enable_permute_local_kv = True - else: - raise RuntimeError( - "Heterogeneous TP expects same kv_cache_layout. " - "Or enable experimental feature to use HND to NHD support by " - "setting 'enable_permute_local_kv'=True in --kv-transfer-config." - ) - # if remote_agent used attn is not same as local, - # hint heterogenuous attn post process - if ( - nixl_agent_meta.attn_backend_name != self.backend_name - and self.backend_name in ["CPU_ATTN"] - ): - if self._is_hma_required: - raise RuntimeError( - "heterogeneous attn post process is not supported with HMA" - ) - logger.info( - "[Experimental] CPU_ATTN backend is used, " - "hint heterogeneous attn post process" - ) - self.enable_heterogeneous_attn_post_process = True - - # Heterogeneous TP requires head-splitting, which only works with - # HND layout. MLA and replicated-KV cases don't split on heads. - # Mamba doesn't support heterogeneous TP. - if ( - abs(tp_ratio) != 1 - and not self.use_mla - and not self.transfer_topo.is_kv_replicated(remote_engine_id) - and kv_cache_layout != "HND" - and not self.enable_permute_local_kv - ): - raise RuntimeError( - "Heterogeneous TP head-dimension splitting requires contiguous heads. " - "Use HND layout on the prefill side." - ) - - # Per-region block_len validation enforcing the P/D invariant. - # REPLICATE regions (MLA, or a whole-model MLA / replicated-KV transfer) - # only allow the number of blocks to differ; SPLIT regions scale with - # tp_ratio. Mamba uses the ssm_sizes counterpart, so skip block_len here. - if not self._has_mamba: - assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), ( - "Number of KV layers must match between prefill and decode" - ) - model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated( - remote_engine_id - ) - for i, local_len in enumerate(self.block_len_per_layer): - replicated = model_replicated or self._is_region_replicated(i) - remote_len = nixl_agent_meta.block_lens[i] - if replicated: - # Whole block copied; only the number of blocks may differ. - assert local_len // block_size_ratio == remote_len, ( - "KV cache sizes must match between P and D when " - f"replicated (region {i}: local={local_len}, " - f"remote={remote_len}, bsr={block_size_ratio})." - ) - elif tp_ratio > 0: - # D_TP >= P_TP: remote holds tp_ratio x local heads. - assert remote_len == (local_len * tp_ratio) // block_size_ratio, ( - f"SPLIT region {i}: remote P KV block_len {remote_len} " - f"must equal local {local_len} * tp_ratio {tp_ratio} " - f"// block_size_ratio {block_size_ratio}." - ) - else: - # P_TP > D_TP: local holds |tp_ratio| x remote heads. - assert block_size_ratio == 1, ( - "Different local/remote block sizes are not supported " - "when P TP > D TP." - ) - assert remote_len == local_len // (-tp_ratio), ( - f"SPLIT region {i}: remote P KV block_len {remote_len} " - f"must equal local {local_len} // |tp_ratio| {-tp_ratio}." - ) - - # TP workers that handhshake with same remote have same #blocks. - assert self.dst_num_blocks[remote_engine_id] == nixl_agent_meta.num_blocks - # Same number of regions/~layers. - assert len(nixl_agent_meta.kv_caches_base_addr) == len(self.block_len_per_layer) - - def sync_recved_kv_to_device(self, req_id: str, meta: ReqMeta): - """copy recved kv from host buffer to device.""" - assert self.use_host_buffer - assert self.copy_blocks is not None - - local_block_ids = meta.local_physical_block_ids - # TODO (NickLucche) D2H<>H2D ops could benefit from coalescing io across groups - for group_block_ids in local_block_ids: - self.copy_blocks( - self.host_xfer_buffers, - self.device_kv_caches, - group_block_ids, - group_block_ids, - "h2d", - ) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "synced recved kv of request[%s] to device kv buffer," - "local_block_ids: %s. ", - req_id, - ",".join(map(str, local_block_ids)), - ) - - def save_kv_to_host(self, metadata: NixlConnectorMetadata): - """copy kv from device to host buffer.""" - assert self.use_host_buffer - assert self.copy_blocks is not None - - for req_id, meta in metadata.reqs_to_save.items(): - meta.local_physical_block_ids = self._logical_to_kernel_block_ids( - meta.local_block_ids - ) - if logger.isEnabledFor(logging.DEBUG): - logger.debug( - "save_load_kv for request[%s] to host xfer buffer." - "local_block_ids: %s. ", - req_id, - ",".join(map(str, meta.local_physical_block_ids)), - ) - # blocking - for group_block_ids in meta.local_physical_block_ids: - self.copy_blocks( - self.device_kv_caches, - self.host_xfer_buffers, - group_block_ids, - group_block_ids, - "d2h", - ) - - def post_process_device_kv_on_receive( - self, - block_size_ratio: int, - block_ids_list: list[list[int]], - ): - """ - Post process device kv cache after receiving from remote. - - 3 types of post processing supported: - * kv_cache_postprocess_layout => convert from HND to NHD - * kv_cache_postprocess_blksize => convert from small block size - to large block size - * kv_cache_postprocess_blksize_and_layout => convert from small - block size to large block size and convert from HND to NHD - - """ - if len(self.device_kv_caches) == 0: - return - assert block_size_ratio >= 1, "Only nP < nD supported currently." - assert self.transfer_topo is not None - if self.enable_permute_local_kv and block_size_ratio > 1: - logger.debug( - "Post-processing device kv cache on receive by converting " - "block_size with %sx bigger and permuting layout from HND" - " to NHD.", - block_size_ratio, - ) - elif self.enable_permute_local_kv: - logger.debug( - "Post-processing device kv cache on receive by permuting layout" - "from HND to NHD." - ) - else: - logger.debug( - "Post-processing device kv cache on receive by converting " - "block_size with %sx bigger.", - block_size_ratio, - ) - - split_k_and_v = self.transfer_topo.split_k_and_v - - for block_ids in block_ids_list: - indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) - - for _, cache_or_caches in self.device_kv_caches.items(): - cache_list = cache_or_caches if split_k_and_v else [cache_or_caches] - for cache in cache_list: - if self.enable_permute_local_kv and block_size_ratio > 1: - kv_postprocess_blksize_and_layout_on_receive( - cache, indices, block_size_ratio - ) - elif self.enable_permute_local_kv: - kv_postprocess_layout_on_receive(cache, indices) - else: - kv_postprocess_blksize_on_receive( - cache, indices, block_size_ratio - ) - - def post_process_device_kv_on_receive_heterogeneous_attn( - self, block_ids: list[int] - ): - """ - Post process device kv cache after receiving from remote - for heterogeneous attention. - """ - assert self.enable_heterogeneous_attn_post_process - - indices = torch.tensor(block_ids, device=self.device_type, dtype=torch.long) - - for _, cache_or_caches in self.device_kv_caches.items(): - blocks_to_update = cache_or_caches.index_select(1, indices) - current_platform.pack_kv_cache( - key=blocks_to_update[0], - value=blocks_to_update[1], - key_cache=cache_or_caches[0], - value_cache=cache_or_caches[1], - block_ids=block_ids, - indices=indices, - ) - - def get_finished(self) -> tuple[set[str], set[str]]: - """ - Get requests that are done sending or recving on this specific worker. - The scheduler process (via the MultiprocExecutor) will use this output - to track which workers are done. - """ - assert self.transfer_topo is not None - done_sending = self._get_new_notifs() - done_recving = self._pop_done_transfers(self._recving_transfers) - - # Drain queue of requests where handshake or transfer setup failed. - failed_recv_reqs = set[ReqId]() - while not self._failed_recv_reqs.empty(): - try: - failed_recv_reqs.add(self._failed_recv_reqs.get_nowait()) - except queue.Empty: - break - - # Add failed requests to done_recving for scheduler tracking - # (blocks are already marked invalid, scheduler will handle recompute) - done_recving.update(failed_recv_reqs) - - if len(done_sending) > 0 or len(done_recving) > 0: - logger.debug( - "Rank %s, get_finished: %s requests done sending " - "and %s requests done recving (%s failed)", - self.tp_rank, - len(done_sending), - len(done_recving), - len(failed_recv_reqs), - ) - - block_ids_for_blocksize_post_process = defaultdict(list) - block_ids_for_heterogeneous_attn_post_process = list[list[int]]() - for req_id in done_recving: - # clean up metadata for completed requests - meta = self._recving_metadata.pop(req_id, None) - assert meta is not None, f"{req_id} not found in recving_metadata list" - - # Skip KV sync and post-processing for failed requests - if req_id in failed_recv_reqs: - logger.warning( - "Skipping KV post-processing for failed request %s", - req_id, - ) - continue - - assert meta.remote is not None - if self.use_host_buffer: - self.sync_recved_kv_to_device(req_id, meta) - - # post processing for heteroblocksize - remote_info = self.transfer_topo.get_engine_info(meta.remote.engine_id) - block_size_ratio = self.transfer_topo.block_size_ratio( - remote_info.remote_block_size - ) - if not self.use_mla and ( - block_size_ratio > 1 or self.enable_permute_local_kv - ): - assert not self._is_hma_required - block_ids_for_blocksize_post_process[block_size_ratio].append( - meta.local_physical_block_ids[0] - ) - # post processing for heterogeneous attention - if self.enable_heterogeneous_attn_post_process: - block_ids_for_heterogeneous_attn_post_process.append( - meta.local_physical_block_ids[0] - ) - for ( - block_size_ratio, - block_ids_list, - ) in block_ids_for_blocksize_post_process.items(): - self.post_process_device_kv_on_receive(block_size_ratio, block_ids_list) - - for block_ids in block_ids_for_heterogeneous_attn_post_process: - self.post_process_device_kv_on_receive_heterogeneous_attn(block_ids) - - # Handle timeout to avoid stranding blocks on remote. - now = time.perf_counter() - while self._reqs_to_send: - req_id, expires = next(iter(self._reqs_to_send.items())) - # Sorted dict, oldest requests are put first so we can exit early. - if now < expires: - break - count = self.consumer_notification_counts_by_req.pop(req_id, 0) - self.xfer_stats.record_kv_expired_req() - logger.warning( - "Releasing expired KV blocks for request %s which were " - "retrieved by %d remote worker(s) before lease expired.", - req_id, - count, - ) - self._reqs_to_process.remove(req_id) - del self._reqs_to_send[req_id] - done_sending.add(req_id) - - return done_sending, done_recving - - def _get_new_notifs(self) -> set[str]: - """ - Get req_ids which got a remote xfer message. When multiple consumers - are reading from the same producer (heterogeneous TP scenario), wait - for all consumers to be done pulling. - - Also handles heartbeat notifications ("HB:req1,req2,...") by - extending the lease on the referenced requests. - """ - assert self.transfer_topo is not None - notified_req_ids: set[str] = set() - for notifs in self.nixl_wrapper.get_new_notifs().values(): - for notif in notifs: - msg = notif.decode("utf-8") - - # Handle heartbeat messages from D-side. - if msg.startswith("HB:"): - self._handle_heartbeat(msg[3:]) - continue - - req_id, tp_size = msg.rsplit(":", 1) - if ( - req_id not in self._reqs_to_send - and req_id not in self._reqs_to_process - ): - logger.error( - "Potentially invalid KV blocks for " - "unrecognized request %s were retrieved by " - "a decode worker. They may have expired.", - req_id, - ) - continue - - # NOTE: `tp_ratio` is the opposite when swapping local<>remote - n_consumers = int(tp_size) - tp_ratio = self.transfer_topo.tp_ratio(n_consumers) - - # Number of reads *per producer* to wait for. - # When remote D TP > local P TP we expect `tp_ratio` reads. - consumers_per_producer = ( - -tp_ratio if n_consumers > self.world_size else 1 - ) - - self.consumer_notification_counts_by_req[req_id] += 1 - # Wait all consumers (D) to be done reading before freeing. - if ( - self.consumer_notification_counts_by_req[req_id] - == consumers_per_producer - ): - notified_req_ids.add(req_id) - del self.consumer_notification_counts_by_req[req_id] - self._reqs_to_process.remove(req_id) - self._reqs_to_send.pop(req_id, None) - return notified_req_ids - - def _handle_heartbeat(self, payload: str) -> None: - """Extend leases for requests referenced in a heartbeat. - - Args: - payload: comma-separated P-side request IDs, e.g. - "req_abc,req_def". - """ - new_expiry = time.perf_counter() + self._lease_extension - for req_id in payload.split(","): - if req_id in self._reqs_to_send: - old = self._reqs_to_send[req_id] - self._reqs_to_send[req_id] = max(old, new_expiry) - logger.debug( - "Heartbeat extended lease for request %s " - "by %ds (old_expiry=%.1f, new_expiry=%.1f)", - req_id, - self._lease_extension, - old, - new_expiry, - ) - - def _pop_done_transfers(self, transfers: dict[str, list[int]]) -> set[str]: - """ - Pop completed xfers by checking for DONE state. - Args: - transfers: dict of req_id -> list[running_xfer] - Returns: - set of req_ids that have all done xfers - """ - done_req_ids: set[str] = set() - for req_id, handles in list(transfers.items()): - in_progress = [] - for handle in handles: - try: - xfer_state = self.nixl_wrapper.check_xfer_state(handle) - if xfer_state == "DONE": - # Get telemetry from NIXL - res = self.nixl_wrapper.get_xfer_telemetry(handle) - self.xfer_stats.record_transfer(res) - self.nixl_wrapper.release_xfer_handle(handle) - elif xfer_state == "PROC": - in_progress.append(handle) - continue - else: - self._log_failure( - failure_type="transfer_failed", - msg="Marking blocks as invalid", - req_id=req_id, - xfer_state=xfer_state, - ) - self._handle_failed_transfer(req_id, handle) - except Exception as e: - self._log_failure( - failure_type="transfer_exception", - msg="Marking blocks as invalid", - req_id=req_id, - error=e, - ) - self._handle_failed_transfer(req_id, handle) - - if not in_progress: - # Only report request as completed when all transfers are done. - done_req_ids.add(req_id) - del transfers[req_id] - else: - transfers[req_id] = in_progress - return done_req_ids - - def _handle_failed_transfer(self, req_id: str, handle: int | None): - """ - Handle a failed transfer by marking all (logical) blocks as invalid and - recording the failure. - - Args: - req_id: The request ID. - handle: The transfer handle. - """ - # Use .get() here as the metadata cleanup is handled by get_finished() - # TODO (NickLucche) handle failed transfer for HMA. - if (meta := self._recving_metadata.get(req_id)) and not self._is_hma_required: - self._invalid_block_ids.put(set(meta.local_block_ids[0])) - self._failed_recv_reqs.put(req_id) - if handle is not None: - self.nixl_wrapper.release_xfer_handle(handle) - self.xfer_stats.record_failed_transfer() - - def start_load_kv(self, metadata: NixlConnectorMetadata): - """ - Start loading by triggering non-blocking nixl_xfer. - We check for these trnxs to complete in each step(). - """ - for req_id, meta in metadata.reqs_to_recv.items(): - meta.local_physical_block_ids = self._logical_to_kernel_block_ids( - meta.local_block_ids - ) - assert meta.remote is not None - # Remote block IDs are kept logical here; expanded in - # _read_blocks_for_req using the remote engine's phys ratio. - remote_engine_id = meta.remote.engine_id - logger.debug( - "start_load_kv for request %s from remote engine %s. " - "Num local_block_ids: %s. Num remote_block_ids: %s. ", - req_id, - remote_engine_id, - len(meta.local_physical_block_ids), - len(meta.remote.block_ids), - ) - # always store metadata for failure recovery - self._recving_metadata[req_id] = meta - if remote_engine_id not in self._remote_agents: - # Initiate handshake with remote engine to exchange metadata. - with self._handshake_lock: - if remote_engine_id not in self._remote_agents: - self._background_nixl_handshake(req_id, remote_engine_id, meta) - continue - - # Handshake already completed, start async read xfer. - self._read_blocks_for_req(req_id, meta) - - # Start transfers for requests whose handshakes have now finished. - while not self._ready_requests.empty(): - self._read_blocks_for_req(*self._ready_requests.get_nowait()) - - # Keep around the requests that have been part of a batch. This is - # needed because async scheduling pushes the misalignment between the - # moment in which requests expiration is set (P side) and the moment in - # which blocks are read from D. As P can now more easily lag behind D - # while processing the next batch, we make sure to only set an - # expiration for requests that have not been read from D yet. - for req_id in metadata.reqs_in_batch: - self._reqs_to_process.add(req_id) - - # Remove all requests that are not to be processed (eg aborted). - for req_id in metadata.reqs_not_processed: - self._reqs_to_process.discard(req_id) - # We should never get an abort after setting an expiry timer - assert req_id not in self._reqs_to_send - - # Add to requests that are waiting to be read and track expiration. - for req_id, expiration_time in metadata.reqs_to_send.items(): - if req_id in self._reqs_to_process: - self._reqs_to_send[req_id] = expiration_time - - # Send heartbeats to P-side engines to keep KV blocks alive while - # requests sit in the D scheduler WAITING queue. - self._send_heartbeats(metadata) - - def _send_heartbeats(self, metadata: NixlConnectorMetadata) -> None: - """ - Send heartbeat notifications to remote engines, extending lease on KV blocks. - """ - for engine_id, hb_info in metadata.heartbeat_by_engine.items(): - # Proactive handshake (this request may still be in waiting queue) so - # the **next** heartbeat for this remote can go through. - if ( - self._ensure_handshake( - engine_id, hb_info.host, hb_info.port, hb_info.tp_size - ) - is not None - ): - continue # handshake is still pending - - # Build the heartbeat message: "HB:req1,req2,..." - hb_msg = ("HB:" + ",".join(hb_info.req_ids)).encode() - for agent_name in self._remote_agents[engine_id].values(): - try: - self.nixl_wrapper.send_notif(agent_name, notif_msg=hb_msg) - except Exception: - logger.debug( - "Failed to send heartbeat to engine %s", - engine_id, - exc_info=True, - ) - - def _read_blocks_for_req(self, req_id: str, meta: ReqMeta): - assert meta.remote is not None and self.transfer_topo is not None - engine_id = meta.remote.engine_id - # Update last activity from this remote. Mind that cleanup is done on main - # thread (this one), so we don't race on this structure. - self._engine_last_active[engine_id] = time.perf_counter() - plan = self.tp_mappings[engine_id] - remote_info = self.transfer_topo.get_engine_info(engine_id) - tp_ratio = self.transfer_topo.tp_ratio(remote_info.remote_tp_size) - - meta.remote.block_ids = self._logical_to_remote_kernel_block_ids( - meta.remote.block_ids, - remote_info.remote_physical_blocks_per_logical, - ) - remote_block_ids = meta.remote.block_ids - local_block_ids = meta.local_physical_block_ids - num_groups = len(local_block_ids) - read_specs = [ - ReadSpec( - remote_rank=rank, - local_block_ids=[ - list(local_block_ids[g]) - if rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ], - remote_block_ids=[ - list(remote_block_ids[g]) - if rank in plan.source_ranks_per_group[g] - else [] - for g in range(num_groups) - ], - ) - for rank in plan.all_source_ranks - ] - - # D may have to perform multiple reads from different remote ranks. - # MLA opt: when P TP > D TP, only a single read is executed for - # the first remote rank (cache is duplicated).. - if self.use_mla and tp_ratio < 0: - assert len(read_specs) == 1 - - for i, spec in enumerate(read_specs): - remote_block_size = remote_info.remote_block_size - logger.debug( - "Remote agent %s available, calling _read_blocks" - " on remote rank %s with remote block size %s for req %s", - meta.remote.engine_id, - spec.remote_rank, - remote_block_size, - req_id, - ) - # Get side handles. - if tp_ratio < 0 and not self.use_mla: - assert remote_block_size == self.block_size - # Remote tp_size > local tp_size: we must perform multiple - # reads. Get the memory chunk onto which we will write to. - local_xfer_side_handle = self.src_xfer_handles_by_tp_ratio[tp_ratio][i] - else: - # Single read from remote, we write to the whole memory region. - # Also handle remote block size different from local block size. - local_xfer_side_handle = self.src_xfer_handles_by_block_size[ - remote_block_size - ] - - # Destination handle: remote_engine_id -> remote_rank -> handle. - remote_xfer_side_handle = self.dst_xfer_side_handles[meta.remote.engine_id][ - spec.remote_rank - ] - - self._read_blocks( - read_spec=spec, - request_id=req_id, - dst_engine_id=meta.remote.engine_id, - remote_request_id=meta.remote.request_id, - local_xfer_side_handle=local_xfer_side_handle, - remote_xfer_side_handle=remote_xfer_side_handle, - ) - - if self.use_mla and tp_ratio < 0 and read_specs: - # ..but we still need to notify the other remote ranks that we - # have the blocks we need so they can update the request state. - notif_id = f"{meta.remote.request_id}:{self.world_size}".encode() - remote_agents = self._remote_agents[meta.remote.engine_id] - for rank_to_notify, agent in remote_agents.items(): - if rank_to_notify != read_specs[0].remote_rank: - self.nixl_wrapper.send_notif(agent, notif_msg=notif_id) - - def _read_blocks( - self, - read_spec: ReadSpec, - dst_engine_id: str, - request_id: str, - remote_request_id: str, - local_xfer_side_handle: int, - remote_xfer_side_handle: int, - ): - """ - Post a READ point-to-point xfer request from a single local worker to - a single remote worker. - """ - assert self.transfer_topo is not None - remote_rank = read_spec.remote_rank - local_block_ids = read_spec.local_block_ids - remote_block_ids = read_spec.remote_block_ids - - remote_info = self.transfer_topo.get_engine_info(dst_engine_id) - block_size_ratio = self.transfer_topo.block_size_ratio( - remote_info.remote_block_size - ) - if block_size_ratio > 1: - # TODO (NickLucche) assume HMA is off. Change to handle multiple KV groups. - assert not self._is_hma_required - local_block_ids0 = local_block_ids[0] if local_block_ids else [] - remote_block_ids0 = remote_block_ids[0] - local_block_ids_mapped = self.get_mapped_blocks( - np.asarray(local_block_ids0), block_size_ratio - ).tolist() - if len(local_block_ids_mapped) > len(remote_block_ids0): - # NOTE: - # get_mapped_blocks will always expand block_ids for n times. - # ex: - # prefill block_ids with block_size as 4: - # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - # Local decode block_ids with block_size as 16: [1, 2, 3] - # expanded decode block_ids with get_mapped_blocks from [1, 2, 3] to - # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] - # Then we clip local to align with prefill - # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] to - # [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - local_block_ids_mapped = local_block_ids_mapped[ - : len(remote_block_ids0) - ] - local_block_ids = [local_block_ids_mapped] if local_block_ids_mapped else [] - remote_block_ids = [remote_block_ids0] - # NOTE(rob): having the staging blocks be on the READER side is - # not going to work well (since we will have to call rearrange tensors). - # after we detect the txn is complete (which means we cannot make the - # read trxn async easily). If we want to make "READ" happen cleanly, - # then we will need to have the staging blocks on the remote side. - - # NOTE(rob): according to nvidia the staging blocks are used to - # saturate IB with heterogeneous TP sizes. - - # Number of D TP workers that will read from dst P. Propagate info - # on notification so that dst worker can wait before freeing blocks. - notif_id = f"{remote_request_id}:{self.world_size}".encode() - - # Full prefix cache hit: do not need to read remote blocks, - # just notify P worker that we have the blocks we need. - if len(local_block_ids) == 0: - # A full prefix cache hit is indicated with an empty list. - agent_name = self._remote_agents[dst_engine_id][remote_rank] - try: - self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id) - except Exception as e: - self._log_failure( - failure_type="notification_failed", - msg="P worker blocks will be freed after timeout. " - "This may indicate network issues.", - req_id=request_id, - error=e, - dst_engine_id=dst_engine_id, - remote_rank=remote_rank, - remote_agent_name=agent_name, - ) - self.xfer_stats.record_failed_notification() - return - - assert ( - len(remote_block_ids) - == len(local_block_ids) - == len(self.kv_cache_config.kv_cache_groups) - ) - remote_physical_per_logical = remote_info.remote_physical_blocks_per_logical - local_block_ids, remote_block_ids = self._apply_prefix_caching( - local_block_ids, remote_block_ids, remote_physical_per_logical - ) - - # NOTE (nicolo) With homogeneous TP, each TP worker loads KV from - # corresponding rank. With heterogeneous TP, fixing D>P, the D tp - # workers will issue xfers to parts of the P worker remote kv caches. - - # Get descs ids. - remote_block_descs_ids = self._compute_desc_ids( - block_ids=remote_block_ids, - dst_num_blocks=self.dst_num_blocks[dst_engine_id], - block_size_ratio=None, - physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical, - ) - local_block_descs_ids = self._compute_desc_ids( - block_ids=local_block_ids, - dst_num_blocks=self.dst_num_blocks[self.engine_id], - block_size_ratio=block_size_ratio, - physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block, - ) - - assert len(local_block_descs_ids) == len(remote_block_descs_ids) - - # Prepare transfer with Nixl. - handle = None - try: - handle = self.nixl_wrapper.make_prepped_xfer( - "READ", - local_xfer_side_handle, - local_block_descs_ids, - remote_xfer_side_handle, - remote_block_descs_ids, - notif_msg=notif_id, - ) - - # Begin async xfer. - self.nixl_wrapper.transfer(handle) - - # Use handle to check completion in future step(). - self._recving_transfers[request_id].append(handle) - except Exception as e: - # mark all (logical) blocks for this request as invalid - self._log_failure( - failure_type="transfer_setup_failed", - req_id=request_id, - msg="Marking blocks as invalid", - error=e, - dst_engine_id=dst_engine_id, - remote_rank=remote_rank, - ) - self._handle_failed_transfer(request_id, handle) - - def get_mapped_blocks( - self, block_ids: np.ndarray, block_size_ratio: int - ) -> np.ndarray: - """ - Calculates the new set of block IDs by mapping every element - in the (potentially sparse) input array. - Example: block_ids=[0, 2], block_size_ratio=2 - get_mapped_blocks 0 1 [2 3] 4 5 - # remote is |h0-b0|h1-b0||h0-b1|h1-b1||h0-b1|h1-b1|| - # local is |h0-b0......||h1-b0......||h2-b0........ - local_block_ids 0 [1] 2 - """ - if block_ids.size == 0: - return np.array([], dtype=np.int64) - - start_ids = block_ids * block_size_ratio - offsets = np.arange(block_size_ratio) - mapped_2d = start_ids[:, None] + offsets[None, :] - - return mapped_2d.flatten().astype(np.int64) - - def _logical_to_kernel_block_ids(self, block_ids: BlockIds) -> BlockIds: - """ - Convert logical block ids to kernel physical block ids. - This is required when the logical block size (the one set by the user) - does not match the one required by the attn backend. - """ - if self._physical_blocks_per_logical_kv_block == 1: - # Noop when physical and logical block sizes are the same - return block_ids - block_arange = np.arange(0, self._physical_blocks_per_logical_kv_block).reshape( - 1, -1 - ) - # Mamba blocks have no logical<>physical discrepancy - group_specs = self.kv_cache_config.kv_cache_groups - return [ - BlockTable.map_to_kernel_blocks( - np.array(group), - self._physical_blocks_per_logical_kv_block, - block_arange, - ).tolist() - if not isinstance(group_specs[i].kv_cache_spec, MambaSpec) - else group - for i, group in enumerate(block_ids) - ] - - def _apply_prefix_caching( - self, - local_block_ids: BlockIds, - remote_block_ids: BlockIds, - remote_physical_per_logical: int, - ) -> tuple[BlockIds, list]: - """Apply prefix caching by trimming local/remote block ID lists. - - For non-Mamba models: end-trim remote to match local count, so that - already-cached prefix blocks are skipped in the transfer. - - For Mamba hybrid (prefix caching not yet supported): front-trim both - to the minimum count to handle kernel block count discrepancies from - logical block rounding in heterogeneous TP. - """ - # Partial prefix cache hit: just read uncomputed blocks. - # Skip mamba groups — their blocks represent full state (conv+ssm), - # not per-token data, so trimming would corrupt the transfer. - remote_block_ids = list(remote_block_ids) - if not self._has_mamba: - for i, remote_group in enumerate(remote_block_ids): - num_local_blocks = len(local_block_ids[i]) - assert num_local_blocks <= len(remote_group) - if num_local_blocks < len(remote_group): - remote_block_ids[i] = remote_group[-num_local_blocks:] - else: - # (NOTE: ZhanqiuHu) Mamba hybrid: no prefix caching support so far.HeteroTP - # can cause different kernel block counts due to logical block rounding. - # Example: 640 prompt tokens, kernel_block_size=64 - # remote physical_per_logical=10, local physical_per_logical=6 - # remote logical ids from kv_transfer_params = [0] - # local logical ids allocated = [0, 1] - # remote kernel blocks: [0..9] (1*10=10) - # local kernel blocks: [0..11] (2*6=12) - # actual data blocks = ceil(640/64) = 10, trim both to 10 - # Vice versa (remote physical_per_logical=6, local=10): - # remote logical ids = [0, 1], local logical ids = [0] - # remote kernel blocks: [0..11] (2*6=12) - # local kernel blocks: [0..9] (1*10=10) - # actual data blocks = ceil(640/64) = 10, trim both to 10 - local_block_ids = list(local_block_ids) - for i, remote_group in enumerate(remote_block_ids): - num_local_blocks = len(local_block_ids[i]) - num_remote_blocks = len(remote_group) - if ( - _is_ssm_spec(self._group_spec_types[i]) - and num_local_blocks < num_remote_blocks - ): - # NOTE (NickLucche): With prefix caching on SSM, (remote) blocks - # prior to the last one are placeholders (null blocks). Mind that - # this doesn't really impact transfer, as we only still care about - # the last "block", the full in-place state. - assert num_local_blocks == 1, "SSM can only have one local block" - remote_block_ids[i] = remote_group[-num_local_blocks:] - elif ( - self._physical_blocks_per_logical_kv_block - == remote_physical_per_logical - and num_local_blocks < num_remote_blocks - ): - # Partial prefix cache hit for FA group. - remote_block_ids[i] = remote_group[-num_local_blocks:] - else: - # TODO Handle prefix caching with different block_sizes - max_padding = max( - self._physical_blocks_per_logical_kv_block, - remote_physical_per_logical, - ) - assert abs(num_local_blocks - num_remote_blocks) < max_padding, ( - f"Group {i}: |{num_local_blocks} - " - f"{num_remote_blocks}| >= {max_padding}" - ) - num_blocks = min(num_local_blocks, num_remote_blocks) - local_block_ids[i] = local_block_ids[i][:num_blocks] - remote_block_ids[i] = remote_group[:num_blocks] - return local_block_ids, remote_block_ids - - def _logical_to_remote_kernel_block_ids( - self, block_ids: BlockIds, remote_physical_per_logical: int - ) -> BlockIds: - """Map logical block IDs to physical kernel block IDs on the remote. - - Args: - block_ids: per-group lists of logical block IDs. - remote_physical_per_logical: remote engine's physical blocks - per logical block. - - Returns: - Same structure with FA groups expanded (each logical block L - becomes kernel blocks [L*remote_physical_per_logical, .. - L*remote_physical_per_logical + - remote_physical_per_logical - 1]). - Mamba groups are passed through unchanged. - """ - if remote_physical_per_logical == 1: - return block_ids - remote_arange = np.arange(remote_physical_per_logical).reshape(1, -1) - group_specs = self.kv_cache_config.kv_cache_groups - result = [ - BlockTable.map_to_kernel_blocks( - np.array(group), - remote_physical_per_logical, - remote_arange, - ).tolist() - if not isinstance(group_specs[i].kv_cache_spec, MambaSpec) - else group - for i, group in enumerate(block_ids) - ] - return result - - def get_backend_aware_kv_block_len( - self, layer_idx: int, first_split: bool = True, mamba_view: bool = False - ) -> int: - """ - Get the block length for one K/V element (K and V have the same size). - - For FA and other backends, this is equal to the length of the whole - block, as K and V are in separate regions. - For FlashInfer, this is half the length of the whole block, as K and V - share the same region. - Similarly, for SSM-based models, state and conv are interleaved, but crucially - the their size differs. - Reference diagram: - KVCacheTensor (Shared) - / \\ - / \\ - / \\ - Attention (FlashInfer) View Mamba View - | | - | | - +-------------------+ +-------------------+ - | KVCacheTensor | | KVCacheTensor | - | | | | - |<----- page ------>| |<----- page ------->| - | size | | size | - | Key 0 | Val 0 | |Conv 0 | SSM 0 | - | Key 1 | Val 1 | |Conv 1 | SSM 1 | - | ... | ... | | ... | ... | - | Key N-2 | Val N-2 | |Conv N-2| SSM N-2 | - | Key N-1 | Val N-1 | |Conv N-1| SSM N-1 | - +-------------------+ +--------------------+ - |1st_split-2nd_split| |1st_split-2nd_split | - """ - assert self.transfer_topo is not None - virtually_split = self.transfer_topo.virtually_split_kv_in_blocks - if virtually_split and mamba_view: - block_len = self._mamba_ssm_size[not first_split] - else: - # Per-descriptor block length: a SPLIT region (full-attn under the - # virtually-split layout) emits separate K and V and uses - # block_len//2; REPLICATE (MLA, key-only) and non-split layouts use - # the whole block. - half_block = virtually_split and not self._is_region_replicated(layer_idx) - block_len = self.block_len_per_layer[layer_idx] // (2 if half_block else 1) - return block_len - - def get_kv_connector_stats(self) -> KVConnectorStats | None: - """ - Get the KV transfer stats for the connector. - """ - # Clear stats for next iteration - if not self.xfer_stats.is_empty(): - return self.xfer_stats.clone_and_reset() - return None - - def get_block_ids_with_load_errors(self) -> set[int]: - """ - Return and clear the set of block IDs that failed to load. - - This is called by the scheduler to identify blocks that need - to be retried after a NIXL transfer failure. - """ - # Drain the queue (thread-safe, no lock needed). - result: set[int] = set() - while not self._invalid_block_ids.empty(): - try: - result.update(self._invalid_block_ids.get_nowait()) - except queue.Empty: - break - return result - - def _evict_stale_engines(self) -> None: - """Scan for and evict remote engines that have exceeded their TTL. - - Called from the main thread in when a new remote engine appears. - We can only go OOM as we discover and register a new remote, therefore we make - sure we clean up stale engine data structures before then. This invariant - prevents us from using background threads, though memory usage is not guaranteed - to be "optimal" until a new handshake is performed. - - Engines with active transfers or pending handshakes cannot be stale: - - Active transfers touch _engine_last_active in start_load_kv. - - Pending handshakes don't have an _engine_last_active entry yet - """ - # NOTE (NickLucche): This does NOT currently prevent OOMing if a huge number - # of remote engines is registered all at once (adding a background cleanup - # thread wouldnt help either). - # If that scenario is plausible, we can follow up with an LRU eviction policy. - if self._engine_ttl <= 0: - return - - now = time.perf_counter() - for eid, last_active in list(self._engine_last_active.items()): - if now - last_active > self._engine_ttl: - self._cleanup_remote_engine(eid) - - def _cleanup_remote_engine( - self, engine_id: EngineId, *, log_eviction: bool = True - ) -> None: - """Remove all state for a single remote engine. - - Releases NIXL resources (dlist handles, remote agents) and clears - all per-engine data structures. Used by both TTL eviction and - shutdown. - """ - assert engine_id in self._remote_agents - - for handle in self.dst_xfer_side_handles.pop(engine_id).values(): - self.nixl_wrapper.release_dlist_handle(handle) - for agent_name in self._remote_agents.pop(engine_id).values(): - self.nixl_wrapper.remove_remote_agent(agent_name) - - del self.kv_caches_base_addr[engine_id] - del self.dst_num_blocks[engine_id] - del self.tp_mappings[engine_id] - if self.transfer_topo is not None: - self.transfer_topo.unregister_remote_engine(engine_id) - - last_active = self._engine_last_active.pop(engine_id) - if log_eviction: - logger.info( - "Evicted stale remote engine %s (inactive for %.1fs).", - engine_id, - time.perf_counter() - last_active, - ) - - def __del__(self): - self.shutdown() - - def shutdown(self): - """Shutdown the connector worker.""" - if not hasattr(self, "_handshake_initiation_executor"): - # error happens during init, no need to shutdown - return - self._handshake_initiation_executor.shutdown(wait=False) - for handles in self._recving_transfers.values(): - for handle in handles: - self.nixl_wrapper.release_xfer_handle(handle) - self._recving_transfers.clear() - for handle in self.src_xfer_handles_by_block_size.values(): - self.nixl_wrapper.release_dlist_handle(handle) - self.src_xfer_handles_by_block_size.clear() - for handles in self.src_xfer_handles_by_tp_ratio.values(): - for handle in handles: - self.nixl_wrapper.release_dlist_handle(handle) - self.src_xfer_handles_by_tp_ratio.clear() - for engine_id in list(self._remote_agents): - self._cleanup_remote_engine(engine_id, log_eviction=False) - for desc in self._registered_descs: - self.nixl_wrapper.deregister_memory(desc) - self._registered_descs.clear() +__all__ = ["NixlConnectorWorker", "NixlPullConnectorWorker"] diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 926f406f19..5ca90fd129 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -2009,6 +2009,19 @@ class Scheduler(SchedulerInterface): ) return len(self.requests) > num_in_queues + def has_requests(self) -> bool: + # Override the interface default to also keep the engine alive while a + # connector still has pending push work (e.g. push-mode WRITE transfers + # in flight after all "live" requests have finished). Without this hook + # the engine would quiesce before the connector can drain completions. + # TODO: replace with a more general mechanism for connectors to keep + # the scheduler alive. + return ( + self.has_unfinished_requests() + or self.has_finished_requests() + or (self.connector is not None and self.connector.has_pending_push_work()) + ) + def reset_prefix_cache( self, reset_running_requests: bool = False, reset_connector: bool = False ) -> bool: