diff --git a/docs/features/kv_offloading_usage.md b/docs/features/kv_offloading_usage.md index 8a48d11be7..7042ef787c 100644 --- a/docs/features/kv_offloading_usage.md +++ b/docs/features/kv_offloading_usage.md @@ -152,16 +152,25 @@ Object keys follow the same run-configuration digest scheme as the filesystem ti The P2P tier (`type: "p2p"`) shares completed KV blocks between vLLM instances over RDMA via NIXL. Each instance binds a control socket on `host:port` and exchanges blocks directly with peers — no shared filesystem required. +PYTHONHASHSEED environment variable must be set to the same fixed value on all nodes. + | Key | Required | Default | Notes | | --- | --- | --- | --- | | `type` | yes | — | Must be `p2p`. | -| `host` | no | `0.0.0.0` | Address the control socket binds to. | -| `port` | no | `7777` | Port for the control socket. Must be reachable from peers. | +| `host` | no | `$VLLM_P2P_SIDE_CHANNEL_HOST` (`localhost`) | Address the control socket binds to, used verbatim as the identity peers dial back. When omitted, resolves from the env var below. The `localhost` default binds loopback only — for cross-host P2P you **must** set it to the node's routable IP (see below). | +| `port` | no | `$VLLM_P2P_SIDE_CHANNEL_PORT` (`5710`) | Base port for the control socket. Must be reachable from peers. The bound port is `base + data_parallel_index` (one socket per DP replica). When omitted, the base resolves from the env var below. | | `backends` | no | `["UCX"]` | NIXL transport backends. See [NixlConnector Usage Guide](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin) for available backends and selection guidance. | | `num_threads` | no | `4` | NIXL agent worker threads. Only used when `backends` is UCX-only; ignored when any non-UCX backend is requested. | The `backends` and `num_threads` options mirror the conditional logic used by [`NixlConnector`](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin): when any non-UCX backend is configured, NIXL is initialised with `backends=...`; otherwise it falls back to a UCX-only agent with the configured `num_threads`. This lets the P2P tier use a different transport (e.g. `MOONCAKE`, `GDS_MT`, `LIBFABRIC`) than the main `NixlConnector` running in the same process. +#### Environment Variables + +Rather than embedding `host`/`port` in each `secondary_tiers` entry, set them once at deploy time via environment variables (mirroring `VLLM_NIXL_SIDE_CHANNEL_HOST`/`VLLM_NIXL_SIDE_CHANNEL_PORT`). Explicit `host`/`port` config keys, when present, take precedence. + +- `VLLM_P2P_SIDE_CHANNEL_HOST` (default `localhost`): address the P2P control socket binds to. It is used **verbatim** as both the bind address and the identity peers dial back — there is no auto-detection (this mirrors `VLLM_NIXL_SIDE_CHANNEL_HOST`). The default binds the loopback interface only, so peers on another host cannot reach it. **For any cross-host P2P deployment you must set this explicitly to the node's routable IP** (e.g. the pod IP) before launching `vllm serve` — otherwise remote peers will fail to connect. The NIXL agent name is a separate per-process identifier, so peers sharing a `host:port` never collide. +- `VLLM_P2P_SIDE_CHANNEL_PORT` (default `5710`): base port for the P2P control socket. The port actually bound is `VLLM_P2P_SIDE_CHANNEL_PORT + data_parallel_index` — one socket per DP replica, matching NIXL (for DP=1 the offset is 0). The peer's port is passed as `remote_port` in `kv_transfer_params`; the router/EPP that selects the DP rank (e.g. via the `X-data-parallel-rank` header) computes `remote_port = base + rank`. The DP-index offset separates replicas *within* one deployment; two co-located *deployments* (a prefiller and a decoder on the same host) still need distinct base ports (e.g. decoder base `5711`) to avoid a bind collision. + ## Tuning Tips - `cpu_bytes_to_use`: a bigger CPU tier means fewer trips to slower secondary tiers and a higher hit rate. The value is total across all workers, not per-worker. Leave headroom for the rest of the host workload. diff --git a/tests/test_envs.py b/tests/test_envs.py index d4d120ecee..56c04dd6f2 100644 --- a/tests/test_envs.py +++ b/tests/test_envs.py @@ -36,6 +36,18 @@ def test_nixl_side_channel_host_is_not_compile_factor( assert "VLLM_NIXL_SIDE_CHANNEL_HOST" not in envs.compile_factors() +def test_p2p_side_channel_defaults_and_override(monkeypatch: pytest.MonkeyPatch): + monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False) + monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False) + assert envs.VLLM_P2P_SIDE_CHANNEL_HOST == "localhost" + assert envs.VLLM_P2P_SIDE_CHANNEL_PORT == 5710 + + monkeypatch.setenv("VLLM_P2P_SIDE_CHANNEL_HOST", "10.0.0.20") + monkeypatch.setenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5799") + assert envs.VLLM_P2P_SIDE_CHANNEL_HOST == "10.0.0.20" + assert envs.VLLM_P2P_SIDE_CHANNEL_PORT == 5799 + + def test_getattr_with_cache(monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("VLLM_HOST_IP", "1.1.1.1") monkeypatch.setenv("VLLM_PORT", "1234") diff --git a/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py b/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py index 10566f40e6..2f0df89902 100644 --- a/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py +++ b/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py @@ -14,7 +14,7 @@ Usage: --port 8192 \ --prefiller-host 127.0.0.1 --prefiller-port 8100 \ --decoder-host 127.0.0.1 --decoder-port 8200 \ - --p2p-connector-host 127.0.0.1 --p2p-connector-port 7777 + --p2p-connector-host 127.0.0.1 --p2p-connector-port 5710 """ import argparse @@ -73,14 +73,23 @@ async def lifespan(app: FastAPI): app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients))) app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients))) + # Round-robin over each role's data-parallel replicas (independent of the + # instance iterators above). For dp_size=1 these yield only rank 0. + app.state.prefill_dp_iterator = itertools.cycle( + range(global_args.prefiller_dp_size) + ) + app.state.decode_dp_iterator = itertools.cycle(range(global_args.decoder_dp_size)) + mode = "decoder-first" if global_args.decoder_first else "prefiller-first" pd_host = global_args.p2p_connector_host pd_port = global_args.p2p_connector_port + n_pref = len(app.state.prefill_clients) + n_dec = len(app.state.decode_clients) print( f"Proxy ready [{mode}]: " - f"{len(app.state.prefill_clients)} prefiller(s), " - f"{len(app.state.decode_clients)} decoder(s). " - f"P2PConnector at {pd_host}:{pd_port}" + f"{n_pref} prefiller(s) x dp={global_args.prefiller_dp_size}, " + f"{n_dec} decoder(s) x dp={global_args.decoder_dp_size}. " + f"P2PConnector base at {pd_host}:{pd_port}" ) yield @@ -111,8 +120,9 @@ def parse_args(): p.add_argument( "--p2p-connector-port", type=int, - default=7777, - help="Port of the prefiller's P2PConnector ZMQ socket", + default=int(os.getenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5710")), + help="Port of the prefiller's P2PConnector ZMQ socket " + "(default: $VLLM_P2P_SIDE_CHANNEL_PORT or 5710)", ) # P2PConnector coordinates of the decoder — injected into prefill requests # so the prefiller's submit_store can resolve the peer to push KV to. @@ -125,8 +135,9 @@ def parse_args(): p.add_argument( "--decoder-p2p-connector-port", type=int, - default=7778, - help="Port of the decoder's P2PConnector ZMQ socket", + default=int(os.getenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5710")) + 1, + help="Port of the decoder's P2PConnector ZMQ socket " + "(default: $VLLM_P2P_SIDE_CHANNEL_PORT + 1 or 5711)", ) p.add_argument( "--decoder-first", @@ -134,6 +145,23 @@ def parse_args(): help="Send decode request before prefill so decoder is already " "waiting when KV blocks arrive (decoder-first mode)", ) + p.add_argument( + "--prefiller-dp-size", + type=int, + default=1, + help="Data-parallel replica count of the prefiller. When >1 the proxy " + "round-robins prefill across ranks via the X-data-parallel-rank header " + "and injects remote_port = p2p-connector-port + rank into the decode " + "request so the decoder pulls KV from that replica's control socket. " + "Assumes a single prefiller instance (one host/port fronting N replicas).", + ) + p.add_argument( + "--decoder-dp-size", + type=int, + default=1, + help="Data-parallel replica count of the decoder. When >1 the proxy " + "round-robins decode across ranks via the X-data-parallel-rank header.", + ) args = p.parse_args() if len(args.prefiller_hosts) != len(args.prefiller_ports): raise ValueError("Prefiller host/port count mismatch") @@ -150,6 +178,23 @@ def _get_next(app, service: str): return app.state.decode_clients[next(app.state.decode_iterator)] +def _next_dp_rank(app, service: str): + """Advance the round-robin DP cursor for a role. + + Returns (rank, header_rank). ``rank`` (0..dp_size-1) is always used for the + P2P remote_port arithmetic; ``header_rank`` is the same value when the role + has dp_size>1 and None otherwise (so dp=1 sends no header, matching the + single-replica behavior). + """ + if service == "prefill": + rank = next(app.state.prefill_dp_iterator) + dp_size = global_args.prefiller_dp_size + else: + rank = next(app.state.decode_dp_iterator) + dp_size = global_args.decoder_dp_size + return rank, (rank if dp_size > 1 else None) + + def _auth_headers(request_id: str) -> dict: headers: dict = {"X-Request-Id": request_id} api_key = os.environ.get("OPENAI_API_KEY", "") @@ -158,7 +203,7 @@ def _auth_headers(request_id: str) -> dict: return headers -async def _prefill(client_info, endpoint, req_data, request_id): +async def _prefill(client_info, endpoint, req_data, request_id, dp_rank=None): """Send a prefill-only request (max_tokens=1) to the prefiller.""" data = req_data.copy() data["kv_transfer_params"] = { @@ -174,14 +219,21 @@ async def _prefill(client_info, endpoint, req_data, request_id): data.pop("min_completion_tokens", None) headers = _auth_headers(request_id) + if dp_rank is not None: + # Pin this prefill to a specific DP replica so its P2PConnector (bound + # at base_port + data_parallel_index) holds the produced KV blocks. + headers["X-data-parallel-rank"] = str(dp_rank) resp = await client_info["client"].post(endpoint, json=data, headers=headers) resp.raise_for_status() await resp.aread() return resp -async def _stream_decode(client_info, endpoint, req_data, request_id): +async def _stream_decode(client_info, endpoint, req_data, request_id, dp_rank=None): headers = _auth_headers(request_id) + if dp_rank is not None: + # Round-robin this decode onto a specific DP replica of the decoder. + headers["X-data-parallel-rank"] = str(dp_rank) async with client_info["client"].stream( "POST", endpoint, json=req_data, headers=headers ) as resp: @@ -195,24 +247,39 @@ async def _handle_completions(api: str, request: Request): req_data = await request.json() request_id = str(uuid.uuid4()) + # Pick DP replicas for both roles up front (before any await) so the + # two round-robin advances are atomic together. Header rank is None for + # a role with dp_size==1 (no header → single replica, unchanged). + prefill_rank, prefill_hdr = _next_dp_rank(request.app, "prefill") + decode_rank, decode_hdr = _next_dp_rank(request.app, "decode") + prefill_client = _get_next(request.app, "prefill") - await _prefill(prefill_client, api, req_data, request_id) + await _prefill(prefill_client, api, req_data, request_id, dp_rank=prefill_hdr) # Inject the prefiller's P2PConnector address so the decoder can pull - # KV blocks from it via the P2PConnector transport. + # KV blocks from it. remote_port = base + prefill_rank targets the + # replica that produced the KV (base+0 == base when dp=1). req_data["kv_transfer_params"] = { "prefill": { "kv_request_id": request_id, "remote_host": global_args.p2p_connector_host, - "remote_port": global_args.p2p_connector_port, + "remote_port": global_args.p2p_connector_port + prefill_rank, }, } decode_client = _get_next(request.app, "decode") - logger.debug("prefill=%s decode=%s", prefill_client, decode_client) + logger.debug( + "prefill=%s dp=%s decode=%s dp=%s", + prefill_client, + prefill_rank, + decode_client, + decode_rank, + ) async def generate(): - async for chunk in _stream_decode(decode_client, api, req_data, request_id): + async for chunk in _stream_decode( + decode_client, api, req_data, request_id, dp_rank=decode_hdr + ): yield chunk return StreamingResponse(generate(), media_type="application/json") @@ -237,6 +304,10 @@ async def _handle_completions_decoder_first(api: str, request: Request): req_data = await request.json() request_id = str(uuid.uuid4()) + # Pick DP replicas up front (see _handle_completions for rationale). + prefill_rank, prefill_hdr = _next_dp_rank(request.app, "prefill") + decode_rank, decode_hdr = _next_dp_rank(request.app, "decode") + prefill_client = _get_next(request.app, "prefill") decode_client = _get_next(request.app, "decode") @@ -245,7 +316,7 @@ async def _handle_completions_decoder_first(api: str, request: Request): "prefill": { "kv_request_id": request_id, "remote_host": global_args.p2p_connector_host, - "remote_port": global_args.p2p_connector_port, + "remote_port": global_args.p2p_connector_port + prefill_rank, }, } @@ -255,7 +326,7 @@ async def _handle_completions_decoder_first(api: str, request: Request): async def _run_decode(): try: async for chunk in _stream_decode( - decode_client, api, decode_data, request_id + decode_client, api, decode_data, request_id, dp_rank=decode_hdr ): await queue.put(("data", chunk)) except Exception as exc: @@ -268,7 +339,9 @@ async def _handle_completions_decoder_first(api: str, request: Request): # 2. Send prefill — blocks are computed and pushed to the decoder try: - await _prefill(prefill_client, api, req_data, request_id) + await _prefill( + prefill_client, api, req_data, request_id, dp_rank=prefill_hdr + ) except Exception as exc: logger.warning("decoder-first: prefill failed: %s", exc) diff --git a/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh b/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh index 82053ebfad..89300376a7 100755 --- a/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh +++ b/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh @@ -16,6 +16,12 @@ # NUM_DECODE_INSTANCES default 1 # PREFILLER_TP_SIZE default 1 # DECODER_TP_SIZE default 1 +# DP_EP when set, deploy the decoder in DP-EP attention +# mode (dp=DECODER_TP_SIZE, tp=1, +# --enable-expert-parallel) instead of tensor +# parallel; the proxy round-robins decode across +# the DP replicas. Mirrors +# nixl_integration/run_accuracy_test.sh. # GPU_MEMORY_UTILIZATION default 0.45 # MAX_MODEL_LEN default 512 # PREFILL_BLOCK_SIZE default 128 @@ -29,6 +35,8 @@ # NUM_PREFILL_INSTANCES=2 NUM_DECODE_INSTANCES=2 \ # bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh # bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh --decoder-first +# DP_EP=1 DECODER_TP_SIZE=2 \ +# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh set -xe @@ -71,6 +79,14 @@ NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} +# DP-EP attention mode (see header). When DP_EP is set the decoder uses +# data-parallel replicas (dp=DECODER_TP_SIZE) that the proxy round-robins over. +DP_EP=${DP_EP:-} +if [[ -n "$DP_EP" ]]; then + DECODER_DP_SIZE=${DECODER_TP_SIZE} +else + DECODER_DP_SIZE=1 +fi GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.45} MAX_MODEL_LEN=${MAX_MODEL_LEN:-512} PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128} @@ -162,7 +178,11 @@ run_tests_for_model() { echo "================================" echo "Testing model: $model_name" echo " prefillers=${NUM_PREFILL_INSTANCES} (tp=${PREFILLER_TP_SIZE})" - echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})" + if [[ -n "$DP_EP" ]]; then + echo " decoders=${NUM_DECODE_INSTANCES} (dp-ep=${DECODER_TP_SIZE}, tp=1)" + else + echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})" + fi echo " decoder_first=${DECODER_FIRST}" echo "================================" @@ -241,9 +261,19 @@ run_tests_for_model() { --block-size ${DECODE_BLOCK_SIZE} \ --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \ --max-model-len ${MAX_MODEL_LEN} \ - --tensor-parallel-size ${DECODER_TP_SIZE} \ --kv-transfer-config '${kv_cfg}'" + # DP-EP attention mode: data-parallel + expert-parallel decode replicas + # (dp=DECODER_TP_SIZE, tp=1) instead of tensor parallel. Mirrors + # nixl_integration/run_accuracy_test.sh. + if [[ -z "$DP_EP" ]]; then + BASE_CMD="${BASE_CMD} --tensor-parallel-size ${DECODER_TP_SIZE}" + else + echo "DP-EP Attention enabled, deploying decoder with dp=${DECODER_TP_SIZE} and tp=1" + BASE_CMD="${BASE_CMD} --data-parallel-size ${DECODER_TP_SIZE} \ + --tensor-parallel-size 1 --enable-expert-parallel" + fi + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" for arg in "${extra_args[@]}"; do @@ -282,7 +312,8 @@ run_tests_for_model() { --p2p-connector-host ${P2P_HOST} \ --p2p-connector-port ${PREFILL_PD_PORTS[0]} \ --decoder-p2p-connector-host ${P2P_HOST} \ - --decoder-p2p-connector-port ${DECODE_PD_PORTS[0]}" + --decoder-p2p-connector-port ${DECODE_PD_PORTS[0]} \ + --decoder-dp-size ${DECODER_DP_SIZE}" if [[ "${DECODER_FIRST}" == "true" ]]; then PROXY_CMD="${PROXY_CMD} --decoder-first" diff --git a/tests/v1/kv_offload/tiering/p2p/test_data_transport.py b/tests/v1/kv_offload/tiering/p2p/test_data_transport.py index d515e102e2..7eb8124352 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_data_transport.py +++ b/tests/v1/kv_offload/tiering/p2p/test_data_transport.py @@ -183,6 +183,35 @@ class TestNixlTransportWithMockedAgent: assert result.done == () assert result.failed == () + def test_poll_peer_id_scopes_to_peer(self): + """poll(peer_id) drains only that peer's transfers. + + Regression: the transport is shared across peer sessions (e.g. a + single prefiller serving a DP>1 decoder). An unscoped poll by one + session used to consume and discard sibling sessions' completions, + starving them until timeout. poll(peer_id) must leave other peers' + transfers inflight. + """ + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + transport.add_remote_peer("peer:2", b"meta", 0x2000, 8, 1024) + + tid1 = transport.write_blocks("peer:1", [0], [1]) + tid2 = transport.write_blocks("peer:2", [2], [3]) + transport._agent.check_xfer_state.return_value = "DONE" + + # Polling peer:1 must not consume peer:2's completed transfer. + result = transport.poll(peer_id="peer:1") + assert tid1 in result.done + assert tid2 not in result.done + assert tid1 not in transport._inflight + assert tid2 in transport._inflight + + # peer:2 sees its own completion when it polls. + result2 = transport.poll(peer_id="peer:2") + assert tid2 in result2.done + assert tid2 not in transport._inflight + def test_cancel_removes_inflight(self): """cancel removes transfers and releases handles.""" transport = self._make_transport() diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py index 73e3655ecc..52e2c83632 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_manager.py +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -9,9 +9,11 @@ using fake transport and session objects. from __future__ import annotations import time +import uuid from types import SimpleNamespace import numpy as np +import pytest from vllm.v1.kv_offload.base import LookupResult, ReqContext, ScheduleEndContext from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult @@ -598,7 +600,7 @@ class _ShutdownFakeData: return list(self._still_queue[0]) return [] - def poll(self): + def poll(self, peer_id=None): self.poll_calls += 1 class _Empty: @@ -831,7 +833,7 @@ class _FakeData: self._inflight_done.append(tid) return tid - def poll(self): + def poll(self, peer_id=None): from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult done = self._inflight_done[:] @@ -1373,3 +1375,115 @@ class TestConnectionDeathMidTransfer: assert (900, False) not in finishes assert (900, True) not in finishes assert "req-store" in mgr_a._unbound_stores + + +# --------------------------------------------------------------------------- +# Tests for host/port resolution in __init__ (env-var defaults) +# --------------------------------------------------------------------------- + + +class TestBindHostPortDefaults: + """host/port fall back to VLLM_P2P_SIDE_CHANNEL_* when not in config.""" + + @staticmethod + def _construct(monkeypatch, dp_index=0, **kwargs) -> P2PSecondaryTierManager: + """Build a manager with the transports/file-mapper stubbed out. + + The host is used verbatim (no resolution). The transport constructor + args are recorded on ``mgr._test_calls`` so tests can assert the ZMQ + identity (``host:port``) stays decoupled from the NIXL agent name + (a uuid). + """ + monkeypatch.setattr( + manager_module, + "FileMapper", + SimpleNamespace( + from_offloading_spec=lambda **_: SimpleNamespace( + get_run_config=lambda: {} + ) + ), + ) + calls: dict = {} + monkeypatch.setattr( + manager_module, + "NixlTransport", + lambda agent_name, *a, **k: calls.update(nixl_name=agent_name) + or SimpleNamespace(), + ) + monkeypatch.setattr( + manager_module, + "ZmqTransport", + lambda local_id, host, port, *a, **k: calls.update( + zmq_id=local_id, zmq_host=host, zmq_port=port + ) + or SimpleNamespace(), + ) + spec = SimpleNamespace( + block_size_factor=1, + vllm_config=SimpleNamespace( + parallel_config=SimpleNamespace(data_parallel_index=dp_index) + ), + ) + mgr = P2PSecondaryTierManager(spec, memoryview(b""), **kwargs) + mgr._test_calls = calls + return mgr + + def test_defaults_from_env_unset(self, monkeypatch): + monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False) + monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False) + mgr = self._construct(monkeypatch) + # localhost default is used verbatim as the dial-back identity. + assert mgr._local_id == "localhost:5710" + + def test_env_override(self, monkeypatch): + monkeypatch.setenv("VLLM_P2P_SIDE_CHANNEL_HOST", "10.1.2.3") + monkeypatch.setenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5799") + mgr = self._construct(monkeypatch) + assert mgr._local_id == "10.1.2.3:5799" + + def test_explicit_config_wins(self, monkeypatch): + monkeypatch.setenv("VLLM_P2P_SIDE_CHANNEL_HOST", "10.1.2.3") + monkeypatch.setenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5799") + mgr = self._construct(monkeypatch, host="192.0.2.5", port=6001) + assert mgr._local_id == "192.0.2.5:6001" + + def test_dp_index_offsets_default_port(self, monkeypatch): + monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_HOST", raising=False) + monkeypatch.delenv("VLLM_P2P_SIDE_CHANNEL_PORT", raising=False) + mgr = self._construct(monkeypatch, dp_index=2) + assert mgr._local_id == "localhost:5712" + + def test_dp_index_offsets_explicit_port(self, monkeypatch): + mgr = self._construct(monkeypatch, dp_index=1, host="192.0.2.5", port=6000) + assert mgr._local_id == "192.0.2.5:6001" + + @pytest.mark.parametrize( + "bind_host", ["localhost", "127.0.0.1", "::1", "0.0.0.0", "::", "192.0.2.5"] + ) + def test_host_used_verbatim(self, monkeypatch, bind_host): + # The host is never rewritten — loopbacks and wildcards alike become + # the dial-back identity verbatim (mirrors the NIXL connector). + mgr = self._construct(monkeypatch, host=bind_host, port=5710) + assert mgr._local_id == f"{bind_host}:5710" + + def test_nixl_name_decoupled_from_identity(self, monkeypatch): + # The ZMQ identity is the verbatim host:port; the NIXL agent name is a + # uuid, distinct from the identity and never used as an address. + mgr = self._construct(monkeypatch, host="127.0.0.1", port=5710) + assert mgr._test_calls["zmq_host"] == "127.0.0.1" + assert mgr._test_calls["zmq_port"] == 5710 + assert mgr._test_calls["zmq_id"] == "127.0.0.1:5710" + assert mgr._local_id == "127.0.0.1:5710" + # nixl_name is a valid uuid4 and not the host:port identity. + nixl_name = mgr._test_calls["nixl_name"] + assert nixl_name != mgr._local_id + assert uuid.UUID(nixl_name).version == 4 + + def test_same_host_port_gets_distinct_nixl_names(self, monkeypatch): + # The original collision: two peers sharing a host:port must still get + # distinct NIXL agent names so add_remote_agent doesn't reject a remote + # whose name equals the local. The per-process uuid guarantees this. + mgr_a = self._construct(monkeypatch, host="localhost", port=5710) + mgr_b = self._construct(monkeypatch, host="localhost", port=5710) + assert mgr_a._local_id == mgr_b._local_id == "localhost:5710" + assert mgr_a._nixl_agent_name != mgr_b._nixl_agent_name diff --git a/tests/v1/kv_offload/tiering/p2p/test_sessions.py b/tests/v1/kv_offload/tiering/p2p/test_sessions.py index 05c0a3abdd..d1c6ad0cda 100644 --- a/tests/v1/kv_offload/tiering/p2p/test_sessions.py +++ b/tests/v1/kv_offload/tiering/p2p/test_sessions.py @@ -110,7 +110,7 @@ class FakeDataTransport: self._transfers[tid] = (peer_id, local_idxs, remote_idxs) return tid - def poll(self): + def poll(self, peer_id=None): from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult result = PollResult(done=list(self._poll_done), failed=list(self._poll_failed)) diff --git a/vllm/envs.py b/vllm/envs.py index adcc3c24f0..5b601cda8c 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -209,6 +209,8 @@ if TYPE_CHECKING: VLLM_DISABLE_REQUEST_ID_RANDOMIZATION: bool = False VLLM_NIXL_SIDE_CHANNEL_HOST: str = "localhost" VLLM_NIXL_SIDE_CHANNEL_PORT: int = 5600 + VLLM_P2P_SIDE_CHANNEL_HOST: str = "localhost" + VLLM_P2P_SIDE_CHANNEL_PORT: int = 5710 VLLM_EC_SIDE_CHANNEL_HOST: str = "localhost" VLLM_EC_SIDE_CHANNEL_PORT: int = 5601 VLLM_MOONCAKE_BOOTSTRAP_PORT: int = 8998 @@ -1586,6 +1588,16 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_NIXL_SIDE_CHANNEL_PORT": lambda: int( os.getenv("VLLM_NIXL_SIDE_CHANNEL_PORT", "5600") ), + # Address the P2P KV-offload control socket binds to. Defaults to + # ``localhost`` (loopback only); must be set to the node IP for + # cross-host P2P so remote peers can reach the socket. + "VLLM_P2P_SIDE_CHANNEL_HOST": lambda: os.getenv( + "VLLM_P2P_SIDE_CHANNEL_HOST", "localhost" + ), + # Port the P2P KV-offload control socket binds to. + "VLLM_P2P_SIDE_CHANNEL_PORT": lambda: int( + os.getenv("VLLM_P2P_SIDE_CHANNEL_PORT", "5710") + ), # IP address used for the EC connector's ZMQ side channel # (producer ROUTER bind, consumer DEALER dial). "VLLM_EC_SIDE_CHANNEL_HOST": lambda: os.getenv( diff --git a/vllm/v1/kv_offload/tiering/p2p/data/base.py b/vllm/v1/kv_offload/tiering/p2p/data/base.py index a18b1ca0b3..ffe609d71c 100644 --- a/vllm/v1/kv_offload/tiering/p2p/data/base.py +++ b/vllm/v1/kv_offload/tiering/p2p/data/base.py @@ -207,8 +207,17 @@ class DataTransport(ABC): ... @abstractmethod - def poll(self) -> PollResult: - """Poll all inflight transfers for completion. + def poll(self, peer_id: str | None = None) -> PollResult: + """Poll inflight transfers for completion. + + Args: + peer_id: If given, only poll (and drain) transfers submitted for + this peer_id — the value passed to ``write_blocks``. This is + required when a single transport is shared across multiple + peer sessions: ``poll()`` pops completed handles, so an + unscoped poll by one session would consume and discard the + completions of its siblings, starving them. ``None`` polls + every peer's transfers (used only for the shutdown drain). Returns: PollResult with lists of completed and failed transfer_ids. diff --git a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py index 5f283c4815..8d53b923be 100644 --- a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py +++ b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py @@ -8,7 +8,7 @@ from __future__ import annotations import itertools from collections.abc import Iterable -from typing import Any +from typing import Any, NamedTuple from vllm.distributed.nixl_utils import NixlWrapper as _NixlAgent from vllm.distributed.nixl_utils import nixl_agent_config as _NixlAgentConfig @@ -27,6 +27,17 @@ logger = init_logger(__name__) _EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=()) +class _Inflight(NamedTuple): + """A submitted-but-not-yet-drained transfer. + + ``peer_id`` lets poll() scope to a single owning session, since the + transport is shared across all peer sessions of the engine. + """ + + peer_id: str + handle: object + + class NixlTransport(DataTransport): """Manages a NIXL agent, memory registration, and block transfers. @@ -37,14 +48,14 @@ class NixlTransport(DataTransport): def __init__( self, - local_id: str, + agent_name: str, view: memoryview, config_fields: dict | None = None, backends: list[str] | None = None, num_threads: int = 4, ) -> None: super().__init__(view, config_fields=config_fields) - self._local_id = local_id + self._agent_name = agent_name self._backends = list(backends) if backends else ["UCX"] self._num_threads = num_threads self._agent: Any = None @@ -52,7 +63,8 @@ class NixlTransport(DataTransport): self._local_dlist: Any = None self._remote_dlists: dict[str, object] = {} self._peer_nixl_names: dict[str, str] = {} - self._inflight: dict[int, object] = {} # transfer_id → handle + # transfer_id → _Inflight(peer_id, handle). + self._inflight: dict[int, _Inflight] = {} self._next_id = itertools.count() self._init(view) @@ -70,7 +82,7 @@ class NixlTransport(DataTransport): cfg = _NixlAgentConfig(backends=self._backends, capture_telemetry=True) logger.info( "NixlTransport %s: NIXL backends=%s", - self._local_id, + self._agent_name, self._backends, ) else: @@ -79,10 +91,10 @@ class NixlTransport(DataTransport): ) logger.info( "NixlTransport %s: NIXL backends=[UCX] num_threads=%d", - self._local_id, + self._agent_name, self._num_threads, ) - self._agent = _NixlAgent(self._local_id, cfg) + self._agent = _NixlAgent(self._agent_name, cfg) total_size = self._num_blocks * self._block_len reg_descs = [(self._base_addr, total_size, 0, "")] @@ -95,7 +107,7 @@ class NixlTransport(DataTransport): xfer_dlist = self._agent.get_xfer_descs(block_tuples, mem_type="DRAM") self._local_dlist = self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", xfer_dlist) logger.info( - "NixlTransport %s: registered %d blocks", self._local_id, self._num_blocks + "NixlTransport %s: registered %d blocks", self._agent_name, self._num_blocks ) def get_agent_metadata(self) -> bytes: @@ -152,14 +164,14 @@ class NixlTransport(DataTransport): logger.warning( "NixlTransport %s: write_blocks NO REMOTE DLIST for peer=%s " "(known peers=%s)", - self._local_id, + self._agent_name, peer_id, list(self._remote_dlists.keys()), ) return None logger.debug( "NixlTransport %s: write_blocks NIXL.transfer peer=%s blocks=%d", - self._local_id, + self._agent_name, peer_id, len(local_idxs), ) @@ -172,11 +184,16 @@ class NixlTransport(DataTransport): ) self._agent.transfer(handle) transfer_id = next(self._next_id) - self._inflight[transfer_id] = handle + self._inflight[transfer_id] = _Inflight(peer_id, handle) return transfer_id - def poll(self) -> PollResult: - """Poll all inflight transfers. + def poll(self, peer_id: str | None = None) -> PollResult: + """Poll inflight transfers. + + When *peer_id* is given, only transfers submitted for that peer_id are + checked and drained — the transport is shared across peer sessions, so + an unscoped poll by one session would consume and discard siblings' + completions. *peer_id* None polls every peer (shutdown drain only). Returns PollResult(done=..., failed=...) with transfer IDs. Completed handles are released automatically. @@ -187,13 +204,15 @@ class NixlTransport(DataTransport): done_ids: list[int] | None = None failed_ids: list[int] | None = None - for transfer_id, handle in self._inflight.items(): + for transfer_id, entry in self._inflight.items(): + if peer_id is not None and entry.peer_id != peer_id: + continue try: - state = self._agent.check_xfer_state(handle) + state = self._agent.check_xfer_state(entry.handle) except Exception as exc: logger.warning( "NixlTransport %s: check_xfer_state failed for transfer_id=%d: %s", - self._local_id, + self._agent_name, transfer_id, exc, ) @@ -212,9 +231,9 @@ class NixlTransport(DataTransport): handles_to_release = [] for tid in done_ids or (): - handles_to_release.append(self._inflight.pop(tid)) + handles_to_release.append(self._inflight.pop(tid).handle) for tid in failed_ids or (): - handles_to_release.append(self._inflight.pop(tid)) + handles_to_release.append(self._inflight.pop(tid).handle) self._release_handles(handles_to_release) return PollResult( @@ -236,22 +255,24 @@ class NixlTransport(DataTransport): """ if mode == "immediate": handles = [ - self._inflight.pop(tid) for tid in transfer_ids if tid in self._inflight + self._inflight.pop(tid).handle + for tid in transfer_ids + if tid in self._inflight ] self._release_handles(handles) return [] still_inflight: list[int] = [] for tid in transfer_ids: - handle = self._inflight.get(tid) - if handle is None: + entry = self._inflight.get(tid) + if entry is None: continue try: - self._agent.release_xfer_handle(handle) + self._agent.release_xfer_handle(entry.handle) except Exception as exc: logger.debug( "NixlTransport %s: cancel pending for transfer_id=%d: %s", - self._local_id, + self._agent_name, tid, exc, ) @@ -267,7 +288,7 @@ class NixlTransport(DataTransport): def close(self) -> None: if self._agent is None: return - self._release_handles(list(self._inflight.values())) + self._release_handles([entry.handle for entry in self._inflight.values()]) self._inflight.clear() for peer_id in list(self._remote_dlists): self.remove_remote_peer(peer_id) @@ -292,6 +313,6 @@ class NixlTransport(DataTransport): except Exception as exc: logger.warning( "NixlTransport %s: release_xfer_handle failed: %s", - self._local_id, + self._agent_name, exc, ) diff --git a/vllm/v1/kv_offload/tiering/p2p/manager.py b/vllm/v1/kv_offload/tiering/p2p/manager.py index 6949686f44..0cb37dff0e 100644 --- a/vllm/v1/kv_offload/tiering/p2p/manager.py +++ b/vllm/v1/kv_offload/tiering/p2p/manager.py @@ -9,12 +9,14 @@ Owns transports and a single bidirectional P2PSession per remote peer. from __future__ import annotations import time +import uuid from collections.abc import Iterable, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any from typing_extensions import override +import vllm.envs as envs from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( LookupResult, @@ -111,8 +113,8 @@ class P2PSecondaryTierManager(SecondaryTierManager): offloading_spec: OffloadingSpec, primary_kv_view: memoryview, tier_type: str = "p2p", - host: str = "0.0.0.0", - port: int = 7777, + host: str | None = None, + port: int | None = None, backends: list[str] | None = None, num_threads: int = 4, **kwargs: Any, @@ -130,9 +132,19 @@ class P2PSecondaryTierManager(SecondaryTierManager): primary_kv_view: Memoryview over the CPU primary tier; the NIXL agent registers this region for RDMA transfers. tier_type: Tier identifier (defaults to ``"p2p"``). - host: Address the ZMQ control socket binds to. - port: Port for the ZMQ control socket. Must be reachable - from peers. + host: Address the ZMQ control socket binds to, used verbatim + as both the bind address and the identity peers dial back + (mirrors the NIXL connector's ``VLLM_NIXL_SIDE_CHANNEL_HOST``; + no auto-detection). Defaults to + ``VLLM_P2P_SIDE_CHANNEL_HOST`` (``localhost``) when not set; + must be set to the node's routable IP for cross-host P2P so + remote peers can reach the socket. + port: Base port for the ZMQ control socket. Must be + reachable from peers. Defaults to + ``VLLM_P2P_SIDE_CHANNEL_PORT`` (``5710``) when not set. + The bound port is ``base + data_parallel_index`` so each + DP replica gets a distinct port (one socket per replica, + like NIXL); for DP=1 the offset is 0. backends: NIXL transport backends (e.g. ``["UCX"]``, ``["MOONCAKE"]``, ``["LIBFABRIC"]``). Defaults to ``["UCX"]``. When any non-UCX backend is requested, the @@ -145,8 +157,26 @@ class P2PSecondaryTierManager(SecondaryTierManager): **kwargs: Reserved for future tier-specific options. """ super().__init__(offloading_spec, primary_kv_view, tier_type) - port = int(port) + if host is None: + host = envs.VLLM_P2P_SIDE_CHANNEL_HOST + if port is None: + port = envs.VLLM_P2P_SIDE_CHANNEL_PORT + # One control socket per DP replica: offset the base by the global + # data-parallel index so replicas on a host don't collide (mirrors + # NIXL). For DP=1 the index is 0, leaving the base port unchanged. + dp_index = offloading_spec.vllm_config.parallel_config.data_parallel_index + port = int(port) + dp_index + # Two decoupled identities: + # _local_id (``host:port``): the ZMQ control identity that peers + # dial back, used verbatim (the socket binds this host/port and + # the address is parsed back into host:port by the remote). + # _nixl_agent_name (uuid4): the NIXL agent name. It is never dialed + # — it travels opaquely inside the agent metadata blob — so it + # only needs to be globally unique. A per-process uuid guarantees + # that even for peers sharing a host:port (mirrors the NIXL + # connector; avoids the "remote agent name equals local" reject). self._local_id = f"{host}:{port}" + self._nixl_agent_name = str(uuid.uuid4()) config_fields = FileMapper.from_offloading_spec( root_dir="", @@ -155,7 +185,7 @@ class P2PSecondaryTierManager(SecondaryTierManager): parallel_agnostic=True, ).get_run_config() self._data: DataTransport = NixlTransport( - self._local_id, + self._nixl_agent_name, primary_kv_view, config_fields=config_fields, backends=backends, diff --git a/vllm/v1/kv_offload/tiering/p2p/session/client.py b/vllm/v1/kv_offload/tiering/p2p/session/client.py index fce5008deb..f080cb79a3 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/client.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/client.py @@ -139,6 +139,14 @@ class ClientRole: """Handle an AbortAckMsg from the peer.""" req = self._inbound.pop(kv_request_id, None) if req is not None: + logger.warning( + "P2PSession %s: load request %s (job_id=%d) timed out; " + "load job completed with failure. If this recurs, ensure " + "PYTHONHASHSEED is set to the same value on all nodes.", + self._peer_id, + kv_request_id, + req.job_id, + ) self._completed_loads.append( LoadResult( job_id=req.job_id, diff --git a/vllm/v1/kv_offload/tiering/p2p/session/server.py b/vllm/v1/kv_offload/tiering/p2p/session/server.py index 6f5059e550..2a6c8c6924 100644 --- a/vllm/v1/kv_offload/tiering/p2p/session/server.py +++ b/vllm/v1/kv_offload/tiering/p2p/session/server.py @@ -282,7 +282,11 @@ class ServerRole: results.extend(self._pending_store_results) self._pending_store_results.clear() - poll_result = self._transport.poll() + # Scope the poll to this peer: the transport is shared across all peer + # sessions of the engine, and poll() drains completed handles. An + # unscoped poll here would consume sibling sessions' completions and + # report them as "unknown transfer_id", starving those sessions. + poll_result = self._transport.poll(self._peer_id) for tid in poll_result.done: xfer = self._inflight_pop(tid)