Secondary tier implementation for PD disaggregation (#42285)

Signed-off-by: Liran Schour <lirans@il.ibm.com>
Signed-off-by: liranschour <liranschour@users.noreply.github.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
liranschour
2026-06-30 07:51:44 +03:00
committed by GitHub
co-authored by Or Ozeri Or Ozeri
parent b5c9e1ac33
commit bec232a914
22 changed files with 7471 additions and 0 deletions
@@ -0,0 +1,331 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
P2PConnector proxy server for OffloadingConnector + TieringOffloadingSpec.
Unlike NixlConnector (which returns remote_host/remote_port in the prefill
response), OffloadingConnector does not embed connector coordinates in its
response. This proxy injects the prefiller's P2PConnector address into
kv_transfer_params before forwarding the decode request so the decoder knows
where to pull KV blocks from.
Usage:
.venv/bin/python p2p_connector_proxy.py \
--port 8192 \
--prefiller-host 127.0.0.1 --prefiller-port 8100 \
--decoder-host 127.0.0.1 --decoder-port 8200 \
--p2p-connector-host 127.0.0.1 --p2p-connector-port 7777
"""
import argparse
import asyncio
import itertools
import logging
import os
import uuid
from contextlib import asynccontextmanager
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import StreamingResponse
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@asynccontextmanager
async def lifespan(app: FastAPI):
app.state.prefill_clients = []
app.state.decode_clients = []
for i, (host, port) in enumerate(global_args.prefiller_instances):
app.state.prefill_clients.append(
{
"client": httpx.AsyncClient(
timeout=None,
base_url=f"http://{host}:{port}/v1",
limits=httpx.Limits(
max_connections=None, max_keepalive_connections=None
),
),
"host": host,
"port": port,
"id": i,
}
)
for i, (host, port) in enumerate(global_args.decoder_instances):
app.state.decode_clients.append(
{
"client": httpx.AsyncClient(
timeout=None,
base_url=f"http://{host}:{port}/v1",
limits=httpx.Limits(
max_connections=None, max_keepalive_connections=None
),
),
"host": host,
"port": port,
"id": i,
}
)
app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients)))
app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients)))
mode = "decoder-first" if global_args.decoder_first else "prefiller-first"
pd_host = global_args.p2p_connector_host
pd_port = global_args.p2p_connector_port
print(
f"Proxy ready [{mode}]: "
f"{len(app.state.prefill_clients)} prefiller(s), "
f"{len(app.state.decode_clients)} decoder(s). "
f"P2PConnector at {pd_host}:{pd_port}"
)
yield
for ci in app.state.prefill_clients:
await ci["client"].aclose()
for ci in app.state.decode_clients:
await ci["client"].aclose()
app = FastAPI(lifespan=lifespan)
def parse_args():
p = argparse.ArgumentParser()
p.add_argument("--port", type=int, default=8192)
p.add_argument("--host", type=str, default="127.0.0.1")
p.add_argument("--prefiller-hosts", type=str, nargs="+", default=["127.0.0.1"])
p.add_argument("--prefiller-ports", type=int, nargs="+", default=[8100])
p.add_argument("--decoder-hosts", type=str, nargs="+", default=["127.0.0.1"])
p.add_argument("--decoder-ports", type=int, nargs="+", default=[8200])
# P2PConnector coordinates of the prefiller — injected into decode requests.
p.add_argument(
"--p2p-connector-host",
type=str,
default="127.0.0.1",
help="Host of the prefiller's P2PConnector ZMQ socket",
)
p.add_argument(
"--p2p-connector-port",
type=int,
default=7777,
help="Port of the prefiller's P2PConnector ZMQ socket",
)
# P2PConnector coordinates of the decoder — injected into prefill requests
# so the prefiller's submit_store can resolve the peer to push KV to.
p.add_argument(
"--decoder-p2p-connector-host",
type=str,
default="127.0.0.1",
help="Host of the decoder's P2PConnector ZMQ socket",
)
p.add_argument(
"--decoder-p2p-connector-port",
type=int,
default=7778,
help="Port of the decoder's P2PConnector ZMQ socket",
)
p.add_argument(
"--decoder-first",
action="store_true",
help="Send decode request before prefill so decoder is already "
"waiting when KV blocks arrive (decoder-first mode)",
)
args = p.parse_args()
if len(args.prefiller_hosts) != len(args.prefiller_ports):
raise ValueError("Prefiller host/port count mismatch")
if len(args.decoder_hosts) != len(args.decoder_ports):
raise ValueError("Decoder host/port count mismatch")
args.prefiller_instances = list(zip(args.prefiller_hosts, args.prefiller_ports))
args.decoder_instances = list(zip(args.decoder_hosts, args.decoder_ports))
return args
def _get_next(app, service: str):
if service == "prefill":
return app.state.prefill_clients[next(app.state.prefill_iterator)]
return app.state.decode_clients[next(app.state.decode_iterator)]
def _auth_headers(request_id: str) -> dict:
headers: dict = {"X-Request-Id": request_id}
api_key = os.environ.get("OPENAI_API_KEY", "")
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
return headers
async def _prefill(client_info, endpoint, req_data, request_id):
"""Send a prefill-only request (max_tokens=1) to the prefiller."""
data = req_data.copy()
data["kv_transfer_params"] = {
"decode": {
"kv_request_id": request_id,
},
}
data["stream"] = False
data["max_tokens"] = 1
data.pop("max_completion_tokens", None)
data.pop("stream_options", None)
data.pop("min_tokens", None)
data.pop("min_completion_tokens", None)
headers = _auth_headers(request_id)
resp = await client_info["client"].post(endpoint, json=data, headers=headers)
resp.raise_for_status()
await resp.aread()
return resp
async def _stream_decode(client_info, endpoint, req_data, request_id):
headers = _auth_headers(request_id)
async with client_info["client"].stream(
"POST", endpoint, json=req_data, headers=headers
) as resp:
resp.raise_for_status()
async for chunk in resp.aiter_bytes():
yield chunk
async def _handle_completions(api: str, request: Request):
try:
req_data = await request.json()
request_id = str(uuid.uuid4())
prefill_client = _get_next(request.app, "prefill")
await _prefill(prefill_client, api, req_data, request_id)
# Inject the prefiller's P2PConnector address so the decoder can pull
# KV blocks from it via the P2PConnector transport.
req_data["kv_transfer_params"] = {
"prefill": {
"kv_request_id": request_id,
"remote_host": global_args.p2p_connector_host,
"remote_port": global_args.p2p_connector_port,
},
}
decode_client = _get_next(request.app, "decode")
logger.debug("prefill=%s decode=%s", prefill_client, decode_client)
async def generate():
async for chunk in _stream_decode(decode_client, api, req_data, request_id):
yield chunk
return StreamingResponse(generate(), media_type="application/json")
except Exception as e:
import sys
import traceback
print(f"Proxy error on {api}: {e}")
print("".join(traceback.format_exception(*sys.exc_info())))
raise
async def _handle_completions_decoder_first(api: str, request: Request):
"""Decoder-first mode: send decode request before prefill.
The decoder establishes its request and starts polling for KV blocks
immediately. The prefill is then sent so the prefiller computes and
pushes blocks to the already-waiting decoder.
"""
try:
req_data = await request.json()
request_id = str(uuid.uuid4())
prefill_client = _get_next(request.app, "prefill")
decode_client = _get_next(request.app, "decode")
decode_data = req_data.copy()
decode_data["kv_transfer_params"] = {
"prefill": {
"kv_request_id": request_id,
"remote_host": global_args.p2p_connector_host,
"remote_port": global_args.p2p_connector_port,
},
}
async def generate():
queue: asyncio.Queue = asyncio.Queue()
async def _run_decode():
try:
async for chunk in _stream_decode(
decode_client, api, decode_data, request_id
):
await queue.put(("data", chunk))
except Exception as exc:
await queue.put(("error", exc))
finally:
await queue.put(("done", None))
# 1. Start decode request — decoder is now waiting for KV blocks
asyncio.create_task(_run_decode())
# 2. Send prefill — blocks are computed and pushed to the decoder
try:
await _prefill(prefill_client, api, req_data, request_id)
except Exception as exc:
logger.warning("decoder-first: prefill failed: %s", exc)
logger.debug(
"decoder-first: prefill done, streaming decode prefill=%s decode=%s",
prefill_client,
decode_client,
)
# 3. Stream the decode response
while True:
kind, value = await queue.get()
if kind == "done":
break
if kind == "error":
raise value # type: ignore[misc]
yield value
return StreamingResponse(generate(), media_type="application/json")
except Exception as e:
import sys
import traceback
print(f"Proxy error on {api}: {e}")
print("".join(traceback.format_exception(*sys.exc_info())))
raise
def _route_handler(api: str):
if global_args.decoder_first:
return lambda req: _handle_completions_decoder_first(api, req)
return lambda req: _handle_completions(api, req)
@app.post("/v1/completions")
async def completions(request: Request):
return await _route_handler("/completions")(request)
@app.post("/v1/chat/completions")
async def chat_completions(request: Request):
return await _route_handler("/chat/completions")(request)
@app.get("/healthcheck")
async def healthcheck():
return {
"status": "ok",
"prefill_instances": len(app.state.prefill_clients),
"decode_instances": len(app.state.decode_clients),
}
if __name__ == "__main__":
global global_args
global_args = parse_args()
import uvicorn
uvicorn.run(app, host=global_args.host, port=global_args.port)
+312
View File
@@ -0,0 +1,312 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Accuracy test driver for the p2p connector
# (OffloadingConnector + TieringOffloadingSpec + p2p tier).
#
# Mirrors tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:
# brings up N prefillers + M decoders on the local host, fronts them
# with p2p_connector_proxy.py, then runs the connector-agnostic
# test_accuracy.py (gsm8k via lm_eval) against the proxy.
#
# Knobs (env vars unless flagged otherwise):
# MODEL_NAMES space-separated model list (default: Llama-3.2-1B-Instruct)
# NUM_PREFILL_INSTANCES default 1
# NUM_DECODE_INSTANCES default 1
# PREFILLER_TP_SIZE default 1
# DECODER_TP_SIZE default 1
# GPU_MEMORY_UTILIZATION default 0.45
# MAX_MODEL_LEN default 512
# PREFILL_BLOCK_SIZE default 128
# DECODE_BLOCK_SIZE default 128
# CPU_BYTES default 209715200 (200 MB)
# VLLM_SERVE_EXTRA_ARGS comma-separated extra args for vllm serve
# --decoder-first toggle decoder-first proxy mode
#
# Examples:
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
# NUM_PREFILL_INSTANCES=2 NUM_DECODE_INSTANCES=2 \
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh --decoder-first
set -xe
# ---------------------------------------------------------------------------
# Args
# ---------------------------------------------------------------------------
DECODER_FIRST="false"
while [[ $# -gt 0 ]]; do
case $1 in
--decoder-first)
DECODER_FIRST="true"
shift 1
;;
*)
echo "Unknown option $1"
echo "Usage: $0 [--decoder-first]"
exit 1
;;
esac
done
# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------
MODEL_NAMES=${MODEL_NAMES:-}
if [[ -n "$MODEL_NAMES" ]]; then
# shellcheck disable=SC2206
MODELS=($MODEL_NAMES)
else
MODELS=(
"meta-llama/Llama-3.2-1B-Instruct"
)
fi
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1}
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1}
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.45}
MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128}
CPU_BYTES=${CPU_BYTES:-209715200}
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
# Base ports — per-instance offsets layered on top.
PREFILL_HTTP_BASE=8100
DECODE_HTTP_BASE=8200
PREFILL_PD_BASE=7777
DECODE_PD_BASE=$((PREFILL_PD_BASE + NUM_PREFILL_INSTANCES))
PROXY_PORT=8192
P2P_HOST=127.0.0.1
# ---------------------------------------------------------------------------
# Resolve repo root + venv (works in .venv and /workspace/venv pods)
# ---------------------------------------------------------------------------
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../../.." && pwd -P)}"
if [[ -z "${VLLM_BIN:-}" ]]; then
if [[ -x "${GIT_ROOT}/.venv/bin/vllm" ]]; then
VLLM_BIN="${GIT_ROOT}/.venv/bin/vllm"
elif [[ -x "/workspace/venv/bin/vllm" ]]; then
VLLM_BIN="/workspace/venv/bin/vllm"
else
VLLM_BIN="$(command -v vllm)"
fi
fi
if [[ -z "${PYTHON_BIN:-}" ]]; then
if [[ -x "${GIT_ROOT}/.venv/bin/python" ]]; then
PYTHON_BIN="${GIT_ROOT}/.venv/bin/python"
elif [[ -x "/workspace/venv/bin/python" ]]; then
PYTHON_BIN="/workspace/venv/bin/python"
else
PYTHON_BIN="$(command -v python3 || command -v python)"
fi
fi
echo "Using vllm: ${VLLM_BIN}"
echo "Using python: ${PYTHON_BIN}"
SMI_BIN=$(command -v nvidia-smi || command -v rocm-smi || echo "")
# Trap SIGINT/SIGTERM/EXIT to kill background jobs.
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
wait_for_server() {
local port=$1
timeout 1200 bash -c "
until curl -s localhost:${port}/v1/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
cleanup_instances() {
echo "Cleaning up any running vLLM / proxy instances..."
pkill -f "vllm serve" || true
pkill -f "p2p_connector_proxy.py" || true
sleep 2
}
get_num_gpus() {
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
$SMI_BIN -l | grep -c GPU
else
echo "1"
fi
}
# Build the OffloadingConnector kv-transfer-config for a given PD port.
# Mirrors deploy_local.sh:131.
build_kv_config() {
local pd_port=$1
printf '{"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":%s,"secondary_tiers":[{"type":"p2p","host":"%s","port":%s}]}}' \
"${CPU_BYTES}" "${P2P_HOST}" "${pd_port}"
}
# ---------------------------------------------------------------------------
# Per-model run
# ---------------------------------------------------------------------------
run_tests_for_model() {
local model_name=$1
echo "================================"
echo "Testing model: $model_name"
echo " prefillers=${NUM_PREFILL_INSTANCES} (tp=${PREFILLER_TP_SIZE})"
echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})"
echo " decoder_first=${DECODER_FIRST}"
echo "================================"
PREFILL_HOSTS=()
PREFILL_PORTS=()
PREFILL_PD_PORTS=()
DECODE_HOSTS=()
DECODE_PORTS=()
DECODE_PD_PORTS=()
local num_gpus
num_gpus=$(get_num_gpus)
local next_gpu=0
# ---- Prefillers ----
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
local gpu_id=$((i * PREFILLER_TP_SIZE % num_gpus))
local cuda_devs="${gpu_id}"
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
done
next_gpu=$(((gpu_id + PREFILLER_TP_SIZE) % num_gpus))
local http_port=$((PREFILL_HTTP_BASE + i))
local pd_port=$((PREFILL_PD_BASE + i))
local kv_cfg
kv_cfg=$(build_kv_config "${pd_port}")
echo "Prefiller $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
PYTHONHASHSEED=42 \
${VLLM_BIN} serve ${model_name} \
--port ${http_port} \
--enforce-eager \
--block-size ${PREFILL_BLOCK_SIZE} \
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
--max-model-len ${MAX_MODEL_LEN} \
--tensor-parallel-size ${PREFILLER_TP_SIZE} \
--kv-transfer-config '${kv_cfg}'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
BASE_CMD="${BASE_CMD} $arg"
done
fi
eval "${BASE_CMD} &"
PREFILL_HOSTS+=("${P2P_HOST}")
PREFILL_PORTS+=("${http_port}")
PREFILL_PD_PORTS+=("${pd_port}")
done
# ---- Decoders ----
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
local gpu_id=$(((next_gpu + i * DECODER_TP_SIZE) % num_gpus))
local cuda_devs="${gpu_id}"
for (( j=1; j < DECODER_TP_SIZE; j++ )); do
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
done
local http_port=$((DECODE_HTTP_BASE + i))
local pd_port=$((DECODE_PD_BASE + i))
local kv_cfg
kv_cfg=$(build_kv_config "${pd_port}")
echo "Decoder $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
PYTHONHASHSEED=42 \
${VLLM_BIN} serve ${model_name} \
--port ${http_port} \
--enforce-eager \
--block-size ${DECODE_BLOCK_SIZE} \
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
--max-model-len ${MAX_MODEL_LEN} \
--tensor-parallel-size ${DECODER_TP_SIZE} \
--kv-transfer-config '${kv_cfg}'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
BASE_CMD="${BASE_CMD} $arg"
done
fi
eval "${BASE_CMD} &"
DECODE_HOSTS+=("${P2P_HOST}")
DECODE_PORTS+=("${http_port}")
DECODE_PD_PORTS+=("${pd_port}")
done
# ---- Wait for HTTP readiness ----
for port in "${PREFILL_PORTS[@]}"; do
echo "Waiting for prefill instance on port $port to start..."
wait_for_server "$port"
done
for port in "${DECODE_PORTS[@]}"; do
echo "Waiting for decode instance on port $port to start..."
wait_for_server "$port"
done
# ---- Proxy ----
# The proxy currently advertises a single prefiller PD address to decoders.
# For the 1xM and matched NxM common cases the first prefiller's PD coords
# are the right pick; multi-prefiller PD round-robin is a follow-up.
PROXY_CMD="${PYTHON_BIN} ${SCRIPT_DIR}/p2p_connector_proxy.py \
--port ${PROXY_PORT} \
--host ${P2P_HOST} \
--prefiller-hosts ${PREFILL_HOSTS[*]} \
--prefiller-ports ${PREFILL_PORTS[*]} \
--decoder-hosts ${DECODE_HOSTS[*]} \
--decoder-ports ${DECODE_PORTS[*]} \
--p2p-connector-host ${P2P_HOST} \
--p2p-connector-port ${PREFILL_PD_PORTS[0]} \
--decoder-p2p-connector-host ${P2P_HOST} \
--decoder-p2p-connector-port ${DECODE_PD_PORTS[0]}"
if [[ "${DECODER_FIRST}" == "true" ]]; then
PROXY_CMD="${PROXY_CMD} --decoder-first"
fi
echo "Starting proxy: ${PROXY_CMD}"
eval "${PROXY_CMD} &"
sleep 5
# ---- Run accuracy test (reused from nixl_integration) ----
echo "Running tests for $model_name"
TEST_MODEL=$model_name "${PYTHON_BIN}" -m pytest -s -x \
"${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py"
cleanup_instances
sleep 3
}
# ---------------------------------------------------------------------------
# Drive
# ---------------------------------------------------------------------------
for model in "${MODELS[@]}"; do
run_tests_for_model "$model"
done
echo "All tests completed!"
@@ -0,0 +1,330 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for DataTransport base class and NixlTransport."""
from __future__ import annotations
import ctypes
from unittest.mock import MagicMock, patch
import numpy as np
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
# ---------------------------------------------------------------------------
# DataTransport base class tests
# ---------------------------------------------------------------------------
class TestDataTransportBase:
"""Tests for the DataTransport abstract base properties."""
def _make_view(self, num_blocks: int = 8, block_len: int = 1024) -> memoryview:
"""Create a memoryview with the given shape."""
buf = np.zeros((num_blocks, block_len), dtype=np.uint8)
return memoryview(buf)
def test_properties(self):
"""base_addr, num_blocks, block_len are set from memoryview shape."""
view = self._make_view(num_blocks=4, block_len=2048)
# Use NixlTransport (concrete) with NIXL mocked away
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view)
assert transport.num_blocks == 4
assert transport.block_len == 2048
assert transport.base_addr == ctypes.addressof(ctypes.c_char.from_buffer(view))
def test_config_fingerprint_empty_when_no_fields(self):
"""No config fields → empty fingerprint."""
view = self._make_view()
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view, config_fields=None)
assert transport.config_fingerprint == ""
def test_config_fingerprint_deterministic(self):
"""Same config fields → same fingerprint."""
view = self._make_view()
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
t1 = NixlTransport("test:1", view, config_fields=fields)
t2 = NixlTransport("test:2", view, config_fields=fields)
assert t1.config_fingerprint == t2.config_fingerprint
assert len(t1.config_fingerprint) == 16
def test_config_fingerprint_differs_for_different_fields(self):
"""Different config fields → different fingerprint."""
view = self._make_view()
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
t1 = NixlTransport("test:1", view, config_fields={"model": "a"})
t2 = NixlTransport("test:2", view, config_fields={"model": "b"})
assert t1.config_fingerprint != t2.config_fingerprint
# ---------------------------------------------------------------------------
# NixlTransport tests (with mocked NIXL agent)
# ---------------------------------------------------------------------------
class TestNixlTransportWithMockedAgent:
"""Tests for NixlTransport logic with a mocked NIXL agent."""
def _make_transport(self) -> NixlTransport:
"""Create a NixlTransport with mocked NIXL internals."""
view = memoryview(np.zeros((8, 1024), dtype=np.uint8))
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view)
# Manually set up a mock agent after construction
agent = MagicMock()
agent.add_remote_agent.return_value = "nixl-peer-name"
agent.get_xfer_descs.return_value = MagicMock()
agent.prep_xfer_dlist.return_value = MagicMock()
agent.make_prepped_xfer.return_value = MagicMock(name="handle")
agent.transfer.return_value = None
agent.check_xfer_state.return_value = "PROC"
agent.get_agent_metadata.return_value = b"test-metadata"
transport._agent = agent
transport._local_dlist = MagicMock()
return transport
def test_available_false_without_nixl(self):
"""Without NIXL installed, available is False."""
view = memoryview(np.zeros((4, 512), dtype=np.uint8))
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view)
assert transport.available is False
def test_available_true_with_agent(self):
transport = self._make_transport()
assert transport.available is True
def test_get_agent_metadata(self):
transport = self._make_transport()
assert transport.get_agent_metadata() == b"test-metadata"
def test_write_blocks_returns_none_for_unknown_peer(self):
"""write_blocks returns None if peer not registered."""
transport = self._make_transport()
result = transport.write_blocks("unknown:1", [0, 1], [2, 3])
assert result is None
def test_write_blocks_returns_transfer_id(self):
"""write_blocks returns an integer transfer_id on success."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0, 1], [2, 3])
assert tid is not None
assert isinstance(tid, int)
def test_write_blocks_increments_transfer_id(self):
"""Each write_blocks call gets a unique transfer_id."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid1 = transport.write_blocks("peer:1", [0], [1])
tid2 = transport.write_blocks("peer:1", [2], [3])
assert tid1 != tid2
def test_poll_empty_when_no_inflight(self):
"""poll returns empty when nothing is inflight."""
transport = self._make_transport()
result = transport.poll()
assert result == PollResult(done=(), failed=())
def test_poll_returns_done_when_transfer_completes(self):
"""Completed transfer appears in poll().done."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
# Simulate completion
transport._agent.check_xfer_state.return_value = "DONE"
result = transport.poll()
assert tid in result.done
assert result.failed == ()
# Handle released
transport._agent.release_xfer_handle.assert_called()
def test_poll_returns_failed_for_error_state(self):
"""Transfer in error state appears in poll().failed."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = "ERR"
result = transport.poll()
assert result.done == ()
assert tid in result.failed
def test_poll_ignores_in_progress(self):
"""Transfers in PROC/PEND state stay inflight."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = "PROC"
result = transport.poll()
assert result.done == ()
assert result.failed == ()
transport._agent.check_xfer_state.return_value = "PEND"
result = transport.poll()
assert result.done == ()
assert result.failed == ()
def test_cancel_removes_inflight(self):
"""cancel removes transfers and releases handles."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
assert tid in transport._inflight
result = transport.cancel([tid])
assert result == []
assert tid not in transport._inflight
transport._agent.release_xfer_handle.assert_called()
def test_cancel_ignores_unknown_ids(self):
"""cancel with unknown IDs doesn't crash."""
transport = self._make_transport()
assert transport.cancel([999, 1000]) == []
assert transport.cancel([999, 1000], mode="wait") == []
def test_cancel_wait_release_succeeds(self):
"""wait-mode cancel that succeeds pops the entry and returns []."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
assert tid in transport._inflight
result = transport.cancel([tid], mode="wait")
assert result == []
assert tid not in transport._inflight
transport._agent.release_xfer_handle.assert_called_once()
def test_cancel_wait_release_raises(self):
"""wait-mode cancel keeps the entry and returns the tid on raise."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.release_xfer_handle.side_effect = RuntimeError(
"NIXL_ERR_REPOST_ACTIVE"
)
result = transport.cancel([tid], mode="wait")
assert result == [tid]
assert tid in transport._inflight
def test_cancel_wait_then_poll_completes(self):
"""A wait-cancel that left a tid pending later completes via poll."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.release_xfer_handle.side_effect = RuntimeError("busy")
assert transport.cancel([tid], mode="wait") == [tid]
assert tid in transport._inflight
transport._agent.release_xfer_handle.side_effect = None
transport._agent.check_xfer_state.return_value = "DONE"
result = transport.poll()
assert tid in result.done
assert tid not in transport._inflight
def test_add_and_remove_remote_peer(self):
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
assert "peer:1" in transport._remote_dlists
transport.remove_remote_peer("peer:1")
assert "peer:1" not in transport._remote_dlists
transport._agent.release_dlist_handle.assert_called()
transport._agent.remove_remote_agent.assert_called()
def test_close_releases_everything(self):
"""close releases all handles and clears state."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
transport.write_blocks("peer:1", [0], [1])
transport.close()
assert transport._agent is None
assert transport._inflight == {}
assert transport._remote_dlists == {}
# ---------------------------------------------------------------------------
# NIXL agent-config selection
# ---------------------------------------------------------------------------
class TestNixlAgentConfigSelection:
"""Tests that backends/num_threads pick the right nixl_agent_config call.
Mirrors the conditional in
vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py:325-329.
"""
def _make_view(self) -> memoryview:
return memoryview(np.zeros((4, 512), dtype=np.uint8))
def test_non_ucx_backends_passes_backends_kwarg(self):
"""When any non-UCX backend is requested, pass backends + telemetry."""
agent_cls = MagicMock()
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
with (
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
patch(
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
),
):
NixlTransport("test:1", self._make_view(), backends=["MOONCAKE"])
config_fn.assert_called_once_with(backends=["MOONCAKE"], capture_telemetry=True)
# num_threads must NOT be passed on the non-UCX branch.
assert "num_threads" not in config_fn.call_args.kwargs
def test_ucx_only_passes_num_threads(self):
"""UCX-only configuration passes num_threads + telemetry, no backends."""
agent_cls = MagicMock()
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
with (
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
patch(
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
),
):
NixlTransport("test:1", self._make_view(), num_threads=8)
config_fn.assert_called_once_with(num_threads=8, capture_telemetry=True)
assert "backends" not in config_fn.call_args.kwargs
def test_default_backends_is_ucx_only(self):
"""No backends arg → defaults to UCX-only branch."""
agent_cls = MagicMock()
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
with (
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
patch(
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
),
):
NixlTransport("test:1", self._make_view())
# Default num_threads=4, no backends kwarg.
config_fn.assert_called_once_with(num_threads=4, capture_telemetry=True)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for vllm.v1.kv_offload.tiering.p2p.control.zmq."""
from __future__ import annotations
import socket
import time
import pytest
import zmq
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
ZmqConnection,
ZmqTransport,
_Sockets,
)
def _free_port() -> int:
"""Find a free TCP port."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return s.getsockname()[1]
def _make_transport(host: str = "127.0.0.1", attempts: int = 8):
"""Construct a ZmqTransport on a fresh port, retrying on bind collisions.
Why: _free_port() releases the probe socket before ZmqTransport binds the
same port — a parallel test run can steal it in between. Retrying on
ZMQError/OSError closes that race without a production change.
"""
last_err: Exception | None = None
for _ in range(attempts):
port = _free_port()
try:
return ZmqTransport(f"{host}:{port}", host, port), port
except (zmq.ZMQError, OSError) as e:
last_err = e
assert last_err is not None
raise last_err
def _wait_for_inbound(transport: ZmqTransport, deadline: float = 2.0):
"""Poll until at least one new inbound connection is accepted, or fail."""
end = time.monotonic() + deadline
while time.monotonic() < end:
new = transport.poll()
if new:
return new
time.sleep(0.005)
raise AssertionError(f"no inbound connection within {deadline}s")
def _wait_for_messages(
transport: ZmqTransport,
conn: ZmqConnection,
n: int,
deadline: float = 2.0,
) -> list[dict]:
"""Poll until `conn` has received at least `n` messages, then return them."""
end = time.monotonic() + deadline
msgs: list[dict] = []
while time.monotonic() < end:
transport.poll()
msgs.extend(conn.recv())
if len(msgs) >= n:
return msgs
time.sleep(0.005)
raise AssertionError(f"got {len(msgs)}/{n} messages within {deadline}s")
def _make_mock_connection(peer_id: str = "test:1234") -> ZmqConnection:
"""Create a ZmqConnection with mock sockets for unit testing."""
from unittest.mock import MagicMock
sockets = _Sockets(dealer=MagicMock(), monitor=MagicMock())
return ZmqConnection(peer_id, sockets)
class TestZmqConnection:
"""Tests for ZmqConnection in isolation (no real sockets)."""
def test_enqueue_and_recv(self):
"""Messages enqueued are returned by recv() in order."""
conn = _make_mock_connection()
conn.enqueue({"type": "a"})
conn.enqueue({"type": "b"})
msgs = conn.recv()
assert list(msgs) == [{"type": "a"}, {"type": "b"}]
# Second recv is empty
assert not conn.recv()
def test_recv_returns_empty_initially(self):
conn = _make_mock_connection()
assert not conn.recv()
def test_alive_initially_true(self):
conn = _make_mock_connection()
assert conn.alive is True
def test_mark_dead(self):
conn = _make_mock_connection()
conn.mark_dead()
assert conn.alive is False
def test_send_raises_when_closed(self):
conn = _make_mock_connection()
conn.mark_dead()
with pytest.raises(RuntimeError, match="closed connection"):
conn.send({"type": "test"})
class TestZmqTransportConnectivity:
"""Integration tests for ZmqTransport with real ZMQ sockets."""
def test_connect_and_send_message(self):
"""Two transports can connect and exchange messages."""
transport_a, port_a = _make_transport()
transport_b, port_b = _make_transport()
try:
peer_a_id = f"127.0.0.1:{port_a}"
conn_b_to_a = transport_b.connect(peer_a_id)
conn_b_to_a.send({"type": "hello", "data": 42})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
conn_a_from_b = new_conns[0]
assert conn_a_from_b.peer_id == f"127.0.0.1:{port_b}"
msgs = _wait_for_messages(transport_a, conn_a_from_b, 1)
assert msgs == [{"type": "hello", "data": 42}]
finally:
transport_a.close()
transport_b.close()
def test_bidirectional_messaging(self):
"""Both sides can send and receive after connection."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"type": "connect", "from": "b"})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
conn_a = new_conns[0]
conn_a.send({"type": "reply", "from": "a"})
msgs = _wait_for_messages(transport_b, conn_b, 1)
assert msgs == [{"type": "reply", "from": "a"}]
finally:
transport_a.close()
transport_b.close()
def test_poll_returns_empty_when_no_connections(self):
transport, _ = _make_transport()
try:
assert not transport.poll()
finally:
transport.close()
def test_multiple_messages(self):
"""Multiple messages are buffered and returned together."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"seq": 1})
conn_b.send({"seq": 2})
conn_b.send({"seq": 3})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
conn_a = new_conns[0]
msgs = _wait_for_messages(transport_a, conn_a, 3)
assert [m["seq"] for m in msgs] == [1, 2, 3]
finally:
transport_a.close()
transport_b.close()
def test_duplicate_connect_asserts(self):
"""Connecting to the same peer twice raises AssertionError."""
# port_a is never bound — we just need a syntactically-valid peer id.
port_a = _free_port()
transport_b, _ = _make_transport()
try:
transport_b.connect(f"127.0.0.1:{port_a}")
with pytest.raises(AssertionError, match="already exists"):
transport_b.connect(f"127.0.0.1:{port_a}")
finally:
transport_b.close()
def test_dead_connection_removed_on_poll(self):
"""Dead connections are cleaned up during poll."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"type": "hello"})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
# Mark the inbound connection dead manually.
new_conns[0].mark_dead()
# Pruning is synchronous within poll().
transport_a.poll()
assert len(transport_a._connections) == 0
finally:
transport_a.close()
transport_b.close()
def test_close_is_idempotent(self):
"""Calling close() twice doesn't raise."""
transport, _ = _make_transport()
transport.close()
transport.close() # should not raise