Secondary tier implementation for PD disaggregation (#42285)
Signed-off-by: Liran Schour <lirans@il.ibm.com> Signed-off-by: liranschour <liranschour@users.noreply.github.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
Or Ozeri
parent
b5c9e1ac33
commit
bec232a914
@@ -0,0 +1,331 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2PConnector proxy server for OffloadingConnector + TieringOffloadingSpec.
|
||||
|
||||
Unlike NixlConnector (which returns remote_host/remote_port in the prefill
|
||||
response), OffloadingConnector does not embed connector coordinates in its
|
||||
response. This proxy injects the prefiller's P2PConnector address into
|
||||
kv_transfer_params before forwarding the decode request so the decoder knows
|
||||
where to pull KV blocks from.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python p2p_connector_proxy.py \
|
||||
--port 8192 \
|
||||
--prefiller-host 127.0.0.1 --prefiller-port 8100 \
|
||||
--decoder-host 127.0.0.1 --decoder-port 8200 \
|
||||
--p2p-connector-host 127.0.0.1 --p2p-connector-port 7777
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
app.state.prefill_clients = []
|
||||
app.state.decode_clients = []
|
||||
|
||||
for i, (host, port) in enumerate(global_args.prefiller_instances):
|
||||
app.state.prefill_clients.append(
|
||||
{
|
||||
"client": httpx.AsyncClient(
|
||||
timeout=None,
|
||||
base_url=f"http://{host}:{port}/v1",
|
||||
limits=httpx.Limits(
|
||||
max_connections=None, max_keepalive_connections=None
|
||||
),
|
||||
),
|
||||
"host": host,
|
||||
"port": port,
|
||||
"id": i,
|
||||
}
|
||||
)
|
||||
|
||||
for i, (host, port) in enumerate(global_args.decoder_instances):
|
||||
app.state.decode_clients.append(
|
||||
{
|
||||
"client": httpx.AsyncClient(
|
||||
timeout=None,
|
||||
base_url=f"http://{host}:{port}/v1",
|
||||
limits=httpx.Limits(
|
||||
max_connections=None, max_keepalive_connections=None
|
||||
),
|
||||
),
|
||||
"host": host,
|
||||
"port": port,
|
||||
"id": i,
|
||||
}
|
||||
)
|
||||
|
||||
app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients)))
|
||||
app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients)))
|
||||
|
||||
mode = "decoder-first" if global_args.decoder_first else "prefiller-first"
|
||||
pd_host = global_args.p2p_connector_host
|
||||
pd_port = global_args.p2p_connector_port
|
||||
print(
|
||||
f"Proxy ready [{mode}]: "
|
||||
f"{len(app.state.prefill_clients)} prefiller(s), "
|
||||
f"{len(app.state.decode_clients)} decoder(s). "
|
||||
f"P2PConnector at {pd_host}:{pd_port}"
|
||||
)
|
||||
yield
|
||||
|
||||
for ci in app.state.prefill_clients:
|
||||
await ci["client"].aclose()
|
||||
for ci in app.state.decode_clients:
|
||||
await ci["client"].aclose()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--port", type=int, default=8192)
|
||||
p.add_argument("--host", type=str, default="127.0.0.1")
|
||||
p.add_argument("--prefiller-hosts", type=str, nargs="+", default=["127.0.0.1"])
|
||||
p.add_argument("--prefiller-ports", type=int, nargs="+", default=[8100])
|
||||
p.add_argument("--decoder-hosts", type=str, nargs="+", default=["127.0.0.1"])
|
||||
p.add_argument("--decoder-ports", type=int, nargs="+", default=[8200])
|
||||
# P2PConnector coordinates of the prefiller — injected into decode requests.
|
||||
p.add_argument(
|
||||
"--p2p-connector-host",
|
||||
type=str,
|
||||
default="127.0.0.1",
|
||||
help="Host of the prefiller's P2PConnector ZMQ socket",
|
||||
)
|
||||
p.add_argument(
|
||||
"--p2p-connector-port",
|
||||
type=int,
|
||||
default=7777,
|
||||
help="Port of the prefiller's P2PConnector ZMQ socket",
|
||||
)
|
||||
# P2PConnector coordinates of the decoder — injected into prefill requests
|
||||
# so the prefiller's submit_store can resolve the peer to push KV to.
|
||||
p.add_argument(
|
||||
"--decoder-p2p-connector-host",
|
||||
type=str,
|
||||
default="127.0.0.1",
|
||||
help="Host of the decoder's P2PConnector ZMQ socket",
|
||||
)
|
||||
p.add_argument(
|
||||
"--decoder-p2p-connector-port",
|
||||
type=int,
|
||||
default=7778,
|
||||
help="Port of the decoder's P2PConnector ZMQ socket",
|
||||
)
|
||||
p.add_argument(
|
||||
"--decoder-first",
|
||||
action="store_true",
|
||||
help="Send decode request before prefill so decoder is already "
|
||||
"waiting when KV blocks arrive (decoder-first mode)",
|
||||
)
|
||||
args = p.parse_args()
|
||||
if len(args.prefiller_hosts) != len(args.prefiller_ports):
|
||||
raise ValueError("Prefiller host/port count mismatch")
|
||||
if len(args.decoder_hosts) != len(args.decoder_ports):
|
||||
raise ValueError("Decoder host/port count mismatch")
|
||||
args.prefiller_instances = list(zip(args.prefiller_hosts, args.prefiller_ports))
|
||||
args.decoder_instances = list(zip(args.decoder_hosts, args.decoder_ports))
|
||||
return args
|
||||
|
||||
|
||||
def _get_next(app, service: str):
|
||||
if service == "prefill":
|
||||
return app.state.prefill_clients[next(app.state.prefill_iterator)]
|
||||
return app.state.decode_clients[next(app.state.decode_iterator)]
|
||||
|
||||
|
||||
def _auth_headers(request_id: str) -> dict:
|
||||
headers: dict = {"X-Request-Id": request_id}
|
||||
api_key = os.environ.get("OPENAI_API_KEY", "")
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
|
||||
async def _prefill(client_info, endpoint, req_data, request_id):
|
||||
"""Send a prefill-only request (max_tokens=1) to the prefiller."""
|
||||
data = req_data.copy()
|
||||
data["kv_transfer_params"] = {
|
||||
"decode": {
|
||||
"kv_request_id": request_id,
|
||||
},
|
||||
}
|
||||
data["stream"] = False
|
||||
data["max_tokens"] = 1
|
||||
data.pop("max_completion_tokens", None)
|
||||
data.pop("stream_options", None)
|
||||
data.pop("min_tokens", None)
|
||||
data.pop("min_completion_tokens", None)
|
||||
|
||||
headers = _auth_headers(request_id)
|
||||
resp = await client_info["client"].post(endpoint, json=data, headers=headers)
|
||||
resp.raise_for_status()
|
||||
await resp.aread()
|
||||
return resp
|
||||
|
||||
|
||||
async def _stream_decode(client_info, endpoint, req_data, request_id):
|
||||
headers = _auth_headers(request_id)
|
||||
async with client_info["client"].stream(
|
||||
"POST", endpoint, json=req_data, headers=headers
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
yield chunk
|
||||
|
||||
|
||||
async def _handle_completions(api: str, request: Request):
|
||||
try:
|
||||
req_data = await request.json()
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
prefill_client = _get_next(request.app, "prefill")
|
||||
await _prefill(prefill_client, api, req_data, request_id)
|
||||
|
||||
# Inject the prefiller's P2PConnector address so the decoder can pull
|
||||
# KV blocks from it via the P2PConnector transport.
|
||||
req_data["kv_transfer_params"] = {
|
||||
"prefill": {
|
||||
"kv_request_id": request_id,
|
||||
"remote_host": global_args.p2p_connector_host,
|
||||
"remote_port": global_args.p2p_connector_port,
|
||||
},
|
||||
}
|
||||
|
||||
decode_client = _get_next(request.app, "decode")
|
||||
logger.debug("prefill=%s decode=%s", prefill_client, decode_client)
|
||||
|
||||
async def generate():
|
||||
async for chunk in _stream_decode(decode_client, api, req_data, request_id):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(generate(), media_type="application/json")
|
||||
|
||||
except Exception as e:
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
print(f"Proxy error on {api}: {e}")
|
||||
print("".join(traceback.format_exception(*sys.exc_info())))
|
||||
raise
|
||||
|
||||
|
||||
async def _handle_completions_decoder_first(api: str, request: Request):
|
||||
"""Decoder-first mode: send decode request before prefill.
|
||||
|
||||
The decoder establishes its request and starts polling for KV blocks
|
||||
immediately. The prefill is then sent so the prefiller computes and
|
||||
pushes blocks to the already-waiting decoder.
|
||||
"""
|
||||
try:
|
||||
req_data = await request.json()
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
prefill_client = _get_next(request.app, "prefill")
|
||||
decode_client = _get_next(request.app, "decode")
|
||||
|
||||
decode_data = req_data.copy()
|
||||
decode_data["kv_transfer_params"] = {
|
||||
"prefill": {
|
||||
"kv_request_id": request_id,
|
||||
"remote_host": global_args.p2p_connector_host,
|
||||
"remote_port": global_args.p2p_connector_port,
|
||||
},
|
||||
}
|
||||
|
||||
async def generate():
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
async def _run_decode():
|
||||
try:
|
||||
async for chunk in _stream_decode(
|
||||
decode_client, api, decode_data, request_id
|
||||
):
|
||||
await queue.put(("data", chunk))
|
||||
except Exception as exc:
|
||||
await queue.put(("error", exc))
|
||||
finally:
|
||||
await queue.put(("done", None))
|
||||
|
||||
# 1. Start decode request — decoder is now waiting for KV blocks
|
||||
asyncio.create_task(_run_decode())
|
||||
|
||||
# 2. Send prefill — blocks are computed and pushed to the decoder
|
||||
try:
|
||||
await _prefill(prefill_client, api, req_data, request_id)
|
||||
except Exception as exc:
|
||||
logger.warning("decoder-first: prefill failed: %s", exc)
|
||||
|
||||
logger.debug(
|
||||
"decoder-first: prefill done, streaming decode prefill=%s decode=%s",
|
||||
prefill_client,
|
||||
decode_client,
|
||||
)
|
||||
|
||||
# 3. Stream the decode response
|
||||
while True:
|
||||
kind, value = await queue.get()
|
||||
if kind == "done":
|
||||
break
|
||||
if kind == "error":
|
||||
raise value # type: ignore[misc]
|
||||
yield value
|
||||
|
||||
return StreamingResponse(generate(), media_type="application/json")
|
||||
|
||||
except Exception as e:
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
print(f"Proxy error on {api}: {e}")
|
||||
print("".join(traceback.format_exception(*sys.exc_info())))
|
||||
raise
|
||||
|
||||
|
||||
def _route_handler(api: str):
|
||||
if global_args.decoder_first:
|
||||
return lambda req: _handle_completions_decoder_first(api, req)
|
||||
return lambda req: _handle_completions(api, req)
|
||||
|
||||
|
||||
@app.post("/v1/completions")
|
||||
async def completions(request: Request):
|
||||
return await _route_handler("/completions")(request)
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
return await _route_handler("/chat/completions")(request)
|
||||
|
||||
|
||||
@app.get("/healthcheck")
|
||||
async def healthcheck():
|
||||
return {
|
||||
"status": "ok",
|
||||
"prefill_instances": len(app.state.prefill_clients),
|
||||
"decode_instances": len(app.state.decode_clients),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
global global_args
|
||||
global_args = parse_args()
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(app, host=global_args.host, port=global_args.port)
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Accuracy test driver for the p2p connector
|
||||
# (OffloadingConnector + TieringOffloadingSpec + p2p tier).
|
||||
#
|
||||
# Mirrors tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:
|
||||
# brings up N prefillers + M decoders on the local host, fronts them
|
||||
# with p2p_connector_proxy.py, then runs the connector-agnostic
|
||||
# test_accuracy.py (gsm8k via lm_eval) against the proxy.
|
||||
#
|
||||
# Knobs (env vars unless flagged otherwise):
|
||||
# MODEL_NAMES space-separated model list (default: Llama-3.2-1B-Instruct)
|
||||
# NUM_PREFILL_INSTANCES default 1
|
||||
# NUM_DECODE_INSTANCES default 1
|
||||
# PREFILLER_TP_SIZE default 1
|
||||
# DECODER_TP_SIZE default 1
|
||||
# GPU_MEMORY_UTILIZATION default 0.45
|
||||
# MAX_MODEL_LEN default 512
|
||||
# PREFILL_BLOCK_SIZE default 128
|
||||
# DECODE_BLOCK_SIZE default 128
|
||||
# CPU_BYTES default 209715200 (200 MB)
|
||||
# VLLM_SERVE_EXTRA_ARGS comma-separated extra args for vllm serve
|
||||
# --decoder-first toggle decoder-first proxy mode
|
||||
#
|
||||
# Examples:
|
||||
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
|
||||
# NUM_PREFILL_INSTANCES=2 NUM_DECODE_INSTANCES=2 \
|
||||
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
|
||||
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh --decoder-first
|
||||
|
||||
set -xe
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Args
|
||||
# ---------------------------------------------------------------------------
|
||||
DECODER_FIRST="false"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--decoder-first)
|
||||
DECODER_FIRST="true"
|
||||
shift 1
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option $1"
|
||||
echo "Usage: $0 [--decoder-first]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
MODEL_NAMES=${MODEL_NAMES:-}
|
||||
if [[ -n "$MODEL_NAMES" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
MODELS=($MODEL_NAMES)
|
||||
else
|
||||
MODELS=(
|
||||
"meta-llama/Llama-3.2-1B-Instruct"
|
||||
)
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1}
|
||||
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1}
|
||||
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
|
||||
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.45}
|
||||
MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}
|
||||
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
|
||||
DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128}
|
||||
CPU_BYTES=${CPU_BYTES:-209715200}
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
|
||||
# Base ports — per-instance offsets layered on top.
|
||||
PREFILL_HTTP_BASE=8100
|
||||
DECODE_HTTP_BASE=8200
|
||||
PREFILL_PD_BASE=7777
|
||||
DECODE_PD_BASE=$((PREFILL_PD_BASE + NUM_PREFILL_INSTANCES))
|
||||
PROXY_PORT=8192
|
||||
P2P_HOST=127.0.0.1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolve repo root + venv (works in .venv and /workspace/venv pods)
|
||||
# ---------------------------------------------------------------------------
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../../.." && pwd -P)}"
|
||||
|
||||
if [[ -z "${VLLM_BIN:-}" ]]; then
|
||||
if [[ -x "${GIT_ROOT}/.venv/bin/vllm" ]]; then
|
||||
VLLM_BIN="${GIT_ROOT}/.venv/bin/vllm"
|
||||
elif [[ -x "/workspace/venv/bin/vllm" ]]; then
|
||||
VLLM_BIN="/workspace/venv/bin/vllm"
|
||||
else
|
||||
VLLM_BIN="$(command -v vllm)"
|
||||
fi
|
||||
fi
|
||||
if [[ -z "${PYTHON_BIN:-}" ]]; then
|
||||
if [[ -x "${GIT_ROOT}/.venv/bin/python" ]]; then
|
||||
PYTHON_BIN="${GIT_ROOT}/.venv/bin/python"
|
||||
elif [[ -x "/workspace/venv/bin/python" ]]; then
|
||||
PYTHON_BIN="/workspace/venv/bin/python"
|
||||
else
|
||||
PYTHON_BIN="$(command -v python3 || command -v python)"
|
||||
fi
|
||||
fi
|
||||
echo "Using vllm: ${VLLM_BIN}"
|
||||
echo "Using python: ${PYTHON_BIN}"
|
||||
|
||||
SMI_BIN=$(command -v nvidia-smi || command -v rocm-smi || echo "")
|
||||
|
||||
# Trap SIGINT/SIGTERM/EXIT to kill background jobs.
|
||||
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
timeout 1200 bash -c "
|
||||
until curl -s localhost:${port}/v1/completions > /dev/null; do
|
||||
sleep 1
|
||||
done" && return 0 || return 1
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
echo "Cleaning up any running vLLM / proxy instances..."
|
||||
pkill -f "vllm serve" || true
|
||||
pkill -f "p2p_connector_proxy.py" || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
get_num_gpus() {
|
||||
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
|
||||
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
|
||||
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
|
||||
$SMI_BIN -l | grep -c GPU
|
||||
else
|
||||
echo "1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Build the OffloadingConnector kv-transfer-config for a given PD port.
|
||||
# Mirrors deploy_local.sh:131.
|
||||
build_kv_config() {
|
||||
local pd_port=$1
|
||||
printf '{"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":%s,"secondary_tiers":[{"type":"p2p","host":"%s","port":%s}]}}' \
|
||||
"${CPU_BYTES}" "${P2P_HOST}" "${pd_port}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-model run
|
||||
# ---------------------------------------------------------------------------
|
||||
run_tests_for_model() {
|
||||
local model_name=$1
|
||||
echo "================================"
|
||||
echo "Testing model: $model_name"
|
||||
echo " prefillers=${NUM_PREFILL_INSTANCES} (tp=${PREFILLER_TP_SIZE})"
|
||||
echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})"
|
||||
echo " decoder_first=${DECODER_FIRST}"
|
||||
echo "================================"
|
||||
|
||||
PREFILL_HOSTS=()
|
||||
PREFILL_PORTS=()
|
||||
PREFILL_PD_PORTS=()
|
||||
DECODE_HOSTS=()
|
||||
DECODE_PORTS=()
|
||||
DECODE_PD_PORTS=()
|
||||
|
||||
local num_gpus
|
||||
num_gpus=$(get_num_gpus)
|
||||
local next_gpu=0
|
||||
|
||||
# ---- Prefillers ----
|
||||
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
|
||||
local gpu_id=$((i * PREFILLER_TP_SIZE % num_gpus))
|
||||
local cuda_devs="${gpu_id}"
|
||||
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
|
||||
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
|
||||
done
|
||||
next_gpu=$(((gpu_id + PREFILLER_TP_SIZE) % num_gpus))
|
||||
|
||||
local http_port=$((PREFILL_HTTP_BASE + i))
|
||||
local pd_port=$((PREFILL_PD_BASE + i))
|
||||
local kv_cfg
|
||||
kv_cfg=$(build_kv_config "${pd_port}")
|
||||
|
||||
echo "Prefiller $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
|
||||
|
||||
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
|
||||
PYTHONHASHSEED=42 \
|
||||
${VLLM_BIN} serve ${model_name} \
|
||||
--port ${http_port} \
|
||||
--enforce-eager \
|
||||
--block-size ${PREFILL_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--max-model-len ${MAX_MODEL_LEN} \
|
||||
--tensor-parallel-size ${PREFILLER_TP_SIZE} \
|
||||
--kv-transfer-config '${kv_cfg}'"
|
||||
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
eval "${BASE_CMD} &"
|
||||
|
||||
PREFILL_HOSTS+=("${P2P_HOST}")
|
||||
PREFILL_PORTS+=("${http_port}")
|
||||
PREFILL_PD_PORTS+=("${pd_port}")
|
||||
done
|
||||
|
||||
# ---- Decoders ----
|
||||
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
|
||||
local gpu_id=$(((next_gpu + i * DECODER_TP_SIZE) % num_gpus))
|
||||
local cuda_devs="${gpu_id}"
|
||||
for (( j=1; j < DECODER_TP_SIZE; j++ )); do
|
||||
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
|
||||
done
|
||||
|
||||
local http_port=$((DECODE_HTTP_BASE + i))
|
||||
local pd_port=$((DECODE_PD_BASE + i))
|
||||
local kv_cfg
|
||||
kv_cfg=$(build_kv_config "${pd_port}")
|
||||
|
||||
echo "Decoder $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
|
||||
|
||||
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
|
||||
PYTHONHASHSEED=42 \
|
||||
${VLLM_BIN} serve ${model_name} \
|
||||
--port ${http_port} \
|
||||
--enforce-eager \
|
||||
--block-size ${DECODE_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--max-model-len ${MAX_MODEL_LEN} \
|
||||
--tensor-parallel-size ${DECODER_TP_SIZE} \
|
||||
--kv-transfer-config '${kv_cfg}'"
|
||||
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
eval "${BASE_CMD} &"
|
||||
|
||||
DECODE_HOSTS+=("${P2P_HOST}")
|
||||
DECODE_PORTS+=("${http_port}")
|
||||
DECODE_PD_PORTS+=("${pd_port}")
|
||||
done
|
||||
|
||||
# ---- Wait for HTTP readiness ----
|
||||
for port in "${PREFILL_PORTS[@]}"; do
|
||||
echo "Waiting for prefill instance on port $port to start..."
|
||||
wait_for_server "$port"
|
||||
done
|
||||
for port in "${DECODE_PORTS[@]}"; do
|
||||
echo "Waiting for decode instance on port $port to start..."
|
||||
wait_for_server "$port"
|
||||
done
|
||||
|
||||
# ---- Proxy ----
|
||||
# The proxy currently advertises a single prefiller PD address to decoders.
|
||||
# For the 1xM and matched NxM common cases the first prefiller's PD coords
|
||||
# are the right pick; multi-prefiller PD round-robin is a follow-up.
|
||||
PROXY_CMD="${PYTHON_BIN} ${SCRIPT_DIR}/p2p_connector_proxy.py \
|
||||
--port ${PROXY_PORT} \
|
||||
--host ${P2P_HOST} \
|
||||
--prefiller-hosts ${PREFILL_HOSTS[*]} \
|
||||
--prefiller-ports ${PREFILL_PORTS[*]} \
|
||||
--decoder-hosts ${DECODE_HOSTS[*]} \
|
||||
--decoder-ports ${DECODE_PORTS[*]} \
|
||||
--p2p-connector-host ${P2P_HOST} \
|
||||
--p2p-connector-port ${PREFILL_PD_PORTS[0]} \
|
||||
--decoder-p2p-connector-host ${P2P_HOST} \
|
||||
--decoder-p2p-connector-port ${DECODE_PD_PORTS[0]}"
|
||||
|
||||
if [[ "${DECODER_FIRST}" == "true" ]]; then
|
||||
PROXY_CMD="${PROXY_CMD} --decoder-first"
|
||||
fi
|
||||
|
||||
echo "Starting proxy: ${PROXY_CMD}"
|
||||
eval "${PROXY_CMD} &"
|
||||
|
||||
sleep 5
|
||||
|
||||
# ---- Run accuracy test (reused from nixl_integration) ----
|
||||
echo "Running tests for $model_name"
|
||||
TEST_MODEL=$model_name "${PYTHON_BIN}" -m pytest -s -x \
|
||||
"${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py"
|
||||
|
||||
cleanup_instances
|
||||
sleep 3
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Drive
|
||||
# ---------------------------------------------------------------------------
|
||||
for model in "${MODELS[@]}"; do
|
||||
run_tests_for_model "$model"
|
||||
done
|
||||
|
||||
echo "All tests completed!"
|
||||
@@ -0,0 +1,330 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for DataTransport base class and NixlTransport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DataTransport base class tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDataTransportBase:
|
||||
"""Tests for the DataTransport abstract base properties."""
|
||||
|
||||
def _make_view(self, num_blocks: int = 8, block_len: int = 1024) -> memoryview:
|
||||
"""Create a memoryview with the given shape."""
|
||||
buf = np.zeros((num_blocks, block_len), dtype=np.uint8)
|
||||
return memoryview(buf)
|
||||
|
||||
def test_properties(self):
|
||||
"""base_addr, num_blocks, block_len are set from memoryview shape."""
|
||||
view = self._make_view(num_blocks=4, block_len=2048)
|
||||
|
||||
# Use NixlTransport (concrete) with NIXL mocked away
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view)
|
||||
|
||||
assert transport.num_blocks == 4
|
||||
assert transport.block_len == 2048
|
||||
assert transport.base_addr == ctypes.addressof(ctypes.c_char.from_buffer(view))
|
||||
|
||||
def test_config_fingerprint_empty_when_no_fields(self):
|
||||
"""No config fields → empty fingerprint."""
|
||||
view = self._make_view()
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view, config_fields=None)
|
||||
assert transport.config_fingerprint == ""
|
||||
|
||||
def test_config_fingerprint_deterministic(self):
|
||||
"""Same config fields → same fingerprint."""
|
||||
view = self._make_view()
|
||||
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
t1 = NixlTransport("test:1", view, config_fields=fields)
|
||||
t2 = NixlTransport("test:2", view, config_fields=fields)
|
||||
assert t1.config_fingerprint == t2.config_fingerprint
|
||||
assert len(t1.config_fingerprint) == 16
|
||||
|
||||
def test_config_fingerprint_differs_for_different_fields(self):
|
||||
"""Different config fields → different fingerprint."""
|
||||
view = self._make_view()
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
t1 = NixlTransport("test:1", view, config_fields={"model": "a"})
|
||||
t2 = NixlTransport("test:2", view, config_fields={"model": "b"})
|
||||
assert t1.config_fingerprint != t2.config_fingerprint
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NixlTransport tests (with mocked NIXL agent)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNixlTransportWithMockedAgent:
|
||||
"""Tests for NixlTransport logic with a mocked NIXL agent."""
|
||||
|
||||
def _make_transport(self) -> NixlTransport:
|
||||
"""Create a NixlTransport with mocked NIXL internals."""
|
||||
view = memoryview(np.zeros((8, 1024), dtype=np.uint8))
|
||||
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view)
|
||||
|
||||
# Manually set up a mock agent after construction
|
||||
agent = MagicMock()
|
||||
agent.add_remote_agent.return_value = "nixl-peer-name"
|
||||
agent.get_xfer_descs.return_value = MagicMock()
|
||||
agent.prep_xfer_dlist.return_value = MagicMock()
|
||||
agent.make_prepped_xfer.return_value = MagicMock(name="handle")
|
||||
agent.transfer.return_value = None
|
||||
agent.check_xfer_state.return_value = "PROC"
|
||||
agent.get_agent_metadata.return_value = b"test-metadata"
|
||||
|
||||
transport._agent = agent
|
||||
transport._local_dlist = MagicMock()
|
||||
return transport
|
||||
|
||||
def test_available_false_without_nixl(self):
|
||||
"""Without NIXL installed, available is False."""
|
||||
view = memoryview(np.zeros((4, 512), dtype=np.uint8))
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view)
|
||||
assert transport.available is False
|
||||
|
||||
def test_available_true_with_agent(self):
|
||||
transport = self._make_transport()
|
||||
assert transport.available is True
|
||||
|
||||
def test_get_agent_metadata(self):
|
||||
transport = self._make_transport()
|
||||
assert transport.get_agent_metadata() == b"test-metadata"
|
||||
|
||||
def test_write_blocks_returns_none_for_unknown_peer(self):
|
||||
"""write_blocks returns None if peer not registered."""
|
||||
transport = self._make_transport()
|
||||
result = transport.write_blocks("unknown:1", [0, 1], [2, 3])
|
||||
assert result is None
|
||||
|
||||
def test_write_blocks_returns_transfer_id(self):
|
||||
"""write_blocks returns an integer transfer_id on success."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0, 1], [2, 3])
|
||||
assert tid is not None
|
||||
assert isinstance(tid, int)
|
||||
|
||||
def test_write_blocks_increments_transfer_id(self):
|
||||
"""Each write_blocks call gets a unique transfer_id."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid1 = transport.write_blocks("peer:1", [0], [1])
|
||||
tid2 = transport.write_blocks("peer:1", [2], [3])
|
||||
assert tid1 != tid2
|
||||
|
||||
def test_poll_empty_when_no_inflight(self):
|
||||
"""poll returns empty when nothing is inflight."""
|
||||
transport = self._make_transport()
|
||||
result = transport.poll()
|
||||
assert result == PollResult(done=(), failed=())
|
||||
|
||||
def test_poll_returns_done_when_transfer_completes(self):
|
||||
"""Completed transfer appears in poll().done."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
# Simulate completion
|
||||
transport._agent.check_xfer_state.return_value = "DONE"
|
||||
result = transport.poll()
|
||||
|
||||
assert tid in result.done
|
||||
assert result.failed == ()
|
||||
# Handle released
|
||||
transport._agent.release_xfer_handle.assert_called()
|
||||
|
||||
def test_poll_returns_failed_for_error_state(self):
|
||||
"""Transfer in error state appears in poll().failed."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
transport._agent.check_xfer_state.return_value = "ERR"
|
||||
result = transport.poll()
|
||||
|
||||
assert result.done == ()
|
||||
assert tid in result.failed
|
||||
|
||||
def test_poll_ignores_in_progress(self):
|
||||
"""Transfers in PROC/PEND state stay inflight."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
transport._agent.check_xfer_state.return_value = "PROC"
|
||||
result = transport.poll()
|
||||
assert result.done == ()
|
||||
assert result.failed == ()
|
||||
|
||||
transport._agent.check_xfer_state.return_value = "PEND"
|
||||
result = transport.poll()
|
||||
assert result.done == ()
|
||||
assert result.failed == ()
|
||||
|
||||
def test_cancel_removes_inflight(self):
|
||||
"""cancel removes transfers and releases handles."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
assert tid in transport._inflight
|
||||
|
||||
result = transport.cancel([tid])
|
||||
assert result == []
|
||||
assert tid not in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_called()
|
||||
|
||||
def test_cancel_ignores_unknown_ids(self):
|
||||
"""cancel with unknown IDs doesn't crash."""
|
||||
transport = self._make_transport()
|
||||
assert transport.cancel([999, 1000]) == []
|
||||
assert transport.cancel([999, 1000], mode="wait") == []
|
||||
|
||||
def test_cancel_wait_release_succeeds(self):
|
||||
"""wait-mode cancel that succeeds pops the entry and returns []."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
assert tid in transport._inflight
|
||||
|
||||
result = transport.cancel([tid], mode="wait")
|
||||
assert result == []
|
||||
assert tid not in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_called_once()
|
||||
|
||||
def test_cancel_wait_release_raises(self):
|
||||
"""wait-mode cancel keeps the entry and returns the tid on raise."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.release_xfer_handle.side_effect = RuntimeError(
|
||||
"NIXL_ERR_REPOST_ACTIVE"
|
||||
)
|
||||
|
||||
result = transport.cancel([tid], mode="wait")
|
||||
assert result == [tid]
|
||||
assert tid in transport._inflight
|
||||
|
||||
def test_cancel_wait_then_poll_completes(self):
|
||||
"""A wait-cancel that left a tid pending later completes via poll."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.release_xfer_handle.side_effect = RuntimeError("busy")
|
||||
assert transport.cancel([tid], mode="wait") == [tid]
|
||||
assert tid in transport._inflight
|
||||
|
||||
transport._agent.release_xfer_handle.side_effect = None
|
||||
transport._agent.check_xfer_state.return_value = "DONE"
|
||||
|
||||
result = transport.poll()
|
||||
assert tid in result.done
|
||||
assert tid not in transport._inflight
|
||||
|
||||
def test_add_and_remove_remote_peer(self):
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
assert "peer:1" in transport._remote_dlists
|
||||
|
||||
transport.remove_remote_peer("peer:1")
|
||||
assert "peer:1" not in transport._remote_dlists
|
||||
transport._agent.release_dlist_handle.assert_called()
|
||||
transport._agent.remove_remote_agent.assert_called()
|
||||
|
||||
def test_close_releases_everything(self):
|
||||
"""close releases all handles and clears state."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
transport.close()
|
||||
assert transport._agent is None
|
||||
assert transport._inflight == {}
|
||||
assert transport._remote_dlists == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NIXL agent-config selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNixlAgentConfigSelection:
|
||||
"""Tests that backends/num_threads pick the right nixl_agent_config call.
|
||||
|
||||
Mirrors the conditional in
|
||||
vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py:325-329.
|
||||
"""
|
||||
|
||||
def _make_view(self) -> memoryview:
|
||||
return memoryview(np.zeros((4, 512), dtype=np.uint8))
|
||||
|
||||
def test_non_ucx_backends_passes_backends_kwarg(self):
|
||||
"""When any non-UCX backend is requested, pass backends + telemetry."""
|
||||
agent_cls = MagicMock()
|
||||
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
|
||||
with (
|
||||
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
|
||||
patch(
|
||||
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
|
||||
),
|
||||
):
|
||||
NixlTransport("test:1", self._make_view(), backends=["MOONCAKE"])
|
||||
|
||||
config_fn.assert_called_once_with(backends=["MOONCAKE"], capture_telemetry=True)
|
||||
# num_threads must NOT be passed on the non-UCX branch.
|
||||
assert "num_threads" not in config_fn.call_args.kwargs
|
||||
|
||||
def test_ucx_only_passes_num_threads(self):
|
||||
"""UCX-only configuration passes num_threads + telemetry, no backends."""
|
||||
agent_cls = MagicMock()
|
||||
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
|
||||
with (
|
||||
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
|
||||
patch(
|
||||
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
|
||||
),
|
||||
):
|
||||
NixlTransport("test:1", self._make_view(), num_threads=8)
|
||||
|
||||
config_fn.assert_called_once_with(num_threads=8, capture_telemetry=True)
|
||||
assert "backends" not in config_fn.call_args.kwargs
|
||||
|
||||
def test_default_backends_is_ucx_only(self):
|
||||
"""No backends arg → defaults to UCX-only branch."""
|
||||
agent_cls = MagicMock()
|
||||
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
|
||||
with (
|
||||
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
|
||||
patch(
|
||||
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
|
||||
),
|
||||
):
|
||||
NixlTransport("test:1", self._make_view())
|
||||
|
||||
# Default num_threads=4, no backends kwarg.
|
||||
config_fn.assert_called_once_with(num_threads=4, capture_telemetry=True)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for vllm.v1.kv_offload.tiering.p2p.control.zmq."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
|
||||
ZmqConnection,
|
||||
ZmqTransport,
|
||||
_Sockets,
|
||||
)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
"""Find a free TCP port."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _make_transport(host: str = "127.0.0.1", attempts: int = 8):
|
||||
"""Construct a ZmqTransport on a fresh port, retrying on bind collisions.
|
||||
|
||||
Why: _free_port() releases the probe socket before ZmqTransport binds the
|
||||
same port — a parallel test run can steal it in between. Retrying on
|
||||
ZMQError/OSError closes that race without a production change.
|
||||
"""
|
||||
last_err: Exception | None = None
|
||||
for _ in range(attempts):
|
||||
port = _free_port()
|
||||
try:
|
||||
return ZmqTransport(f"{host}:{port}", host, port), port
|
||||
except (zmq.ZMQError, OSError) as e:
|
||||
last_err = e
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
|
||||
def _wait_for_inbound(transport: ZmqTransport, deadline: float = 2.0):
|
||||
"""Poll until at least one new inbound connection is accepted, or fail."""
|
||||
end = time.monotonic() + deadline
|
||||
while time.monotonic() < end:
|
||||
new = transport.poll()
|
||||
if new:
|
||||
return new
|
||||
time.sleep(0.005)
|
||||
raise AssertionError(f"no inbound connection within {deadline}s")
|
||||
|
||||
|
||||
def _wait_for_messages(
|
||||
transport: ZmqTransport,
|
||||
conn: ZmqConnection,
|
||||
n: int,
|
||||
deadline: float = 2.0,
|
||||
) -> list[dict]:
|
||||
"""Poll until `conn` has received at least `n` messages, then return them."""
|
||||
end = time.monotonic() + deadline
|
||||
msgs: list[dict] = []
|
||||
while time.monotonic() < end:
|
||||
transport.poll()
|
||||
msgs.extend(conn.recv())
|
||||
if len(msgs) >= n:
|
||||
return msgs
|
||||
time.sleep(0.005)
|
||||
raise AssertionError(f"got {len(msgs)}/{n} messages within {deadline}s")
|
||||
|
||||
|
||||
def _make_mock_connection(peer_id: str = "test:1234") -> ZmqConnection:
|
||||
"""Create a ZmqConnection with mock sockets for unit testing."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
sockets = _Sockets(dealer=MagicMock(), monitor=MagicMock())
|
||||
return ZmqConnection(peer_id, sockets)
|
||||
|
||||
|
||||
class TestZmqConnection:
|
||||
"""Tests for ZmqConnection in isolation (no real sockets)."""
|
||||
|
||||
def test_enqueue_and_recv(self):
|
||||
"""Messages enqueued are returned by recv() in order."""
|
||||
conn = _make_mock_connection()
|
||||
|
||||
conn.enqueue({"type": "a"})
|
||||
conn.enqueue({"type": "b"})
|
||||
|
||||
msgs = conn.recv()
|
||||
assert list(msgs) == [{"type": "a"}, {"type": "b"}]
|
||||
# Second recv is empty
|
||||
assert not conn.recv()
|
||||
|
||||
def test_recv_returns_empty_initially(self):
|
||||
conn = _make_mock_connection()
|
||||
assert not conn.recv()
|
||||
|
||||
def test_alive_initially_true(self):
|
||||
conn = _make_mock_connection()
|
||||
assert conn.alive is True
|
||||
|
||||
def test_mark_dead(self):
|
||||
conn = _make_mock_connection()
|
||||
conn.mark_dead()
|
||||
assert conn.alive is False
|
||||
|
||||
def test_send_raises_when_closed(self):
|
||||
conn = _make_mock_connection()
|
||||
conn.mark_dead()
|
||||
|
||||
with pytest.raises(RuntimeError, match="closed connection"):
|
||||
conn.send({"type": "test"})
|
||||
|
||||
|
||||
class TestZmqTransportConnectivity:
|
||||
"""Integration tests for ZmqTransport with real ZMQ sockets."""
|
||||
|
||||
def test_connect_and_send_message(self):
|
||||
"""Two transports can connect and exchange messages."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, port_b = _make_transport()
|
||||
|
||||
try:
|
||||
peer_a_id = f"127.0.0.1:{port_a}"
|
||||
conn_b_to_a = transport_b.connect(peer_a_id)
|
||||
conn_b_to_a.send({"type": "hello", "data": 42})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
|
||||
conn_a_from_b = new_conns[0]
|
||||
assert conn_a_from_b.peer_id == f"127.0.0.1:{port_b}"
|
||||
|
||||
msgs = _wait_for_messages(transport_a, conn_a_from_b, 1)
|
||||
assert msgs == [{"type": "hello", "data": 42}]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_bidirectional_messaging(self):
|
||||
"""Both sides can send and receive after connection."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"type": "connect", "from": "b"})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
conn_a = new_conns[0]
|
||||
|
||||
conn_a.send({"type": "reply", "from": "a"})
|
||||
|
||||
msgs = _wait_for_messages(transport_b, conn_b, 1)
|
||||
assert msgs == [{"type": "reply", "from": "a"}]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_poll_returns_empty_when_no_connections(self):
|
||||
transport, _ = _make_transport()
|
||||
try:
|
||||
assert not transport.poll()
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
def test_multiple_messages(self):
|
||||
"""Multiple messages are buffered and returned together."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"seq": 1})
|
||||
conn_b.send({"seq": 2})
|
||||
conn_b.send({"seq": 3})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
conn_a = new_conns[0]
|
||||
|
||||
msgs = _wait_for_messages(transport_a, conn_a, 3)
|
||||
assert [m["seq"] for m in msgs] == [1, 2, 3]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_duplicate_connect_asserts(self):
|
||||
"""Connecting to the same peer twice raises AssertionError."""
|
||||
# port_a is never bound — we just need a syntactically-valid peer id.
|
||||
port_a = _free_port()
|
||||
transport_b, _ = _make_transport()
|
||||
try:
|
||||
transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
with pytest.raises(AssertionError, match="already exists"):
|
||||
transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
finally:
|
||||
transport_b.close()
|
||||
|
||||
def test_dead_connection_removed_on_poll(self):
|
||||
"""Dead connections are cleaned up during poll."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"type": "hello"})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
|
||||
# Mark the inbound connection dead manually.
|
||||
new_conns[0].mark_dead()
|
||||
|
||||
# Pruning is synchronous within poll().
|
||||
transport_a.poll()
|
||||
assert len(transport_a._connections) == 0
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_close_is_idempotent(self):
|
||||
"""Calling close() twice doesn't raise."""
|
||||
transport, _ = _make_transport()
|
||||
transport.close()
|
||||
transport.close() # should not raise
|
||||
Reference in New Issue
Block a user