Secondary tier implementation for PD disaggregation (#42285)
Signed-off-by: Liran Schour <lirans@il.ibm.com> Signed-off-by: liranschour <liranschour@users.noreply.github.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
Or Ozeri
parent
b5c9e1ac33
commit
bec232a914
@@ -120,6 +120,20 @@ To enable KV cache sharing between multiple vLLM instances using the same `root_
|
||||
PYTHONHASHSEED=0 vllm serve ...
|
||||
```
|
||||
|
||||
### P2P (Including P/D)
|
||||
|
||||
The P2P tier (`type: "p2p"`) shares completed KV blocks between vLLM instances over RDMA via NIXL. Each instance binds a control socket on `host:port` and exchanges blocks directly with peers — no shared filesystem required.
|
||||
|
||||
| Key | Required | Default | Notes |
|
||||
| --- | --- | --- | --- |
|
||||
| `type` | yes | — | Must be `p2p`. |
|
||||
| `host` | no | `0.0.0.0` | Address the control socket binds to. |
|
||||
| `port` | no | `7777` | Port for the control socket. Must be reachable from peers. |
|
||||
| `backends` | no | `["UCX"]` | NIXL transport backends. See [NixlConnector Usage Guide](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin) for available backends and selection guidance. |
|
||||
| `num_threads` | no | `4` | NIXL agent worker threads. Only used when `backends` is UCX-only; ignored when any non-UCX backend is requested. |
|
||||
|
||||
The `backends` and `num_threads` options mirror the conditional logic used by [`NixlConnector`](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin): when any non-UCX backend is configured, NIXL is initialised with `backends=...`; otherwise it falls back to a UCX-only agent with the configured `num_threads`. This lets the P2P tier use a different transport (e.g. `MOONCAKE`, `GDS_MT`, `LIBFABRIC`) than the main `NixlConnector` running in the same process.
|
||||
|
||||
## Tuning Tips
|
||||
|
||||
- `cpu_bytes_to_use`: a bigger CPU tier means fewer trips to slower secondary tiers and a higher hit rate. The value is total across all workers, not per-worker. Leave headroom for the rest of the host workload.
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2PConnector proxy server for OffloadingConnector + TieringOffloadingSpec.
|
||||
|
||||
Unlike NixlConnector (which returns remote_host/remote_port in the prefill
|
||||
response), OffloadingConnector does not embed connector coordinates in its
|
||||
response. This proxy injects the prefiller's P2PConnector address into
|
||||
kv_transfer_params before forwarding the decode request so the decoder knows
|
||||
where to pull KV blocks from.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python p2p_connector_proxy.py \
|
||||
--port 8192 \
|
||||
--prefiller-host 127.0.0.1 --prefiller-port 8100 \
|
||||
--decoder-host 127.0.0.1 --decoder-port 8200 \
|
||||
--p2p-connector-host 127.0.0.1 --p2p-connector-port 7777
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
app.state.prefill_clients = []
|
||||
app.state.decode_clients = []
|
||||
|
||||
for i, (host, port) in enumerate(global_args.prefiller_instances):
|
||||
app.state.prefill_clients.append(
|
||||
{
|
||||
"client": httpx.AsyncClient(
|
||||
timeout=None,
|
||||
base_url=f"http://{host}:{port}/v1",
|
||||
limits=httpx.Limits(
|
||||
max_connections=None, max_keepalive_connections=None
|
||||
),
|
||||
),
|
||||
"host": host,
|
||||
"port": port,
|
||||
"id": i,
|
||||
}
|
||||
)
|
||||
|
||||
for i, (host, port) in enumerate(global_args.decoder_instances):
|
||||
app.state.decode_clients.append(
|
||||
{
|
||||
"client": httpx.AsyncClient(
|
||||
timeout=None,
|
||||
base_url=f"http://{host}:{port}/v1",
|
||||
limits=httpx.Limits(
|
||||
max_connections=None, max_keepalive_connections=None
|
||||
),
|
||||
),
|
||||
"host": host,
|
||||
"port": port,
|
||||
"id": i,
|
||||
}
|
||||
)
|
||||
|
||||
app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients)))
|
||||
app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients)))
|
||||
|
||||
mode = "decoder-first" if global_args.decoder_first else "prefiller-first"
|
||||
pd_host = global_args.p2p_connector_host
|
||||
pd_port = global_args.p2p_connector_port
|
||||
print(
|
||||
f"Proxy ready [{mode}]: "
|
||||
f"{len(app.state.prefill_clients)} prefiller(s), "
|
||||
f"{len(app.state.decode_clients)} decoder(s). "
|
||||
f"P2PConnector at {pd_host}:{pd_port}"
|
||||
)
|
||||
yield
|
||||
|
||||
for ci in app.state.prefill_clients:
|
||||
await ci["client"].aclose()
|
||||
for ci in app.state.decode_clients:
|
||||
await ci["client"].aclose()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
|
||||
def parse_args():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--port", type=int, default=8192)
|
||||
p.add_argument("--host", type=str, default="127.0.0.1")
|
||||
p.add_argument("--prefiller-hosts", type=str, nargs="+", default=["127.0.0.1"])
|
||||
p.add_argument("--prefiller-ports", type=int, nargs="+", default=[8100])
|
||||
p.add_argument("--decoder-hosts", type=str, nargs="+", default=["127.0.0.1"])
|
||||
p.add_argument("--decoder-ports", type=int, nargs="+", default=[8200])
|
||||
# P2PConnector coordinates of the prefiller — injected into decode requests.
|
||||
p.add_argument(
|
||||
"--p2p-connector-host",
|
||||
type=str,
|
||||
default="127.0.0.1",
|
||||
help="Host of the prefiller's P2PConnector ZMQ socket",
|
||||
)
|
||||
p.add_argument(
|
||||
"--p2p-connector-port",
|
||||
type=int,
|
||||
default=7777,
|
||||
help="Port of the prefiller's P2PConnector ZMQ socket",
|
||||
)
|
||||
# P2PConnector coordinates of the decoder — injected into prefill requests
|
||||
# so the prefiller's submit_store can resolve the peer to push KV to.
|
||||
p.add_argument(
|
||||
"--decoder-p2p-connector-host",
|
||||
type=str,
|
||||
default="127.0.0.1",
|
||||
help="Host of the decoder's P2PConnector ZMQ socket",
|
||||
)
|
||||
p.add_argument(
|
||||
"--decoder-p2p-connector-port",
|
||||
type=int,
|
||||
default=7778,
|
||||
help="Port of the decoder's P2PConnector ZMQ socket",
|
||||
)
|
||||
p.add_argument(
|
||||
"--decoder-first",
|
||||
action="store_true",
|
||||
help="Send decode request before prefill so decoder is already "
|
||||
"waiting when KV blocks arrive (decoder-first mode)",
|
||||
)
|
||||
args = p.parse_args()
|
||||
if len(args.prefiller_hosts) != len(args.prefiller_ports):
|
||||
raise ValueError("Prefiller host/port count mismatch")
|
||||
if len(args.decoder_hosts) != len(args.decoder_ports):
|
||||
raise ValueError("Decoder host/port count mismatch")
|
||||
args.prefiller_instances = list(zip(args.prefiller_hosts, args.prefiller_ports))
|
||||
args.decoder_instances = list(zip(args.decoder_hosts, args.decoder_ports))
|
||||
return args
|
||||
|
||||
|
||||
def _get_next(app, service: str):
|
||||
if service == "prefill":
|
||||
return app.state.prefill_clients[next(app.state.prefill_iterator)]
|
||||
return app.state.decode_clients[next(app.state.decode_iterator)]
|
||||
|
||||
|
||||
def _auth_headers(request_id: str) -> dict:
|
||||
headers: dict = {"X-Request-Id": request_id}
|
||||
api_key = os.environ.get("OPENAI_API_KEY", "")
|
||||
if api_key:
|
||||
headers["Authorization"] = f"Bearer {api_key}"
|
||||
return headers
|
||||
|
||||
|
||||
async def _prefill(client_info, endpoint, req_data, request_id):
|
||||
"""Send a prefill-only request (max_tokens=1) to the prefiller."""
|
||||
data = req_data.copy()
|
||||
data["kv_transfer_params"] = {
|
||||
"decode": {
|
||||
"kv_request_id": request_id,
|
||||
},
|
||||
}
|
||||
data["stream"] = False
|
||||
data["max_tokens"] = 1
|
||||
data.pop("max_completion_tokens", None)
|
||||
data.pop("stream_options", None)
|
||||
data.pop("min_tokens", None)
|
||||
data.pop("min_completion_tokens", None)
|
||||
|
||||
headers = _auth_headers(request_id)
|
||||
resp = await client_info["client"].post(endpoint, json=data, headers=headers)
|
||||
resp.raise_for_status()
|
||||
await resp.aread()
|
||||
return resp
|
||||
|
||||
|
||||
async def _stream_decode(client_info, endpoint, req_data, request_id):
|
||||
headers = _auth_headers(request_id)
|
||||
async with client_info["client"].stream(
|
||||
"POST", endpoint, json=req_data, headers=headers
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
yield chunk
|
||||
|
||||
|
||||
async def _handle_completions(api: str, request: Request):
|
||||
try:
|
||||
req_data = await request.json()
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
prefill_client = _get_next(request.app, "prefill")
|
||||
await _prefill(prefill_client, api, req_data, request_id)
|
||||
|
||||
# Inject the prefiller's P2PConnector address so the decoder can pull
|
||||
# KV blocks from it via the P2PConnector transport.
|
||||
req_data["kv_transfer_params"] = {
|
||||
"prefill": {
|
||||
"kv_request_id": request_id,
|
||||
"remote_host": global_args.p2p_connector_host,
|
||||
"remote_port": global_args.p2p_connector_port,
|
||||
},
|
||||
}
|
||||
|
||||
decode_client = _get_next(request.app, "decode")
|
||||
logger.debug("prefill=%s decode=%s", prefill_client, decode_client)
|
||||
|
||||
async def generate():
|
||||
async for chunk in _stream_decode(decode_client, api, req_data, request_id):
|
||||
yield chunk
|
||||
|
||||
return StreamingResponse(generate(), media_type="application/json")
|
||||
|
||||
except Exception as e:
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
print(f"Proxy error on {api}: {e}")
|
||||
print("".join(traceback.format_exception(*sys.exc_info())))
|
||||
raise
|
||||
|
||||
|
||||
async def _handle_completions_decoder_first(api: str, request: Request):
|
||||
"""Decoder-first mode: send decode request before prefill.
|
||||
|
||||
The decoder establishes its request and starts polling for KV blocks
|
||||
immediately. The prefill is then sent so the prefiller computes and
|
||||
pushes blocks to the already-waiting decoder.
|
||||
"""
|
||||
try:
|
||||
req_data = await request.json()
|
||||
request_id = str(uuid.uuid4())
|
||||
|
||||
prefill_client = _get_next(request.app, "prefill")
|
||||
decode_client = _get_next(request.app, "decode")
|
||||
|
||||
decode_data = req_data.copy()
|
||||
decode_data["kv_transfer_params"] = {
|
||||
"prefill": {
|
||||
"kv_request_id": request_id,
|
||||
"remote_host": global_args.p2p_connector_host,
|
||||
"remote_port": global_args.p2p_connector_port,
|
||||
},
|
||||
}
|
||||
|
||||
async def generate():
|
||||
queue: asyncio.Queue = asyncio.Queue()
|
||||
|
||||
async def _run_decode():
|
||||
try:
|
||||
async for chunk in _stream_decode(
|
||||
decode_client, api, decode_data, request_id
|
||||
):
|
||||
await queue.put(("data", chunk))
|
||||
except Exception as exc:
|
||||
await queue.put(("error", exc))
|
||||
finally:
|
||||
await queue.put(("done", None))
|
||||
|
||||
# 1. Start decode request — decoder is now waiting for KV blocks
|
||||
asyncio.create_task(_run_decode())
|
||||
|
||||
# 2. Send prefill — blocks are computed and pushed to the decoder
|
||||
try:
|
||||
await _prefill(prefill_client, api, req_data, request_id)
|
||||
except Exception as exc:
|
||||
logger.warning("decoder-first: prefill failed: %s", exc)
|
||||
|
||||
logger.debug(
|
||||
"decoder-first: prefill done, streaming decode prefill=%s decode=%s",
|
||||
prefill_client,
|
||||
decode_client,
|
||||
)
|
||||
|
||||
# 3. Stream the decode response
|
||||
while True:
|
||||
kind, value = await queue.get()
|
||||
if kind == "done":
|
||||
break
|
||||
if kind == "error":
|
||||
raise value # type: ignore[misc]
|
||||
yield value
|
||||
|
||||
return StreamingResponse(generate(), media_type="application/json")
|
||||
|
||||
except Exception as e:
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
print(f"Proxy error on {api}: {e}")
|
||||
print("".join(traceback.format_exception(*sys.exc_info())))
|
||||
raise
|
||||
|
||||
|
||||
def _route_handler(api: str):
|
||||
if global_args.decoder_first:
|
||||
return lambda req: _handle_completions_decoder_first(api, req)
|
||||
return lambda req: _handle_completions(api, req)
|
||||
|
||||
|
||||
@app.post("/v1/completions")
|
||||
async def completions(request: Request):
|
||||
return await _route_handler("/completions")(request)
|
||||
|
||||
|
||||
@app.post("/v1/chat/completions")
|
||||
async def chat_completions(request: Request):
|
||||
return await _route_handler("/chat/completions")(request)
|
||||
|
||||
|
||||
@app.get("/healthcheck")
|
||||
async def healthcheck():
|
||||
return {
|
||||
"status": "ok",
|
||||
"prefill_instances": len(app.state.prefill_clients),
|
||||
"decode_instances": len(app.state.decode_clients),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
global global_args
|
||||
global_args = parse_args()
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(app, host=global_args.host, port=global_args.port)
|
||||
+312
@@ -0,0 +1,312 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Accuracy test driver for the p2p connector
|
||||
# (OffloadingConnector + TieringOffloadingSpec + p2p tier).
|
||||
#
|
||||
# Mirrors tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:
|
||||
# brings up N prefillers + M decoders on the local host, fronts them
|
||||
# with p2p_connector_proxy.py, then runs the connector-agnostic
|
||||
# test_accuracy.py (gsm8k via lm_eval) against the proxy.
|
||||
#
|
||||
# Knobs (env vars unless flagged otherwise):
|
||||
# MODEL_NAMES space-separated model list (default: Llama-3.2-1B-Instruct)
|
||||
# NUM_PREFILL_INSTANCES default 1
|
||||
# NUM_DECODE_INSTANCES default 1
|
||||
# PREFILLER_TP_SIZE default 1
|
||||
# DECODER_TP_SIZE default 1
|
||||
# GPU_MEMORY_UTILIZATION default 0.45
|
||||
# MAX_MODEL_LEN default 512
|
||||
# PREFILL_BLOCK_SIZE default 128
|
||||
# DECODE_BLOCK_SIZE default 128
|
||||
# CPU_BYTES default 209715200 (200 MB)
|
||||
# VLLM_SERVE_EXTRA_ARGS comma-separated extra args for vllm serve
|
||||
# --decoder-first toggle decoder-first proxy mode
|
||||
#
|
||||
# Examples:
|
||||
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
|
||||
# NUM_PREFILL_INSTANCES=2 NUM_DECODE_INSTANCES=2 \
|
||||
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
|
||||
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh --decoder-first
|
||||
|
||||
set -xe
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Args
|
||||
# ---------------------------------------------------------------------------
|
||||
DECODER_FIRST="false"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--decoder-first)
|
||||
DECODER_FIRST="true"
|
||||
shift 1
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option $1"
|
||||
echo "Usage: $0 [--decoder-first]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Models
|
||||
# ---------------------------------------------------------------------------
|
||||
MODEL_NAMES=${MODEL_NAMES:-}
|
||||
if [[ -n "$MODEL_NAMES" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
MODELS=($MODEL_NAMES)
|
||||
else
|
||||
MODELS=(
|
||||
"meta-llama/Llama-3.2-1B-Instruct"
|
||||
)
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1}
|
||||
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1}
|
||||
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
|
||||
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
|
||||
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.45}
|
||||
MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}
|
||||
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
|
||||
DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128}
|
||||
CPU_BYTES=${CPU_BYTES:-209715200}
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
|
||||
# Base ports — per-instance offsets layered on top.
|
||||
PREFILL_HTTP_BASE=8100
|
||||
DECODE_HTTP_BASE=8200
|
||||
PREFILL_PD_BASE=7777
|
||||
DECODE_PD_BASE=$((PREFILL_PD_BASE + NUM_PREFILL_INSTANCES))
|
||||
PROXY_PORT=8192
|
||||
P2P_HOST=127.0.0.1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Resolve repo root + venv (works in .venv and /workspace/venv pods)
|
||||
# ---------------------------------------------------------------------------
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../../.." && pwd -P)}"
|
||||
|
||||
if [[ -z "${VLLM_BIN:-}" ]]; then
|
||||
if [[ -x "${GIT_ROOT}/.venv/bin/vllm" ]]; then
|
||||
VLLM_BIN="${GIT_ROOT}/.venv/bin/vllm"
|
||||
elif [[ -x "/workspace/venv/bin/vllm" ]]; then
|
||||
VLLM_BIN="/workspace/venv/bin/vllm"
|
||||
else
|
||||
VLLM_BIN="$(command -v vllm)"
|
||||
fi
|
||||
fi
|
||||
if [[ -z "${PYTHON_BIN:-}" ]]; then
|
||||
if [[ -x "${GIT_ROOT}/.venv/bin/python" ]]; then
|
||||
PYTHON_BIN="${GIT_ROOT}/.venv/bin/python"
|
||||
elif [[ -x "/workspace/venv/bin/python" ]]; then
|
||||
PYTHON_BIN="/workspace/venv/bin/python"
|
||||
else
|
||||
PYTHON_BIN="$(command -v python3 || command -v python)"
|
||||
fi
|
||||
fi
|
||||
echo "Using vllm: ${VLLM_BIN}"
|
||||
echo "Using python: ${PYTHON_BIN}"
|
||||
|
||||
SMI_BIN=$(command -v nvidia-smi || command -v rocm-smi || echo "")
|
||||
|
||||
# Trap SIGINT/SIGTERM/EXIT to kill background jobs.
|
||||
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
timeout 1200 bash -c "
|
||||
until curl -s localhost:${port}/v1/completions > /dev/null; do
|
||||
sleep 1
|
||||
done" && return 0 || return 1
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
echo "Cleaning up any running vLLM / proxy instances..."
|
||||
pkill -f "vllm serve" || true
|
||||
pkill -f "p2p_connector_proxy.py" || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
get_num_gpus() {
|
||||
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
|
||||
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
|
||||
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
|
||||
$SMI_BIN -l | grep -c GPU
|
||||
else
|
||||
echo "1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Build the OffloadingConnector kv-transfer-config for a given PD port.
|
||||
# Mirrors deploy_local.sh:131.
|
||||
build_kv_config() {
|
||||
local pd_port=$1
|
||||
printf '{"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":%s,"secondary_tiers":[{"type":"p2p","host":"%s","port":%s}]}}' \
|
||||
"${CPU_BYTES}" "${P2P_HOST}" "${pd_port}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-model run
|
||||
# ---------------------------------------------------------------------------
|
||||
run_tests_for_model() {
|
||||
local model_name=$1
|
||||
echo "================================"
|
||||
echo "Testing model: $model_name"
|
||||
echo " prefillers=${NUM_PREFILL_INSTANCES} (tp=${PREFILLER_TP_SIZE})"
|
||||
echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})"
|
||||
echo " decoder_first=${DECODER_FIRST}"
|
||||
echo "================================"
|
||||
|
||||
PREFILL_HOSTS=()
|
||||
PREFILL_PORTS=()
|
||||
PREFILL_PD_PORTS=()
|
||||
DECODE_HOSTS=()
|
||||
DECODE_PORTS=()
|
||||
DECODE_PD_PORTS=()
|
||||
|
||||
local num_gpus
|
||||
num_gpus=$(get_num_gpus)
|
||||
local next_gpu=0
|
||||
|
||||
# ---- Prefillers ----
|
||||
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
|
||||
local gpu_id=$((i * PREFILLER_TP_SIZE % num_gpus))
|
||||
local cuda_devs="${gpu_id}"
|
||||
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
|
||||
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
|
||||
done
|
||||
next_gpu=$(((gpu_id + PREFILLER_TP_SIZE) % num_gpus))
|
||||
|
||||
local http_port=$((PREFILL_HTTP_BASE + i))
|
||||
local pd_port=$((PREFILL_PD_BASE + i))
|
||||
local kv_cfg
|
||||
kv_cfg=$(build_kv_config "${pd_port}")
|
||||
|
||||
echo "Prefiller $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
|
||||
|
||||
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
|
||||
PYTHONHASHSEED=42 \
|
||||
${VLLM_BIN} serve ${model_name} \
|
||||
--port ${http_port} \
|
||||
--enforce-eager \
|
||||
--block-size ${PREFILL_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--max-model-len ${MAX_MODEL_LEN} \
|
||||
--tensor-parallel-size ${PREFILLER_TP_SIZE} \
|
||||
--kv-transfer-config '${kv_cfg}'"
|
||||
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
eval "${BASE_CMD} &"
|
||||
|
||||
PREFILL_HOSTS+=("${P2P_HOST}")
|
||||
PREFILL_PORTS+=("${http_port}")
|
||||
PREFILL_PD_PORTS+=("${pd_port}")
|
||||
done
|
||||
|
||||
# ---- Decoders ----
|
||||
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
|
||||
local gpu_id=$(((next_gpu + i * DECODER_TP_SIZE) % num_gpus))
|
||||
local cuda_devs="${gpu_id}"
|
||||
for (( j=1; j < DECODER_TP_SIZE; j++ )); do
|
||||
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
|
||||
done
|
||||
|
||||
local http_port=$((DECODE_HTTP_BASE + i))
|
||||
local pd_port=$((DECODE_PD_BASE + i))
|
||||
local kv_cfg
|
||||
kv_cfg=$(build_kv_config "${pd_port}")
|
||||
|
||||
echo "Decoder $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
|
||||
|
||||
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
|
||||
PYTHONHASHSEED=42 \
|
||||
${VLLM_BIN} serve ${model_name} \
|
||||
--port ${http_port} \
|
||||
--enforce-eager \
|
||||
--block-size ${DECODE_BLOCK_SIZE} \
|
||||
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
|
||||
--max-model-len ${MAX_MODEL_LEN} \
|
||||
--tensor-parallel-size ${DECODER_TP_SIZE} \
|
||||
--kv-transfer-config '${kv_cfg}'"
|
||||
|
||||
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
|
||||
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
|
||||
for arg in "${extra_args[@]}"; do
|
||||
BASE_CMD="${BASE_CMD} $arg"
|
||||
done
|
||||
fi
|
||||
|
||||
eval "${BASE_CMD} &"
|
||||
|
||||
DECODE_HOSTS+=("${P2P_HOST}")
|
||||
DECODE_PORTS+=("${http_port}")
|
||||
DECODE_PD_PORTS+=("${pd_port}")
|
||||
done
|
||||
|
||||
# ---- Wait for HTTP readiness ----
|
||||
for port in "${PREFILL_PORTS[@]}"; do
|
||||
echo "Waiting for prefill instance on port $port to start..."
|
||||
wait_for_server "$port"
|
||||
done
|
||||
for port in "${DECODE_PORTS[@]}"; do
|
||||
echo "Waiting for decode instance on port $port to start..."
|
||||
wait_for_server "$port"
|
||||
done
|
||||
|
||||
# ---- Proxy ----
|
||||
# The proxy currently advertises a single prefiller PD address to decoders.
|
||||
# For the 1xM and matched NxM common cases the first prefiller's PD coords
|
||||
# are the right pick; multi-prefiller PD round-robin is a follow-up.
|
||||
PROXY_CMD="${PYTHON_BIN} ${SCRIPT_DIR}/p2p_connector_proxy.py \
|
||||
--port ${PROXY_PORT} \
|
||||
--host ${P2P_HOST} \
|
||||
--prefiller-hosts ${PREFILL_HOSTS[*]} \
|
||||
--prefiller-ports ${PREFILL_PORTS[*]} \
|
||||
--decoder-hosts ${DECODE_HOSTS[*]} \
|
||||
--decoder-ports ${DECODE_PORTS[*]} \
|
||||
--p2p-connector-host ${P2P_HOST} \
|
||||
--p2p-connector-port ${PREFILL_PD_PORTS[0]} \
|
||||
--decoder-p2p-connector-host ${P2P_HOST} \
|
||||
--decoder-p2p-connector-port ${DECODE_PD_PORTS[0]}"
|
||||
|
||||
if [[ "${DECODER_FIRST}" == "true" ]]; then
|
||||
PROXY_CMD="${PROXY_CMD} --decoder-first"
|
||||
fi
|
||||
|
||||
echo "Starting proxy: ${PROXY_CMD}"
|
||||
eval "${PROXY_CMD} &"
|
||||
|
||||
sleep 5
|
||||
|
||||
# ---- Run accuracy test (reused from nixl_integration) ----
|
||||
echo "Running tests for $model_name"
|
||||
TEST_MODEL=$model_name "${PYTHON_BIN}" -m pytest -s -x \
|
||||
"${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py"
|
||||
|
||||
cleanup_instances
|
||||
sleep 3
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Drive
|
||||
# ---------------------------------------------------------------------------
|
||||
for model in "${MODELS[@]}"; do
|
||||
run_tests_for_model "$model"
|
||||
done
|
||||
|
||||
echo "All tests completed!"
|
||||
@@ -0,0 +1,330 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for DataTransport base class and NixlTransport."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# DataTransport base class tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDataTransportBase:
|
||||
"""Tests for the DataTransport abstract base properties."""
|
||||
|
||||
def _make_view(self, num_blocks: int = 8, block_len: int = 1024) -> memoryview:
|
||||
"""Create a memoryview with the given shape."""
|
||||
buf = np.zeros((num_blocks, block_len), dtype=np.uint8)
|
||||
return memoryview(buf)
|
||||
|
||||
def test_properties(self):
|
||||
"""base_addr, num_blocks, block_len are set from memoryview shape."""
|
||||
view = self._make_view(num_blocks=4, block_len=2048)
|
||||
|
||||
# Use NixlTransport (concrete) with NIXL mocked away
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view)
|
||||
|
||||
assert transport.num_blocks == 4
|
||||
assert transport.block_len == 2048
|
||||
assert transport.base_addr == ctypes.addressof(ctypes.c_char.from_buffer(view))
|
||||
|
||||
def test_config_fingerprint_empty_when_no_fields(self):
|
||||
"""No config fields → empty fingerprint."""
|
||||
view = self._make_view()
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view, config_fields=None)
|
||||
assert transport.config_fingerprint == ""
|
||||
|
||||
def test_config_fingerprint_deterministic(self):
|
||||
"""Same config fields → same fingerprint."""
|
||||
view = self._make_view()
|
||||
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
t1 = NixlTransport("test:1", view, config_fields=fields)
|
||||
t2 = NixlTransport("test:2", view, config_fields=fields)
|
||||
assert t1.config_fingerprint == t2.config_fingerprint
|
||||
assert len(t1.config_fingerprint) == 16
|
||||
|
||||
def test_config_fingerprint_differs_for_different_fields(self):
|
||||
"""Different config fields → different fingerprint."""
|
||||
view = self._make_view()
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
t1 = NixlTransport("test:1", view, config_fields={"model": "a"})
|
||||
t2 = NixlTransport("test:2", view, config_fields={"model": "b"})
|
||||
assert t1.config_fingerprint != t2.config_fingerprint
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NixlTransport tests (with mocked NIXL agent)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNixlTransportWithMockedAgent:
|
||||
"""Tests for NixlTransport logic with a mocked NIXL agent."""
|
||||
|
||||
def _make_transport(self) -> NixlTransport:
|
||||
"""Create a NixlTransport with mocked NIXL internals."""
|
||||
view = memoryview(np.zeros((8, 1024), dtype=np.uint8))
|
||||
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view)
|
||||
|
||||
# Manually set up a mock agent after construction
|
||||
agent = MagicMock()
|
||||
agent.add_remote_agent.return_value = "nixl-peer-name"
|
||||
agent.get_xfer_descs.return_value = MagicMock()
|
||||
agent.prep_xfer_dlist.return_value = MagicMock()
|
||||
agent.make_prepped_xfer.return_value = MagicMock(name="handle")
|
||||
agent.transfer.return_value = None
|
||||
agent.check_xfer_state.return_value = "PROC"
|
||||
agent.get_agent_metadata.return_value = b"test-metadata"
|
||||
|
||||
transport._agent = agent
|
||||
transport._local_dlist = MagicMock()
|
||||
return transport
|
||||
|
||||
def test_available_false_without_nixl(self):
|
||||
"""Without NIXL installed, available is False."""
|
||||
view = memoryview(np.zeros((4, 512), dtype=np.uint8))
|
||||
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
|
||||
transport = NixlTransport("test:1", view)
|
||||
assert transport.available is False
|
||||
|
||||
def test_available_true_with_agent(self):
|
||||
transport = self._make_transport()
|
||||
assert transport.available is True
|
||||
|
||||
def test_get_agent_metadata(self):
|
||||
transport = self._make_transport()
|
||||
assert transport.get_agent_metadata() == b"test-metadata"
|
||||
|
||||
def test_write_blocks_returns_none_for_unknown_peer(self):
|
||||
"""write_blocks returns None if peer not registered."""
|
||||
transport = self._make_transport()
|
||||
result = transport.write_blocks("unknown:1", [0, 1], [2, 3])
|
||||
assert result is None
|
||||
|
||||
def test_write_blocks_returns_transfer_id(self):
|
||||
"""write_blocks returns an integer transfer_id on success."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0, 1], [2, 3])
|
||||
assert tid is not None
|
||||
assert isinstance(tid, int)
|
||||
|
||||
def test_write_blocks_increments_transfer_id(self):
|
||||
"""Each write_blocks call gets a unique transfer_id."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid1 = transport.write_blocks("peer:1", [0], [1])
|
||||
tid2 = transport.write_blocks("peer:1", [2], [3])
|
||||
assert tid1 != tid2
|
||||
|
||||
def test_poll_empty_when_no_inflight(self):
|
||||
"""poll returns empty when nothing is inflight."""
|
||||
transport = self._make_transport()
|
||||
result = transport.poll()
|
||||
assert result == PollResult(done=(), failed=())
|
||||
|
||||
def test_poll_returns_done_when_transfer_completes(self):
|
||||
"""Completed transfer appears in poll().done."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
# Simulate completion
|
||||
transport._agent.check_xfer_state.return_value = "DONE"
|
||||
result = transport.poll()
|
||||
|
||||
assert tid in result.done
|
||||
assert result.failed == ()
|
||||
# Handle released
|
||||
transport._agent.release_xfer_handle.assert_called()
|
||||
|
||||
def test_poll_returns_failed_for_error_state(self):
|
||||
"""Transfer in error state appears in poll().failed."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
transport._agent.check_xfer_state.return_value = "ERR"
|
||||
result = transport.poll()
|
||||
|
||||
assert result.done == ()
|
||||
assert tid in result.failed
|
||||
|
||||
def test_poll_ignores_in_progress(self):
|
||||
"""Transfers in PROC/PEND state stay inflight."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
transport._agent.check_xfer_state.return_value = "PROC"
|
||||
result = transport.poll()
|
||||
assert result.done == ()
|
||||
assert result.failed == ()
|
||||
|
||||
transport._agent.check_xfer_state.return_value = "PEND"
|
||||
result = transport.poll()
|
||||
assert result.done == ()
|
||||
assert result.failed == ()
|
||||
|
||||
def test_cancel_removes_inflight(self):
|
||||
"""cancel removes transfers and releases handles."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
assert tid in transport._inflight
|
||||
|
||||
result = transport.cancel([tid])
|
||||
assert result == []
|
||||
assert tid not in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_called()
|
||||
|
||||
def test_cancel_ignores_unknown_ids(self):
|
||||
"""cancel with unknown IDs doesn't crash."""
|
||||
transport = self._make_transport()
|
||||
assert transport.cancel([999, 1000]) == []
|
||||
assert transport.cancel([999, 1000], mode="wait") == []
|
||||
|
||||
def test_cancel_wait_release_succeeds(self):
|
||||
"""wait-mode cancel that succeeds pops the entry and returns []."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
assert tid in transport._inflight
|
||||
|
||||
result = transport.cancel([tid], mode="wait")
|
||||
assert result == []
|
||||
assert tid not in transport._inflight
|
||||
transport._agent.release_xfer_handle.assert_called_once()
|
||||
|
||||
def test_cancel_wait_release_raises(self):
|
||||
"""wait-mode cancel keeps the entry and returns the tid on raise."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.release_xfer_handle.side_effect = RuntimeError(
|
||||
"NIXL_ERR_REPOST_ACTIVE"
|
||||
)
|
||||
|
||||
result = transport.cancel([tid], mode="wait")
|
||||
assert result == [tid]
|
||||
assert tid in transport._inflight
|
||||
|
||||
def test_cancel_wait_then_poll_completes(self):
|
||||
"""A wait-cancel that left a tid pending later completes via poll."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
|
||||
tid = transport.write_blocks("peer:1", [0], [1])
|
||||
transport._agent.release_xfer_handle.side_effect = RuntimeError("busy")
|
||||
assert transport.cancel([tid], mode="wait") == [tid]
|
||||
assert tid in transport._inflight
|
||||
|
||||
transport._agent.release_xfer_handle.side_effect = None
|
||||
transport._agent.check_xfer_state.return_value = "DONE"
|
||||
|
||||
result = transport.poll()
|
||||
assert tid in result.done
|
||||
assert tid not in transport._inflight
|
||||
|
||||
def test_add_and_remove_remote_peer(self):
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
assert "peer:1" in transport._remote_dlists
|
||||
|
||||
transport.remove_remote_peer("peer:1")
|
||||
assert "peer:1" not in transport._remote_dlists
|
||||
transport._agent.release_dlist_handle.assert_called()
|
||||
transport._agent.remove_remote_agent.assert_called()
|
||||
|
||||
def test_close_releases_everything(self):
|
||||
"""close releases all handles and clears state."""
|
||||
transport = self._make_transport()
|
||||
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
|
||||
transport.write_blocks("peer:1", [0], [1])
|
||||
|
||||
transport.close()
|
||||
assert transport._agent is None
|
||||
assert transport._inflight == {}
|
||||
assert transport._remote_dlists == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# NIXL agent-config selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestNixlAgentConfigSelection:
|
||||
"""Tests that backends/num_threads pick the right nixl_agent_config call.
|
||||
|
||||
Mirrors the conditional in
|
||||
vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py:325-329.
|
||||
"""
|
||||
|
||||
def _make_view(self) -> memoryview:
|
||||
return memoryview(np.zeros((4, 512), dtype=np.uint8))
|
||||
|
||||
def test_non_ucx_backends_passes_backends_kwarg(self):
|
||||
"""When any non-UCX backend is requested, pass backends + telemetry."""
|
||||
agent_cls = MagicMock()
|
||||
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
|
||||
with (
|
||||
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
|
||||
patch(
|
||||
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
|
||||
),
|
||||
):
|
||||
NixlTransport("test:1", self._make_view(), backends=["MOONCAKE"])
|
||||
|
||||
config_fn.assert_called_once_with(backends=["MOONCAKE"], capture_telemetry=True)
|
||||
# num_threads must NOT be passed on the non-UCX branch.
|
||||
assert "num_threads" not in config_fn.call_args.kwargs
|
||||
|
||||
def test_ucx_only_passes_num_threads(self):
|
||||
"""UCX-only configuration passes num_threads + telemetry, no backends."""
|
||||
agent_cls = MagicMock()
|
||||
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
|
||||
with (
|
||||
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
|
||||
patch(
|
||||
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
|
||||
),
|
||||
):
|
||||
NixlTransport("test:1", self._make_view(), num_threads=8)
|
||||
|
||||
config_fn.assert_called_once_with(num_threads=8, capture_telemetry=True)
|
||||
assert "backends" not in config_fn.call_args.kwargs
|
||||
|
||||
def test_default_backends_is_ucx_only(self):
|
||||
"""No backends arg → defaults to UCX-only branch."""
|
||||
agent_cls = MagicMock()
|
||||
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
|
||||
with (
|
||||
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
|
||||
patch(
|
||||
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
|
||||
),
|
||||
):
|
||||
NixlTransport("test:1", self._make_view())
|
||||
|
||||
# Default num_threads=4, no backends kwarg.
|
||||
config_fn.assert_called_once_with(num_threads=4, capture_telemetry=True)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for vllm.v1.kv_offload.tiering.p2p.control.zmq."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
|
||||
ZmqConnection,
|
||||
ZmqTransport,
|
||||
_Sockets,
|
||||
)
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
"""Find a free TCP port."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _make_transport(host: str = "127.0.0.1", attempts: int = 8):
|
||||
"""Construct a ZmqTransport on a fresh port, retrying on bind collisions.
|
||||
|
||||
Why: _free_port() releases the probe socket before ZmqTransport binds the
|
||||
same port — a parallel test run can steal it in between. Retrying on
|
||||
ZMQError/OSError closes that race without a production change.
|
||||
"""
|
||||
last_err: Exception | None = None
|
||||
for _ in range(attempts):
|
||||
port = _free_port()
|
||||
try:
|
||||
return ZmqTransport(f"{host}:{port}", host, port), port
|
||||
except (zmq.ZMQError, OSError) as e:
|
||||
last_err = e
|
||||
assert last_err is not None
|
||||
raise last_err
|
||||
|
||||
|
||||
def _wait_for_inbound(transport: ZmqTransport, deadline: float = 2.0):
|
||||
"""Poll until at least one new inbound connection is accepted, or fail."""
|
||||
end = time.monotonic() + deadline
|
||||
while time.monotonic() < end:
|
||||
new = transport.poll()
|
||||
if new:
|
||||
return new
|
||||
time.sleep(0.005)
|
||||
raise AssertionError(f"no inbound connection within {deadline}s")
|
||||
|
||||
|
||||
def _wait_for_messages(
|
||||
transport: ZmqTransport,
|
||||
conn: ZmqConnection,
|
||||
n: int,
|
||||
deadline: float = 2.0,
|
||||
) -> list[dict]:
|
||||
"""Poll until `conn` has received at least `n` messages, then return them."""
|
||||
end = time.monotonic() + deadline
|
||||
msgs: list[dict] = []
|
||||
while time.monotonic() < end:
|
||||
transport.poll()
|
||||
msgs.extend(conn.recv())
|
||||
if len(msgs) >= n:
|
||||
return msgs
|
||||
time.sleep(0.005)
|
||||
raise AssertionError(f"got {len(msgs)}/{n} messages within {deadline}s")
|
||||
|
||||
|
||||
def _make_mock_connection(peer_id: str = "test:1234") -> ZmqConnection:
|
||||
"""Create a ZmqConnection with mock sockets for unit testing."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
sockets = _Sockets(dealer=MagicMock(), monitor=MagicMock())
|
||||
return ZmqConnection(peer_id, sockets)
|
||||
|
||||
|
||||
class TestZmqConnection:
|
||||
"""Tests for ZmqConnection in isolation (no real sockets)."""
|
||||
|
||||
def test_enqueue_and_recv(self):
|
||||
"""Messages enqueued are returned by recv() in order."""
|
||||
conn = _make_mock_connection()
|
||||
|
||||
conn.enqueue({"type": "a"})
|
||||
conn.enqueue({"type": "b"})
|
||||
|
||||
msgs = conn.recv()
|
||||
assert list(msgs) == [{"type": "a"}, {"type": "b"}]
|
||||
# Second recv is empty
|
||||
assert not conn.recv()
|
||||
|
||||
def test_recv_returns_empty_initially(self):
|
||||
conn = _make_mock_connection()
|
||||
assert not conn.recv()
|
||||
|
||||
def test_alive_initially_true(self):
|
||||
conn = _make_mock_connection()
|
||||
assert conn.alive is True
|
||||
|
||||
def test_mark_dead(self):
|
||||
conn = _make_mock_connection()
|
||||
conn.mark_dead()
|
||||
assert conn.alive is False
|
||||
|
||||
def test_send_raises_when_closed(self):
|
||||
conn = _make_mock_connection()
|
||||
conn.mark_dead()
|
||||
|
||||
with pytest.raises(RuntimeError, match="closed connection"):
|
||||
conn.send({"type": "test"})
|
||||
|
||||
|
||||
class TestZmqTransportConnectivity:
|
||||
"""Integration tests for ZmqTransport with real ZMQ sockets."""
|
||||
|
||||
def test_connect_and_send_message(self):
|
||||
"""Two transports can connect and exchange messages."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, port_b = _make_transport()
|
||||
|
||||
try:
|
||||
peer_a_id = f"127.0.0.1:{port_a}"
|
||||
conn_b_to_a = transport_b.connect(peer_a_id)
|
||||
conn_b_to_a.send({"type": "hello", "data": 42})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
|
||||
conn_a_from_b = new_conns[0]
|
||||
assert conn_a_from_b.peer_id == f"127.0.0.1:{port_b}"
|
||||
|
||||
msgs = _wait_for_messages(transport_a, conn_a_from_b, 1)
|
||||
assert msgs == [{"type": "hello", "data": 42}]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_bidirectional_messaging(self):
|
||||
"""Both sides can send and receive after connection."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"type": "connect", "from": "b"})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
conn_a = new_conns[0]
|
||||
|
||||
conn_a.send({"type": "reply", "from": "a"})
|
||||
|
||||
msgs = _wait_for_messages(transport_b, conn_b, 1)
|
||||
assert msgs == [{"type": "reply", "from": "a"}]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_poll_returns_empty_when_no_connections(self):
|
||||
transport, _ = _make_transport()
|
||||
try:
|
||||
assert not transport.poll()
|
||||
finally:
|
||||
transport.close()
|
||||
|
||||
def test_multiple_messages(self):
|
||||
"""Multiple messages are buffered and returned together."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"seq": 1})
|
||||
conn_b.send({"seq": 2})
|
||||
conn_b.send({"seq": 3})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
conn_a = new_conns[0]
|
||||
|
||||
msgs = _wait_for_messages(transport_a, conn_a, 3)
|
||||
assert [m["seq"] for m in msgs] == [1, 2, 3]
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_duplicate_connect_asserts(self):
|
||||
"""Connecting to the same peer twice raises AssertionError."""
|
||||
# port_a is never bound — we just need a syntactically-valid peer id.
|
||||
port_a = _free_port()
|
||||
transport_b, _ = _make_transport()
|
||||
try:
|
||||
transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
with pytest.raises(AssertionError, match="already exists"):
|
||||
transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
finally:
|
||||
transport_b.close()
|
||||
|
||||
def test_dead_connection_removed_on_poll(self):
|
||||
"""Dead connections are cleaned up during poll."""
|
||||
transport_a, port_a = _make_transport()
|
||||
transport_b, _ = _make_transport()
|
||||
|
||||
try:
|
||||
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
|
||||
conn_b.send({"type": "hello"})
|
||||
|
||||
new_conns = _wait_for_inbound(transport_a)
|
||||
assert len(new_conns) == 1
|
||||
|
||||
# Mark the inbound connection dead manually.
|
||||
new_conns[0].mark_dead()
|
||||
|
||||
# Pruning is synchronous within poll().
|
||||
transport_a.poll()
|
||||
assert len(transport_a._connections) == 0
|
||||
finally:
|
||||
transport_a.close()
|
||||
transport_b.close()
|
||||
|
||||
def test_close_is_idempotent(self):
|
||||
"""Calling close() twice doesn't raise."""
|
||||
transport, _ = _make_transport()
|
||||
transport.close()
|
||||
transport.close() # should not raise
|
||||
@@ -66,6 +66,12 @@ SecondaryTierFactory.register_tier(
|
||||
"FileSystemTierManager",
|
||||
)
|
||||
|
||||
SecondaryTierFactory.register_tier(
|
||||
"p2p",
|
||||
"vllm.v1.kv_offload.tiering.p2p.manager",
|
||||
"P2PSecondaryTierManager",
|
||||
)
|
||||
|
||||
SecondaryTierFactory.register_tier(
|
||||
"obj",
|
||||
"vllm.v1.kv_offload.tiering.obj.manager",
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import (
|
||||
ControlConnection,
|
||||
ControlTransport,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
|
||||
ZmqConnection,
|
||||
ZmqTransport,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ControlConnection",
|
||||
"ControlTransport",
|
||||
"ZmqConnection",
|
||||
"ZmqTransport",
|
||||
]
|
||||
@@ -0,0 +1,166 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base classes for the P2P control-plane transport.
|
||||
|
||||
The control plane handles peer discovery, connection lifecycle, and
|
||||
message routing. It is message-content agnostic — it moves opaque
|
||||
dicts between peers without interpreting them.
|
||||
|
||||
Architecture
|
||||
------------
|
||||
|
||||
ControlTransport (one per node)
|
||||
├── listen for inbound connections
|
||||
├── connect() to outbound peers
|
||||
└── poll() → new ControlConnections
|
||||
|
||||
ControlConnection (one per peer)
|
||||
├── send(msg) — enqueue a message to the peer
|
||||
├── recv() — drain buffered inbound messages
|
||||
├── mark_dead() — signal that the peer is gone
|
||||
└── close() — tear down the connection
|
||||
|
||||
Threading model: all I/O is driven by the caller invoking poll().
|
||||
No background threads. poll() must be called periodically to:
|
||||
- receive messages (buffered per-connection)
|
||||
- accept new inbound peers
|
||||
- detect disconnections
|
||||
|
||||
Implementor contracts
|
||||
---------------------
|
||||
|
||||
- ControlConnection.send() must not block. Messages are serialized
|
||||
and queued for the next I/O pass.
|
||||
- ControlConnection.recv() returns all messages received since the
|
||||
last call (may be empty). Messages are dicts (already deserialized).
|
||||
- ControlConnection.alive returns False after mark_dead() or close().
|
||||
- ControlTransport.poll() returns newly accepted connections only —
|
||||
not previously returned ones. Each connection appears exactly once.
|
||||
- ControlTransport.connect() creates an outbound connection to a peer
|
||||
identified by peer_id (format: "host:port"). Raises on failure.
|
||||
- Messages may arrive from unknown peers (new inbound connections).
|
||||
The transport creates a ControlConnection and returns it from poll()
|
||||
with the first message(s) already in its recv() buffer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
|
||||
|
||||
class ControlConnection(ABC):
|
||||
"""Bidirectional message channel to a single remote peer.
|
||||
|
||||
Lifecycle:
|
||||
1. Created by ControlTransport (connect() or poll())
|
||||
2. Used for send/recv by sessions
|
||||
3. Marked dead on disconnect (mark_dead())
|
||||
4. Cleaned up with close()
|
||||
|
||||
Once mark_dead() is called, alive becomes False and the owning
|
||||
session should stop using this connection. close() releases
|
||||
underlying resources (sockets, monitors).
|
||||
"""
|
||||
|
||||
def __init__(self, peer_id: str) -> None:
|
||||
self.peer_id = peer_id
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def alive(self) -> bool:
|
||||
"""True if the connection is usable. False after mark_dead/close."""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def send(self, msg: dict) -> None:
|
||||
"""Enqueue a message for delivery to the peer.
|
||||
|
||||
Must not block. Serialization happens internally.
|
||||
Raises on closed connection.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def recv(self) -> Sequence[dict]:
|
||||
"""Drain and return all messages received since the last call.
|
||||
|
||||
Returns an empty sequence if no messages are pending.
|
||||
The returned sequence is read-only — callers must not mutate it.
|
||||
Messages are dicts deserialized from the wire format.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def mark_dead(self) -> None:
|
||||
"""Mark this connection as dead (peer disconnected).
|
||||
|
||||
After this call, alive returns False. The session should
|
||||
stop using this connection and the transport will clean it up.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources (sockets, monitors).
|
||||
|
||||
Idempotent. After close(), alive returns False.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class ControlTransport(ABC):
|
||||
"""Manages peer connections and drives all control-plane I/O.
|
||||
|
||||
Owns the listening socket and all active connections.
|
||||
The caller must invoke poll() periodically to process I/O.
|
||||
|
||||
Lifecycle:
|
||||
1. Constructed with a local identity and listen address
|
||||
2. connect() to reach remote peers
|
||||
3. poll() to accept inbound peers and process messages
|
||||
4. close() to shut down
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def connect(self, peer_id: str) -> ControlConnection:
|
||||
"""Create an outbound connection to a remote peer.
|
||||
|
||||
Args:
|
||||
peer_id: Remote peer identity (format: "host:port").
|
||||
|
||||
Returns:
|
||||
A new ControlConnection ready for send/recv.
|
||||
|
||||
The connection's send queue is live immediately — messages
|
||||
sent before the remote peer's poll() will be buffered.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def poll(self) -> Sequence[ControlConnection]:
|
||||
"""Process all pending I/O and return newly accepted connections.
|
||||
|
||||
This is the main I/O driver. Each call:
|
||||
- Receives messages from all connected peers (buffered in
|
||||
each connection's recv() queue)
|
||||
- Accepts new inbound peers and creates connections for them
|
||||
(first message already in recv() buffer)
|
||||
- Detects disconnections and marks connections dead
|
||||
|
||||
Returns:
|
||||
Newly accepted inbound connections (not previously returned).
|
||||
The returned sequence is read-only — callers must not mutate
|
||||
it. The caller is responsible for creating sessions for
|
||||
these connections.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Shut down the transport and all connections.
|
||||
|
||||
Closes the listening socket and all active connections.
|
||||
Idempotent.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,307 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ZMQ-based transport layer for P2P KV cache sharing.
|
||||
|
||||
Provides ZmqConnection (per-peer messaging) and ZmqTransport (connection
|
||||
management). Message-content agnostic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
import msgspec
|
||||
import zmq
|
||||
import zmq.utils.monitor
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import (
|
||||
ControlConnection,
|
||||
ControlTransport,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HEARTBEAT_IVL_MS = 2000
|
||||
_HEARTBEAT_TIMEOUT_MS = 10000
|
||||
_HEARTBEAT_TTL_MS = 10000
|
||||
|
||||
# Shared sentinels returned when there is nothing to report.
|
||||
_EMPTY_INBOX: tuple[dict, ...] = ()
|
||||
_EMPTY_NEW_CONNECTIONS: tuple[ControlConnection, ...] = ()
|
||||
|
||||
|
||||
def _tcp_addr(host: str, port: int | str) -> str:
|
||||
return f"tcp://{host}:{port}"
|
||||
|
||||
|
||||
def _apply_heartbeat(sock: zmq.Socket) -> None:
|
||||
sock.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
|
||||
sock.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
|
||||
sock.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Sockets:
|
||||
dealer: zmq.Socket
|
||||
monitor: zmq.Socket
|
||||
|
||||
|
||||
class ZmqConnection(ControlConnection):
|
||||
"""Bidirectional message channel to a single remote peer."""
|
||||
|
||||
def __init__(self, peer_id: str, sockets: _Sockets) -> None:
|
||||
super().__init__(peer_id)
|
||||
self._sockets = sockets
|
||||
self._closed = False
|
||||
self._inbox: list[dict] = []
|
||||
|
||||
def send(self, msg: dict) -> None:
|
||||
"""Send a msgpack-encoded message to this peer."""
|
||||
if self._closed:
|
||||
raise RuntimeError(
|
||||
f"ZmqConnection: send on closed connection to {self.peer_id}"
|
||||
)
|
||||
data = msgspec.msgpack.encode(msg)
|
||||
self._sockets.dealer.send(data)
|
||||
|
||||
def recv(self) -> Sequence[dict]:
|
||||
"""Drain and return all buffered incoming messages."""
|
||||
if not self._inbox:
|
||||
return _EMPTY_INBOX
|
||||
msgs = self._inbox
|
||||
self._inbox = []
|
||||
return msgs
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return not self._closed
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
logger.info("ZmqConnection: closing connection to %s", self.peer_id)
|
||||
self._sockets.monitor.close()
|
||||
self._sockets.dealer.close()
|
||||
|
||||
def enqueue(self, msg: dict) -> None:
|
||||
"""Buffer an incoming message."""
|
||||
self._inbox.append(msg)
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
"""Mark connection as disconnected."""
|
||||
self._closed = True
|
||||
|
||||
@property
|
||||
def monitor_socket(self) -> zmq.Socket:
|
||||
"""Monitor socket for disconnect detection (used by ZmqTransport)."""
|
||||
return self._sockets.monitor
|
||||
|
||||
|
||||
class ZmqTransport(ControlTransport):
|
||||
"""ZMQ implementation of ControlTransport.
|
||||
|
||||
Manages a ROUTER socket for accepting connections and DEALER sockets
|
||||
for outbound connections. Message-content agnostic.
|
||||
"""
|
||||
|
||||
def __init__(self, local_id: str, host: str, port: int) -> None:
|
||||
self._local_id = local_id
|
||||
self._closed = False
|
||||
|
||||
self._connections: dict[str, ZmqConnection] = {}
|
||||
self._pending_inbound: list[tuple[str, dict]] = []
|
||||
|
||||
self._zmq_ctx = zmq.Context()
|
||||
self._router: zmq.Socket = self._zmq_ctx.socket(zmq.ROUTER)
|
||||
_apply_heartbeat(self._router)
|
||||
bind_addr = _tcp_addr(host, port)
|
||||
self._router.bind(bind_addr)
|
||||
logger.info("ZmqTransport %s: ROUTER bound on %s", self._local_id, bind_addr)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# ZmqConnection lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def connect(self, peer_id: str) -> ZmqConnection:
|
||||
"""Open an outbound connection to a remote peer."""
|
||||
assert peer_id not in self._connections, (
|
||||
f"ZmqConnection to {peer_id} already exists"
|
||||
)
|
||||
logger.info(
|
||||
"ZmqTransport %s: opening OUTBOUND connection to %s",
|
||||
self._local_id,
|
||||
peer_id,
|
||||
)
|
||||
return self._open_connection(peer_id, direction="outbound")
|
||||
|
||||
def poll(self) -> Sequence[ControlConnection]:
|
||||
"""Process all pending I/O. Returns newly accepted connections.
|
||||
|
||||
- Receives messages (buffered in each connection's inbox)
|
||||
- Creates connections for new inbound peers (connect msg in inbox)
|
||||
- Checks monitors for disconnections
|
||||
- Removes and closes dead connections
|
||||
"""
|
||||
self._recv_router()
|
||||
self._check_monitors()
|
||||
|
||||
# Create connections for new inbound peers
|
||||
new_connections: list[ControlConnection] | None = None
|
||||
for sender_id, msg in self._pending_inbound:
|
||||
conn = self._connections.get(sender_id)
|
||||
if conn is None:
|
||||
logger.info(
|
||||
"ZmqTransport %s: accepting INBOUND connection from %s",
|
||||
self._local_id,
|
||||
sender_id,
|
||||
)
|
||||
conn = self._open_connection(sender_id, direction="inbound")
|
||||
if new_connections is None:
|
||||
new_connections = []
|
||||
new_connections.append(conn)
|
||||
conn.enqueue(msg)
|
||||
self._pending_inbound.clear()
|
||||
|
||||
# Remove dead connections
|
||||
for pid in [p for p, c in self._connections.items() if not c.alive]:
|
||||
self._connections.pop(pid).close()
|
||||
|
||||
return (
|
||||
new_connections if new_connections is not None else _EMPTY_NEW_CONNECTIONS
|
||||
)
|
||||
|
||||
def close(self) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
|
||||
for conn in self._connections.values():
|
||||
conn.close()
|
||||
self._connections.clear()
|
||||
|
||||
self._router.setsockopt(zmq.LINGER, 0)
|
||||
self._router.close()
|
||||
self._zmq_ctx.destroy(linger=0)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _open_connection(
|
||||
self, peer_id: str, direction: str = "outbound"
|
||||
) -> ZmqConnection:
|
||||
"""Create a DEALER socket + monitor and register the connection."""
|
||||
host, port_str = peer_id.rsplit(":", 1)
|
||||
dealer_addr = _tcp_addr(host, port_str)
|
||||
|
||||
logger.debug(
|
||||
"ZmqTransport %s: creating DEALER for %s peer %s -> %s",
|
||||
self._local_id,
|
||||
direction,
|
||||
peer_id,
|
||||
dealer_addr,
|
||||
)
|
||||
|
||||
dealer = self._zmq_ctx.socket(zmq.DEALER)
|
||||
_apply_heartbeat(dealer)
|
||||
dealer.identity = self._local_id.encode()
|
||||
|
||||
safe_id = peer_id.replace(":", "-").replace("/", "-")
|
||||
monitor_addr = f"inproc://p2p-monitor-{safe_id}"
|
||||
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
|
||||
|
||||
monitor_sock = self._zmq_ctx.socket(zmq.PAIR)
|
||||
monitor_sock.connect(monitor_addr)
|
||||
|
||||
dealer.connect(dealer_addr)
|
||||
|
||||
sockets = _Sockets(dealer=dealer, monitor=monitor_sock)
|
||||
conn = ZmqConnection(peer_id, sockets)
|
||||
self._connections[peer_id] = conn
|
||||
logger.info(
|
||||
"ZmqTransport %s: %s connection established to %s (active connections: %d)",
|
||||
self._local_id,
|
||||
direction,
|
||||
peer_id,
|
||||
len(self._connections),
|
||||
)
|
||||
return conn
|
||||
|
||||
def _recv_router(self) -> None:
|
||||
"""Non-blocking: receive all pending messages from ROUTER."""
|
||||
while True:
|
||||
try:
|
||||
frames = self._router.recv_multipart(zmq.NOBLOCK)
|
||||
except zmq.Again:
|
||||
break
|
||||
except zmq.ZMQError as exc:
|
||||
logger.warning("ZmqTransport %s: recv error: %s", self._local_id, exc)
|
||||
break
|
||||
|
||||
if len(frames) != 2:
|
||||
logger.warning(
|
||||
"ZmqTransport %s: dropping message with %d frames (expected 2)",
|
||||
self._local_id,
|
||||
len(frames),
|
||||
)
|
||||
continue
|
||||
|
||||
identity, data = frames
|
||||
sender_id = identity.decode()
|
||||
|
||||
logger.debug(
|
||||
"ZmqTransport %s: ROUTER recv from %s (%d bytes)",
|
||||
self._local_id,
|
||||
sender_id,
|
||||
len(data),
|
||||
)
|
||||
|
||||
try:
|
||||
msg = msgspec.msgpack.decode(data)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"ZmqTransport %s: failed to decode message from %s: %s",
|
||||
self._local_id,
|
||||
sender_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
conn = self._connections.get(sender_id)
|
||||
|
||||
if conn is not None:
|
||||
conn.enqueue(msg)
|
||||
else:
|
||||
self._pending_inbound.append((sender_id, msg))
|
||||
|
||||
def _check_monitors(self) -> None:
|
||||
"""Non-blocking: check all monitor sockets for disconnection."""
|
||||
for conn in self._connections.values():
|
||||
if not conn.alive:
|
||||
continue
|
||||
try:
|
||||
event = zmq.utils.monitor.recv_monitor_message(
|
||||
conn.monitor_socket, zmq.NOBLOCK
|
||||
)
|
||||
except zmq.Again:
|
||||
continue
|
||||
except zmq.ZMQError as exc:
|
||||
logger.warning(
|
||||
"ZmqTransport %s: monitor error for peer %s: %s",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
|
||||
if event["event"] == zmq.EVENT_DISCONNECTED:
|
||||
logger.debug(
|
||||
"ZmqTransport %s: peer %s disconnected",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
)
|
||||
conn.mark_dead()
|
||||
@@ -0,0 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import DataTransport, PollResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
|
||||
|
||||
__all__ = [
|
||||
"DataTransport",
|
||||
"NixlTransport",
|
||||
"PollResult",
|
||||
]
|
||||
@@ -0,0 +1,259 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base class for the P2P data-plane transport.
|
||||
|
||||
The data plane handles RDMA (or similar) block transfers between peers.
|
||||
It is independent of the control plane — the control plane establishes
|
||||
who can talk to whom, then the data plane moves blocks at wire speed.
|
||||
|
||||
Architecture
|
||||
------------
|
||||
|
||||
DataTransport (one per node)
|
||||
├── owns the local KV block memory region
|
||||
├── registers remote peers (add_remote_peer / remove_remote_peer)
|
||||
├── submits block writes (write_blocks → transfer_id)
|
||||
├── polls for completion (poll → done/failed IDs)
|
||||
└── cancels inflight transfers (cancel)
|
||||
|
||||
Memory model
|
||||
------------
|
||||
|
||||
The local node exposes a contiguous block region:
|
||||
|
||||
base_addr ──► ┌─────────────┐ block 0
|
||||
├─────────────┤ block 1
|
||||
├─────────────┤ ...
|
||||
└─────────────┘ block (num_blocks - 1)
|
||||
|
||||
Each block is block_len bytes.
|
||||
|
||||
Remote peers expose the same layout. write_blocks() copies local
|
||||
blocks to a remote peer's block region by index:
|
||||
|
||||
write_blocks("peer:1", local_idxs=[0, 3], remote_idxs=[5, 7])
|
||||
→ writes local block 0 → remote block 5
|
||||
local block 3 → remote block 7
|
||||
|
||||
Transfer lifecycle
|
||||
------------------
|
||||
|
||||
1. Register remote peer: add_remote_peer(peer_id, metadata, ...)
|
||||
- Provides the remote memory layout so transfers can target it
|
||||
2. Submit: write_blocks(peer_id, local_idxs, remote_idxs) → int
|
||||
- Returns a transfer_id (opaque int) for tracking
|
||||
- Returns None if peer not registered or submission fails
|
||||
3. Poll: poll() → PollResult(done=[...], failed=[...])
|
||||
- Returns transfer_ids that completed or failed since last poll
|
||||
- Completed transfers are automatically cleaned up
|
||||
4. Cancel: cancel(transfer_ids, mode="immediate" | "wait")
|
||||
- Best-effort cancellation of inflight transfers
|
||||
- mode="wait" returns ids still in PROC/PEND so the caller can poll
|
||||
them to completion
|
||||
|
||||
Implementor contracts
|
||||
---------------------
|
||||
|
||||
- write_blocks() must not block. The transfer runs asynchronously.
|
||||
- poll() must be called periodically. It drives completion checking.
|
||||
- transfer_ids are unique across the lifetime of the transport.
|
||||
- add_remote_peer() must be called before write_blocks() to that peer.
|
||||
- get_agent_metadata() returns opaque bytes that the remote peer
|
||||
needs to call add_remote_peer() (e.g., RDMA connection info).
|
||||
- config_fingerprint is a content hash of the model configuration.
|
||||
Peers with different fingerprints are incompatible and must not
|
||||
exchange blocks (validated during the control-plane handshake).
|
||||
- close() releases all resources (memory registrations, handles).
|
||||
After close(), no other methods may be called.
|
||||
|
||||
Threading model: no background threads. All I/O driven by poll().
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import hashlib
|
||||
import json
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Literal, NamedTuple
|
||||
|
||||
CancelMode = Literal["immediate", "wait"]
|
||||
|
||||
|
||||
class PollResult(NamedTuple):
|
||||
"""Result of polling inflight transfers.
|
||||
|
||||
Attributes:
|
||||
done: Transfer IDs that completed successfully.
|
||||
failed: Transfer IDs that failed (error, timeout, etc.).
|
||||
"""
|
||||
|
||||
done: Sequence[int]
|
||||
failed: Sequence[int]
|
||||
|
||||
|
||||
class DataTransport(ABC):
|
||||
"""Abstract data-plane transport for RDMA-style block transfers.
|
||||
|
||||
Owns the local KV block memory region and manages transfers to/from
|
||||
registered remote peers.
|
||||
|
||||
Construction:
|
||||
view: A 2D memoryview (num_blocks × block_len bytes) over the
|
||||
local KV cache block storage.
|
||||
config_fields: Dict of model config values used to compute the
|
||||
compatibility fingerprint. None → empty fingerprint
|
||||
(compatible with any peer).
|
||||
"""
|
||||
|
||||
def __init__(self, view: memoryview, config_fields: dict | None = None) -> None:
|
||||
assert view.shape is not None
|
||||
self._view = view
|
||||
self._base_addr = ctypes.addressof(ctypes.c_char.from_buffer(view))
|
||||
self._num_blocks = view.shape[0]
|
||||
self._block_len = view.shape[1]
|
||||
self._config_fingerprint = self._compute_fingerprint(config_fields)
|
||||
|
||||
@property
|
||||
def base_addr(self) -> int:
|
||||
"""Base address of the local block memory region."""
|
||||
return self._base_addr
|
||||
|
||||
@property
|
||||
def num_blocks(self) -> int:
|
||||
"""Number of blocks in the local region."""
|
||||
return self._num_blocks
|
||||
|
||||
@property
|
||||
def block_len(self) -> int:
|
||||
"""Size of each block in bytes."""
|
||||
return self._block_len
|
||||
|
||||
@property
|
||||
def config_fingerprint(self) -> str:
|
||||
"""Content-hash of the model configuration (hex string).
|
||||
|
||||
Peers must have matching fingerprints to exchange blocks.
|
||||
Empty string means no fingerprint (always compatible).
|
||||
"""
|
||||
return self._config_fingerprint
|
||||
|
||||
@staticmethod
|
||||
def _compute_fingerprint(config_fields: dict | None) -> str:
|
||||
if not config_fields:
|
||||
return ""
|
||||
canonical = json.dumps(config_fields, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode()).hexdigest()[:16]
|
||||
|
||||
@abstractmethod
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
"""Return opaque metadata needed by remote peers to connect.
|
||||
|
||||
The returned bytes are sent during the control-plane handshake
|
||||
and passed to the remote peer's add_remote_peer().
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def add_remote_peer(
|
||||
self,
|
||||
peer_id: str,
|
||||
agent_metadata: bytes,
|
||||
base_addr: int,
|
||||
num_blocks: int,
|
||||
block_len: int,
|
||||
) -> None:
|
||||
"""Register a remote peer for block transfers.
|
||||
|
||||
Must be called before write_blocks() to this peer.
|
||||
|
||||
Args:
|
||||
peer_id: Unique identifier for the remote peer.
|
||||
agent_metadata: Opaque bytes from the peer's get_agent_metadata().
|
||||
base_addr: Base address of the peer's block memory region.
|
||||
num_blocks: Number of blocks in the peer's region.
|
||||
block_len: Size of each block (must match local block_len).
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def remove_remote_peer(self, peer_id: str) -> None:
|
||||
"""Unregister a remote peer and release associated resources.
|
||||
|
||||
Inflight transfers to this peer should be cancelled first.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def write_blocks(
|
||||
self,
|
||||
peer_id: str,
|
||||
local_idxs: list[int],
|
||||
remote_idxs: list[int],
|
||||
) -> int | None:
|
||||
"""Submit a WRITE transfer: local blocks → remote peer's blocks.
|
||||
|
||||
Args:
|
||||
peer_id: Target peer (must be registered via add_remote_peer).
|
||||
local_idxs: Indexes of local blocks to read from.
|
||||
remote_idxs: Indexes of remote blocks to write to.
|
||||
Must be same length as local_idxs.
|
||||
|
||||
Returns:
|
||||
A unique transfer_id (int) to track this transfer, or
|
||||
None if the peer is not registered or submission failed.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def poll(self) -> PollResult:
|
||||
"""Poll all inflight transfers for completion.
|
||||
|
||||
Returns:
|
||||
PollResult with lists of completed and failed transfer_ids.
|
||||
Completed/failed transfers are removed from the inflight set.
|
||||
|
||||
Must be called periodically to drive progress checking.
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def cancel(
|
||||
self,
|
||||
transfer_ids: Iterable[int],
|
||||
mode: CancelMode = "immediate",
|
||||
) -> list[int]:
|
||||
"""Cancel inflight transfers by their IDs.
|
||||
|
||||
Best-effort: transfers that already completed are ignored.
|
||||
|
||||
Args:
|
||||
transfer_ids: IDs to cancel. Unknown IDs are ignored.
|
||||
mode:
|
||||
"immediate" (default): pop and release each handle and
|
||||
return []. Matches the legacy fire-and-forget
|
||||
behavior — the caller does not wait for the
|
||||
underlying transfer to drain.
|
||||
"wait": attempt to release each handle. If the release
|
||||
cannot complete because the transfer is still
|
||||
PROC/PEND, the entry stays in the inflight set and
|
||||
its id is included in the returned list. The
|
||||
caller is expected to keep calling poll() until
|
||||
every returned id surfaces in done/failed.
|
||||
|
||||
Returns:
|
||||
For mode="wait", the subset of *transfer_ids* still
|
||||
tracked as inflight after the cancel attempt. For
|
||||
mode="immediate", always [].
|
||||
"""
|
||||
...
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources (registrations, handles, memory).
|
||||
|
||||
Cancels any remaining inflight transfers. Idempotent.
|
||||
After close(), no other methods may be called.
|
||||
"""
|
||||
...
|
||||
@@ -0,0 +1,297 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import itertools
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
from vllm.distributed.nixl_utils import NixlWrapper as _NixlAgent
|
||||
from vllm.distributed.nixl_utils import nixl_agent_config as _NixlAgentConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.p2p.data.base import (
|
||||
CancelMode,
|
||||
DataTransport,
|
||||
PollResult,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Shared sentinel returned by poll() in the steady state (no inflight, or
|
||||
# no transfer changed state since the last poll). Tuples make it immutable;
|
||||
# callers only iterate / membership-test / equality-check.
|
||||
_EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=())
|
||||
|
||||
|
||||
class NixlTransport(DataTransport):
|
||||
"""Manages a NIXL agent, memory registration, and block transfers.
|
||||
|
||||
Wraps the NIXL C library behind a Python interface so the rest of the
|
||||
P2P tier code never touches NIXL types directly. Tracks inflight
|
||||
handles internally and returns completed/failed tags on poll.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
local_id: str,
|
||||
view: memoryview,
|
||||
config_fields: dict | None = None,
|
||||
backends: list[str] | None = None,
|
||||
num_threads: int = 4,
|
||||
) -> None:
|
||||
super().__init__(view, config_fields=config_fields)
|
||||
self._local_id = local_id
|
||||
self._backends = list(backends) if backends else ["UCX"]
|
||||
self._num_threads = num_threads
|
||||
self._agent: Any = None
|
||||
self._reg: Any = None
|
||||
self._local_dlist: Any = None
|
||||
self._remote_dlists: dict[str, object] = {}
|
||||
self._peer_nixl_names: dict[str, str] = {}
|
||||
self._inflight: dict[int, object] = {} # transfer_id → handle
|
||||
self._next_id = itertools.count()
|
||||
|
||||
self._init(view)
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._agent is not None
|
||||
|
||||
def _init(self, view: memoryview) -> None:
|
||||
if _NixlAgent is None:
|
||||
return
|
||||
|
||||
non_ucx_backends = [b for b in self._backends if b != "UCX"]
|
||||
if non_ucx_backends:
|
||||
cfg = _NixlAgentConfig(backends=self._backends, capture_telemetry=True)
|
||||
logger.info(
|
||||
"NixlTransport %s: NIXL backends=%s",
|
||||
self._local_id,
|
||||
self._backends,
|
||||
)
|
||||
else:
|
||||
cfg = _NixlAgentConfig(
|
||||
num_threads=self._num_threads, capture_telemetry=True
|
||||
)
|
||||
logger.info(
|
||||
"NixlTransport %s: NIXL backends=[UCX] num_threads=%d",
|
||||
self._local_id,
|
||||
self._num_threads,
|
||||
)
|
||||
self._agent = _NixlAgent(self._local_id, cfg)
|
||||
|
||||
total_size = self._num_blocks * self._block_len
|
||||
reg_descs = [(self._base_addr, total_size, 0, "")]
|
||||
self._reg = self._agent.register_memory(reg_descs, mem_type="DRAM")
|
||||
|
||||
block_tuples = [
|
||||
(self._base_addr + i * self._block_len, self._block_len, 0)
|
||||
for i in range(self._num_blocks)
|
||||
]
|
||||
xfer_dlist = self._agent.get_xfer_descs(block_tuples, mem_type="DRAM")
|
||||
self._local_dlist = self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", xfer_dlist)
|
||||
logger.info(
|
||||
"NixlTransport %s: registered %d blocks", self._local_id, self._num_blocks
|
||||
)
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
assert self._agent is not None
|
||||
return self._agent.get_agent_metadata()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Peer management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_remote_peer(
|
||||
self,
|
||||
peer_id: str,
|
||||
agent_metadata: bytes,
|
||||
base_addr: int,
|
||||
num_blocks: int,
|
||||
block_len: int,
|
||||
) -> None:
|
||||
nixl_name = self._agent.add_remote_agent(agent_metadata)
|
||||
block_descs = [
|
||||
(base_addr + i * block_len, block_len, 0) for i in range(num_blocks)
|
||||
]
|
||||
xfer_dlist = self._agent.get_xfer_descs(block_descs, mem_type="DRAM")
|
||||
remote_dlist = self._agent.prep_xfer_dlist(nixl_name, xfer_dlist)
|
||||
self._peer_nixl_names[peer_id] = nixl_name
|
||||
self._remote_dlists[peer_id] = remote_dlist
|
||||
|
||||
def remove_remote_peer(self, peer_id: str) -> None:
|
||||
nixl_name = self._peer_nixl_names.pop(peer_id, None)
|
||||
dlist = self._remote_dlists.pop(peer_id, None)
|
||||
if self._agent is not None:
|
||||
if dlist is not None:
|
||||
self._agent.release_dlist_handle(dlist)
|
||||
if nixl_name:
|
||||
self._agent.remove_remote_agent(nixl_name)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Transfer submission and polling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def write_blocks(
|
||||
self,
|
||||
peer_id: str,
|
||||
local_idxs: list[int],
|
||||
remote_idxs: list[int],
|
||||
) -> int | None:
|
||||
"""Submit a WRITE transfer to *peer_id*.
|
||||
|
||||
Returns a transfer ID, or None if the peer is not registered.
|
||||
The ID is returned via poll() when the transfer completes or fails.
|
||||
"""
|
||||
remote_dlist = self._remote_dlists.get(peer_id)
|
||||
if remote_dlist is None:
|
||||
logger.warning(
|
||||
"NixlTransport %s: write_blocks NO REMOTE DLIST for peer=%s "
|
||||
"(known peers=%s)",
|
||||
self._local_id,
|
||||
peer_id,
|
||||
list(self._remote_dlists.keys()),
|
||||
)
|
||||
return None
|
||||
logger.debug(
|
||||
"NixlTransport %s: write_blocks NIXL.transfer peer=%s blocks=%d",
|
||||
self._local_id,
|
||||
peer_id,
|
||||
len(local_idxs),
|
||||
)
|
||||
handle = self._agent.make_prepped_xfer(
|
||||
"WRITE",
|
||||
self._local_dlist,
|
||||
local_idxs,
|
||||
remote_dlist,
|
||||
remote_idxs,
|
||||
)
|
||||
self._agent.transfer(handle)
|
||||
transfer_id = next(self._next_id)
|
||||
self._inflight[transfer_id] = handle
|
||||
return transfer_id
|
||||
|
||||
def poll(self) -> PollResult:
|
||||
"""Poll all inflight transfers.
|
||||
|
||||
Returns PollResult(done=..., failed=...) with transfer IDs.
|
||||
Completed handles are released automatically.
|
||||
"""
|
||||
if not self._inflight:
|
||||
return _EMPTY_POLL_RESULT
|
||||
|
||||
done_ids: list[int] | None = None
|
||||
failed_ids: list[int] | None = None
|
||||
|
||||
for transfer_id, handle in self._inflight.items():
|
||||
try:
|
||||
state = self._agent.check_xfer_state(handle)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"NixlTransport %s: check_xfer_state failed for transfer_id=%d: %s",
|
||||
self._local_id,
|
||||
transfer_id,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
if state == "DONE":
|
||||
if done_ids is None:
|
||||
done_ids = []
|
||||
done_ids.append(transfer_id)
|
||||
elif state not in ("PROC", "PEND"):
|
||||
if failed_ids is None:
|
||||
failed_ids = []
|
||||
failed_ids.append(transfer_id)
|
||||
|
||||
if done_ids is None and failed_ids is None:
|
||||
return _EMPTY_POLL_RESULT
|
||||
|
||||
handles_to_release = []
|
||||
for tid in done_ids or ():
|
||||
handles_to_release.append(self._inflight.pop(tid))
|
||||
for tid in failed_ids or ():
|
||||
handles_to_release.append(self._inflight.pop(tid))
|
||||
self._release_handles(handles_to_release)
|
||||
|
||||
return PollResult(
|
||||
done=done_ids if done_ids is not None else _EMPTY_POLL_RESULT.done,
|
||||
failed=failed_ids if failed_ids is not None else _EMPTY_POLL_RESULT.failed,
|
||||
)
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
transfer_ids: Iterable[int],
|
||||
mode: CancelMode = "immediate",
|
||||
) -> list[int]:
|
||||
"""Cancel inflight transfers by their IDs.
|
||||
|
||||
See ``DataTransport.cancel`` for the contract. In "wait" mode,
|
||||
transfers whose ``release_xfer_handle`` raises (NIXL could not
|
||||
complete the abort because the backend is still draining) stay
|
||||
in ``self._inflight`` so a later ``poll()`` will observe them.
|
||||
"""
|
||||
if mode == "immediate":
|
||||
handles = [
|
||||
self._inflight.pop(tid) for tid in transfer_ids if tid in self._inflight
|
||||
]
|
||||
self._release_handles(handles)
|
||||
return []
|
||||
|
||||
still_inflight: list[int] = []
|
||||
for tid in transfer_ids:
|
||||
handle = self._inflight.get(tid)
|
||||
if handle is None:
|
||||
continue
|
||||
try:
|
||||
self._agent.release_xfer_handle(handle)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"NixlTransport %s: cancel pending for transfer_id=%d: %s",
|
||||
self._local_id,
|
||||
tid,
|
||||
exc,
|
||||
)
|
||||
still_inflight.append(tid)
|
||||
continue
|
||||
del self._inflight[tid]
|
||||
return still_inflight
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def close(self) -> None:
|
||||
if self._agent is None:
|
||||
return
|
||||
self._release_handles(list(self._inflight.values()))
|
||||
self._inflight.clear()
|
||||
for peer_id in list(self._remote_dlists):
|
||||
self.remove_remote_peer(peer_id)
|
||||
if self._local_dlist is not None:
|
||||
self._agent.release_dlist_handle(self._local_dlist)
|
||||
self._local_dlist = None
|
||||
if self._reg is not None:
|
||||
self._agent.deregister_memory(self._reg)
|
||||
self._reg = None
|
||||
self._agent = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _release_handles(self, handles: list[object]) -> None:
|
||||
if self._agent is None:
|
||||
return
|
||||
for handle in handles:
|
||||
try:
|
||||
self._agent.release_xfer_handle(handle)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"NixlTransport %s: release_xfer_handle failed: %s",
|
||||
self._local_id,
|
||||
exc,
|
||||
)
|
||||
@@ -0,0 +1,664 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2PSecondaryTierManager: Secondary tier for P2P KV cache sharing.
|
||||
|
||||
Owns transports and a single bidirectional P2PSession per remote peer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport
|
||||
from vllm.v1.kv_offload.tiering.p2p.session import P2PSession
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.base import OffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Reap unbound store batches that have been parked without a FetchMsg
|
||||
# binding them to a session for longer than this. Protects against the
|
||||
# prefiller buffering blocks for a decoder that never asks (decoder died,
|
||||
# network partition, lost kv_request_id). Must be longer than the per-store
|
||||
# deadline so the store-timeout path fires first for individual jobs.
|
||||
_UNBOUND_STORE_TIMEOUT_S = 60.0
|
||||
|
||||
# Time we wait during shutdown for inflight transfers to drain via
|
||||
# cancel(mode="wait") before falling back to mode="immediate". Bounded
|
||||
# so a wedged peer can't hang shutdown.
|
||||
_SHUTDOWN_DRAIN_TIMEOUT_S = 3.0
|
||||
|
||||
# Sleep between iterations of the bounded drain loops in drain_jobs() and
|
||||
# _drain_inflight_for_shutdown(). Short enough to keep latency low, long
|
||||
# enough to avoid busy-spinning the scheduler thread.
|
||||
_DRAIN_SLEEP_S = 0.001
|
||||
|
||||
|
||||
def _prefill_params(kv_params: dict | None) -> dict | None:
|
||||
"""Return the ``prefill`` sub-dict, or None if absent.
|
||||
|
||||
Set on decoder requests; carries kv_request_id, remote_host, remote_port.
|
||||
"""
|
||||
if not kv_params:
|
||||
return None
|
||||
return kv_params.get("prefill")
|
||||
|
||||
|
||||
def _decode_params(kv_params: dict | None) -> dict | None:
|
||||
"""Return the ``decode`` sub-dict, or None if absent.
|
||||
|
||||
Set on prefiller requests; carries kv_request_id.
|
||||
"""
|
||||
if not kv_params:
|
||||
return None
|
||||
return kv_params.get("decode")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _UnboundStoreBatch:
|
||||
"""A submit_store batch parked at the manager before any peer has fetched.
|
||||
|
||||
Indexed by kv_request_id only — the prefiller no longer learns the peer
|
||||
identity at store time. When a FetchMsg(kv_request_id) arrives on some
|
||||
session, the manager binds the kv_request_id to that session and replays
|
||||
every parked batch into ServerRole via session.add_stored_blocks.
|
||||
"""
|
||||
|
||||
job_id: int
|
||||
keys: list[OffloadKey]
|
||||
block_ids: Sequence[int]
|
||||
submitted_at: float = field(default_factory=time.monotonic)
|
||||
|
||||
|
||||
class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
"""Secondary tier for P2P KV cache sharing.
|
||||
|
||||
A single P2PSession per remote peer handles both client-role (loading
|
||||
blocks from the peer) and server-role (serving blocks to the peer)
|
||||
over the same control connection.
|
||||
|
||||
Single-threaded: every public method runs on the scheduler thread, and
|
||||
the engine drives polling via ``get_finished_jobs()`` once per step.
|
||||
``has_pending_work()`` keeps the engine ticking so the control transport
|
||||
and existing sessions are polled even when no requests are scheduled.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: OffloadingSpec,
|
||||
primary_kv_view: memoryview,
|
||||
tier_type: str = "p2p",
|
||||
host: str = "0.0.0.0",
|
||||
port: int = 7777,
|
||||
backends: list[str] | None = None,
|
||||
num_threads: int = 4,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""Initialize the P2P secondary tier manager.
|
||||
|
||||
All keyword arguments after ``primary_kv_view`` come from the
|
||||
``secondary_tiers`` entry in ``kv_connector_extra_config``. See
|
||||
``docs/features/kv_offloading_usage.md`` for the user-facing
|
||||
configuration reference.
|
||||
|
||||
Args:
|
||||
offloading_spec: Owning ``OffloadingSpec`` (provides
|
||||
``vllm_config`` and the offloaded block layout).
|
||||
primary_kv_view: Memoryview over the CPU primary tier; the
|
||||
NIXL agent registers this region for RDMA transfers.
|
||||
tier_type: Tier identifier (defaults to ``"p2p"``).
|
||||
host: Address the ZMQ control socket binds to.
|
||||
port: Port for the ZMQ control socket. Must be reachable
|
||||
from peers.
|
||||
backends: NIXL transport backends (e.g. ``["UCX"]``,
|
||||
``["MOONCAKE"]``, ``["LIBFABRIC"]``). Defaults to
|
||||
``["UCX"]``. When any non-UCX backend is requested, the
|
||||
NIXL agent is initialized with ``backends=...``;
|
||||
otherwise it falls back to a UCX-only agent with
|
||||
``num_threads`` threads.
|
||||
num_threads: NIXL agent worker threads for the UCX-only
|
||||
branch. Ignored when ``backends`` contains a non-UCX
|
||||
entry.
|
||||
**kwargs: Reserved for future tier-specific options.
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
port = int(port)
|
||||
self._local_id = f"{host}:{port}"
|
||||
|
||||
config_fields = FileMapper.from_offloading_spec(
|
||||
root_dir="",
|
||||
offloading_spec=offloading_spec,
|
||||
gpu_blocks_per_file=offloading_spec.block_size_factor,
|
||||
parallel_agnostic=True,
|
||||
).get_run_config()
|
||||
self._data: DataTransport = NixlTransport(
|
||||
self._local_id,
|
||||
primary_kv_view,
|
||||
config_fields=config_fields,
|
||||
backends=backends,
|
||||
num_threads=int(num_threads),
|
||||
)
|
||||
self._control: ControlTransport = ZmqTransport(self._local_id, host, port)
|
||||
|
||||
self._sessions: dict[str, P2PSession] = {}
|
||||
# kv_request_id → session, set when the bound session has received
|
||||
# FetchMsg for that id. submit_store after binding routes directly
|
||||
# to the session; before binding, batches are parked in
|
||||
# _unbound_stores below. Stays in sync with _sessions: entries
|
||||
# pointing to a reaped session are purged in _reap_dead_sessions.
|
||||
self._kv_to_session: dict[str, P2PSession] = {}
|
||||
# kv_request_id → list of batches submit_store'd before any peer
|
||||
# asked for that id. Drained into a session by _on_session_fetch
|
||||
# when the corresponding FetchMsg arrives, or surfaced as failures
|
||||
# by _reap_unbound_stores after _UNBOUND_STORE_TIMEOUT_S.
|
||||
self._unbound_stores: dict[str, list[_UnboundStoreBatch]] = {}
|
||||
|
||||
self._finished_jobs: list[JobResult] = []
|
||||
# kv_request_ids that hit a transport/session failure; On load lookup()
|
||||
# rejects them so the request falls back to local prefill.
|
||||
self._failed_req_ids: set[str] = set()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# SecondaryTierManager interface
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
prefill = _prefill_params(req_context.kv_transfer_params)
|
||||
if (
|
||||
not prefill
|
||||
or not prefill.get("remote_host")
|
||||
or not prefill.get("remote_port")
|
||||
or not prefill.get("kv_request_id")
|
||||
):
|
||||
return LookupResult.MISS
|
||||
|
||||
kv_request_id = prefill["kv_request_id"]
|
||||
if kv_request_id in self._failed_req_ids:
|
||||
return LookupResult.MISS
|
||||
return LookupResult.HIT
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
"""Open the outbound session toward the producer if needed.
|
||||
|
||||
On the decoder side (``prefill`` set), open a session toward the
|
||||
producer at remote_host:remote_port so submit_load can issue
|
||||
FetchMsg as soon as it fires. On the prefiller side, sessions
|
||||
are created when the consumer's inbound connection arrives in
|
||||
_accept_new_peers — submit_store no longer pre-creates anything.
|
||||
"""
|
||||
prefill = _prefill_params(req_context.kv_transfer_params)
|
||||
if prefill:
|
||||
peer_id = self._remote_id_from_params(prefill)
|
||||
if peer_id:
|
||||
self._get_or_create_session(peer_id)
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
"""Cancels pending loads and prunes session-scoped state.
|
||||
|
||||
Decoder side (``prefill`` set): looks up the session by peer_id
|
||||
because the producer's address is what addresses the client-role
|
||||
load to cancel. Prefiller side (``decode`` set): looks up via
|
||||
kv_request_id because peer_id is no longer carried on store-time
|
||||
kv_transfer_params; if a session has bound the id, finish it. If
|
||||
no session has bound the id yet, this is a no-op: parked batches
|
||||
in `_unbound_stores` are left in place and cleaned up only by
|
||||
`_reap_unbound_stores` after `_UNBOUND_STORE_TIMEOUT_S`.
|
||||
"""
|
||||
kv_params = req_context.kv_transfer_params
|
||||
if not kv_params:
|
||||
return
|
||||
prefill = _prefill_params(kv_params)
|
||||
decode = _decode_params(kv_params)
|
||||
kv_request_id = (prefill or decode or {}).get("kv_request_id")
|
||||
if not kv_request_id:
|
||||
return
|
||||
self._failed_req_ids.discard(kv_request_id)
|
||||
|
||||
if prefill:
|
||||
peer_id = self._remote_id_from_params(prefill)
|
||||
if peer_id:
|
||||
session = self._sessions.get(peer_id)
|
||||
if session is not None:
|
||||
session.finish_request(kv_request_id)
|
||||
return
|
||||
|
||||
# Prefiller-side finish: identify the session via kv_request_id.
|
||||
session = self._kv_to_session.pop(kv_request_id, None)
|
||||
if session is not None:
|
||||
session.finish_request(kv_request_id)
|
||||
return
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
job_id = job_metadata.job_id
|
||||
keys = list(job_metadata.keys)
|
||||
block_ids = job_metadata.block_ids
|
||||
|
||||
assert len(keys) == len(block_ids)
|
||||
|
||||
kv_params = job_metadata.req_context.kv_transfer_params
|
||||
decode = _decode_params(kv_params)
|
||||
logger.debug(
|
||||
"P2P %s: submit_store ENTRY job_id=%d blocks=%d decode=%s kv_request_id=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
len(block_ids),
|
||||
decode is not None,
|
||||
(decode or {}).get("kv_request_id"),
|
||||
)
|
||||
# Absent ``decode`` block => not a remote-decode request: succeed
|
||||
# locally without parking. An empty/malformed dict is still a
|
||||
# remote-decode signal and must fail the missing-id check below.
|
||||
if decode is None:
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=True))
|
||||
return
|
||||
|
||||
kv_request_id = decode.get("kv_request_id")
|
||||
if not kv_request_id:
|
||||
logger.warning(
|
||||
"P2P %s: submit_store missing kv_request_id",
|
||||
self._local_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
return
|
||||
|
||||
# Fast path: a session has already received FetchMsg for this id,
|
||||
# so we can route the batch straight into its ServerRole.
|
||||
session = self._kv_to_session.get(kv_request_id)
|
||||
if session is not None:
|
||||
session.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
|
||||
return
|
||||
|
||||
# No session bound yet — park the batch keyed by kv_request_id.
|
||||
# _on_session_fetch drains it on the first FetchMsg; if no peer
|
||||
# ever asks, _reap_unbound_stores surfaces the job as failed.
|
||||
self._unbound_stores.setdefault(kv_request_id, []).append(
|
||||
_UnboundStoreBatch(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=block_ids,
|
||||
)
|
||||
)
|
||||
logger.debug(
|
||||
"P2P %s: parked submit_store kv_request_id=%s job_id=%d blocks=%d",
|
||||
self._local_id,
|
||||
kv_request_id,
|
||||
job_id,
|
||||
len(block_ids),
|
||||
)
|
||||
|
||||
@override
|
||||
def submit_load(self, job_metadata: JobMetadata) -> None:
|
||||
job_id = job_metadata.job_id
|
||||
keys = list(job_metadata.keys)
|
||||
block_ids = job_metadata.block_ids
|
||||
|
||||
prefill = _prefill_params(job_metadata.req_context.kv_transfer_params)
|
||||
logger.debug(
|
||||
"P2P %s: submit_load ENTRY job_id=%d blocks=%d kv_request_id=%s peer=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
len(block_ids),
|
||||
(prefill or {}).get("kv_request_id"),
|
||||
self._remote_id_from_params(prefill or {}),
|
||||
)
|
||||
if (
|
||||
not prefill
|
||||
or not prefill.get("remote_host")
|
||||
or not prefill.get("remote_port")
|
||||
or not prefill.get("kv_request_id")
|
||||
):
|
||||
logger.debug(
|
||||
"P2P %s: submit_load job_id=%d FAILED missing prefill params",
|
||||
self._local_id,
|
||||
job_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
return
|
||||
|
||||
kv_request_id = prefill["kv_request_id"]
|
||||
peer_id = self._remote_id_from_params(prefill)
|
||||
assert peer_id is not None # guaranteed by prefill checks above
|
||||
|
||||
if not keys:
|
||||
logger.debug(
|
||||
"P2P %s: submit_load job_id=%d short-circuit success (no keys)",
|
||||
self._local_id,
|
||||
job_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=True))
|
||||
return
|
||||
|
||||
session = self._sessions.get(peer_id)
|
||||
if session is None:
|
||||
logger.warning(
|
||||
"P2P %s: submit_load job_id=%d NO SESSION for peer=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
peer_id,
|
||||
)
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
self._failed_req_ids.add(kv_request_id)
|
||||
return
|
||||
logger.debug(
|
||||
"P2P %s: submit_load job_id=%d -> request_blocks peer=%s "
|
||||
"kv_request_id=%s blocks=%d session_ready=%s",
|
||||
self._local_id,
|
||||
job_id,
|
||||
peer_id,
|
||||
kv_request_id,
|
||||
len(block_ids),
|
||||
session.ready,
|
||||
)
|
||||
session.request_blocks(job_id, kv_request_id, keys, block_ids)
|
||||
|
||||
@override
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
# Drive one polling sweep on the scheduler thread, then hand off
|
||||
# whatever has accumulated. The engine calls this once per step
|
||||
# (and keeps stepping while has_pending_work() is True).
|
||||
self._poll_once()
|
||||
result = self._finished_jobs
|
||||
self._finished_jobs = []
|
||||
return result
|
||||
|
||||
@override
|
||||
def has_pending_work(self) -> bool:
|
||||
# The engine tick is the only driver of _control.poll() and
|
||||
# session.poll(); without it we miss new peer connects and
|
||||
# inbound fetch messages on existing sessions. Keep the engine
|
||||
# ticking for the lifetime of this manager.
|
||||
return True
|
||||
|
||||
@override
|
||||
def drain_jobs(self) -> None:
|
||||
"""Block until every submitted load/store job has completed or failed.
|
||||
|
||||
Loops calling ``_poll_once()`` until no session has outstanding
|
||||
inbound loads or in-flight outbound stores. Mid-flight transfers
|
||||
are NOT cancelled — the caller (``TieringOffloadingManager.reset_cache``)
|
||||
needs the primary memoryview to be quiescent, not aborted. Results
|
||||
accumulate in ``_finished_jobs`` and are surfaced by the next
|
||||
``get_finished_jobs()`` call.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
warned = False
|
||||
while True:
|
||||
self._poll_once()
|
||||
pending = any(
|
||||
s._client._inbound or s._server._inflight
|
||||
for s in self._sessions.values()
|
||||
)
|
||||
if not pending:
|
||||
return
|
||||
if not warned and time.monotonic() - start > 5.0:
|
||||
logger.warning(
|
||||
"P2PSecondaryTierManager.drain_jobs: still draining "
|
||||
"after 5s; a stuck transfer will block the engine.",
|
||||
)
|
||||
warned = True
|
||||
time.sleep(_DRAIN_SLEEP_S)
|
||||
|
||||
@override
|
||||
def on_schedule_end(self) -> None:
|
||||
return
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _remote_id_from_params(role_params: dict) -> str | None:
|
||||
"""Build peer_id from a role-scoped sub-dict (``prefill``/``p2p``)."""
|
||||
host = role_params.get("remote_host")
|
||||
port = role_params.get("remote_port")
|
||||
if host and port:
|
||||
return f"{host}:{port}"
|
||||
return None
|
||||
|
||||
def _get_or_create_session(self, peer_id: str) -> P2PSession:
|
||||
"""Return the existing session for peer_id, or open one outbound.
|
||||
|
||||
Decoder-side helper for on_new_request: when ``prefill`` is set,
|
||||
the consumer must reach the producer at peer_id. If we already
|
||||
have a session toward that peer (from a prior load or a
|
||||
peer-initiated inbound), reuse it; otherwise open an outbound
|
||||
ControlConnection and build a connected session.
|
||||
"""
|
||||
session = self._sessions.get(peer_id)
|
||||
if session is not None:
|
||||
return session
|
||||
conn = self._control.connect(peer_id)
|
||||
session = P2PSession(
|
||||
peer_id=peer_id,
|
||||
local_id=self._local_id,
|
||||
transport=self._data,
|
||||
local_block_len=self._data.block_len,
|
||||
conn=conn,
|
||||
)
|
||||
self._sessions[peer_id] = session
|
||||
return session
|
||||
|
||||
def _accept_new_peers(self, new_connections: Sequence[ControlConnection]) -> None:
|
||||
for conn in new_connections:
|
||||
logger.info(
|
||||
"P2P %s: accepting incoming connection from %s",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
)
|
||||
try:
|
||||
existing = self._sessions.get(conn.peer_id)
|
||||
if existing is not None:
|
||||
raise ValueError(f"duplicate connection from {conn.peer_id}")
|
||||
self._sessions[conn.peer_id] = P2PSession(
|
||||
peer_id=conn.peer_id,
|
||||
local_id=self._local_id,
|
||||
transport=self._data,
|
||||
local_block_len=self._data.block_len,
|
||||
conn=conn,
|
||||
)
|
||||
logger.info(
|
||||
"P2P %s: created connected session for %s",
|
||||
self._local_id,
|
||||
conn.peer_id,
|
||||
)
|
||||
except (ValueError, KeyError, TypeError, AssertionError) as exc:
|
||||
logger.error("P2P %s: rejecting peer: %s", self._local_id, exc)
|
||||
conn.close()
|
||||
|
||||
def _reap_dead_sessions(self) -> None:
|
||||
# Reap connected sessions whose connection died — peer is gone.
|
||||
# Stranded prefiller-side stores are no longer tracked through a
|
||||
# session (they live in _unbound_stores keyed by kv_request_id);
|
||||
# _reap_unbound_stores handles their timeout independently.
|
||||
dead: list[str] | None = None
|
||||
for pid, s in self._sessions.items():
|
||||
if s.connected and not s.alive:
|
||||
if dead is None:
|
||||
dead = []
|
||||
dead.append(pid)
|
||||
if dead is None:
|
||||
return
|
||||
for pid in dead:
|
||||
session = self._sessions.pop(pid)
|
||||
# Purge any kv_request_id → session entries pointing at this
|
||||
# session so subsequent submit_stores fall back to the unbound
|
||||
# path (which will time out into failure if no peer rebinds).
|
||||
stale_kv_ids = [
|
||||
kid for kid, s in self._kv_to_session.items() if s is session
|
||||
]
|
||||
for kid in stale_kv_ids:
|
||||
del self._kv_to_session[kid]
|
||||
failed_loads, failed_stores = session.close()
|
||||
for job_id, kv_request_id in failed_loads:
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
self._failed_req_ids.add(kv_request_id)
|
||||
for job_id in failed_stores:
|
||||
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
|
||||
self._data.remove_remote_peer(pid)
|
||||
logger.warning("P2P %s: peer %s down", self._local_id, pid)
|
||||
|
||||
def _reap_unbound_stores(self) -> None:
|
||||
"""Time out submit_store batches that no peer has ever fetched.
|
||||
|
||||
Walks `_unbound_stores` for entries whose oldest batch is older
|
||||
than `_UNBOUND_STORE_TIMEOUT_S`. Drops the kv_request_id, surfaces
|
||||
every batched job as failed, and adds the id to `_failed_req_ids`
|
||||
so a late inbound FetchMsg short-circuits to a clean rejection.
|
||||
"""
|
||||
if not self._unbound_stores:
|
||||
return
|
||||
deadline = time.monotonic() - _UNBOUND_STORE_TIMEOUT_S
|
||||
expired: list[str] | None = None
|
||||
for kid, batches in self._unbound_stores.items():
|
||||
# Batches are appended in arrival order, so the head is oldest.
|
||||
if batches and batches[0].submitted_at <= deadline:
|
||||
if expired is None:
|
||||
expired = []
|
||||
expired.append(kid)
|
||||
if expired is None:
|
||||
return
|
||||
for kid in expired:
|
||||
batches = self._unbound_stores.pop(kid)
|
||||
self._failed_req_ids.add(kid)
|
||||
for batch in batches:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=batch.job_id, success=False)
|
||||
)
|
||||
logger.warning(
|
||||
"P2P %s: unbound store kv_request_id=%s timed out after %.0fs "
|
||||
"without a fetch — failing %d job(s)",
|
||||
self._local_id,
|
||||
kid,
|
||||
_UNBOUND_STORE_TIMEOUT_S,
|
||||
len(batches),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Polling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _poll_once(self) -> None:
|
||||
"""One sweep of the polling work.
|
||||
|
||||
Drains the control transport, polls every session, accumulates
|
||||
their results into ``_finished_jobs``, and reaps any dead sessions.
|
||||
Runs on the scheduler thread.
|
||||
"""
|
||||
new_connections = self._control.poll()
|
||||
if new_connections:
|
||||
logger.info(
|
||||
"P2P %s: _poll_once got %d new connection(s): %s",
|
||||
self._local_id,
|
||||
len(new_connections),
|
||||
[c.peer_id for c in new_connections],
|
||||
)
|
||||
|
||||
self._accept_new_peers(new_connections)
|
||||
|
||||
for session in self._sessions.values():
|
||||
result = session.poll()
|
||||
for lr in result.loads:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=lr.job_id, success=lr.success)
|
||||
)
|
||||
if not lr.success:
|
||||
self._failed_req_ids.add(lr.kv_request_id)
|
||||
for sr in result.stores:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=sr.job_id, success=sr.success)
|
||||
)
|
||||
# Bind kv_request_id → session for any FetchMsg this tick and
|
||||
# replay any submit_store batches parked while no peer was
|
||||
# asking. ServerRole.on_fetch already recorded the demand
|
||||
# inline in dispatch, so the replayed add_stored_blocks calls
|
||||
# match that demand and submit transfers immediately.
|
||||
for kv_request_id in result.new_fetch_ids:
|
||||
self._kv_to_session[kv_request_id] = session
|
||||
for batch in self._unbound_stores.pop(kv_request_id, ()):
|
||||
session.add_stored_blocks(
|
||||
kv_request_id, batch.keys, batch.block_ids, batch.job_id
|
||||
)
|
||||
|
||||
self._reap_dead_sessions()
|
||||
self._reap_unbound_stores()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@override
|
||||
def shutdown(self) -> None:
|
||||
self._drain_inflight_for_shutdown()
|
||||
for session in self._sessions.values():
|
||||
session.close()
|
||||
self._sessions.clear()
|
||||
self._kv_to_session.clear()
|
||||
# Surface buffered store jobs as failed so the engine doesn't
|
||||
# leak them; the manager is going away after this call.
|
||||
for batches in self._unbound_stores.values():
|
||||
for batch in batches:
|
||||
self._finished_jobs.append(
|
||||
JobResult(job_id=batch.job_id, success=False)
|
||||
)
|
||||
self._unbound_stores.clear()
|
||||
self._control.close()
|
||||
self._data.close()
|
||||
|
||||
def _drain_inflight_for_shutdown(self) -> None:
|
||||
"""Best-effort drain of inflight transfers before closing _data.
|
||||
|
||||
Mirrors session._drain_abort but as a single bounded loop. Collects
|
||||
inflight transfer_ids from each session, repeatedly calls
|
||||
_data.cancel(..., mode="wait") and _data.poll() so handles can
|
||||
surface as done/failed, and falls back to mode="immediate" once
|
||||
_SHUTDOWN_DRAIN_TIMEOUT_S elapses so a wedged peer can't hang us.
|
||||
"""
|
||||
ids = [tid for s in self._sessions.values() for tid in s._server._inflight]
|
||||
if not ids:
|
||||
return
|
||||
deadline = time.monotonic() + _SHUTDOWN_DRAIN_TIMEOUT_S
|
||||
still: list[int] = ids
|
||||
while still and time.monotonic() < deadline:
|
||||
still = list(self._data.cancel(still, mode="wait"))
|
||||
if not still:
|
||||
break
|
||||
# poll() advances NIXL handle state so the next wait-cancel
|
||||
# has a chance to release the handles.
|
||||
self._data.poll()
|
||||
time.sleep(_DRAIN_SLEEP_S)
|
||||
if still:
|
||||
logger.warning(
|
||||
"P2P %s: shutdown drain timed out after %.1fs with %d "
|
||||
"transfers still inflight — force-cancelling",
|
||||
self._local_id,
|
||||
_SHUTDOWN_DRAIN_TIMEOUT_S,
|
||||
len(still),
|
||||
)
|
||||
self._data.cancel(still, mode="immediate")
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.session import (
|
||||
LoadResult,
|
||||
P2PSession,
|
||||
SessionPollResult,
|
||||
StoreResult,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LoadResult",
|
||||
"P2PSession",
|
||||
"SessionPollResult",
|
||||
"StoreResult",
|
||||
]
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Client-role state machine for a single peer session.
|
||||
|
||||
Handles outgoing fetch requests, abort-on-timeout, abort-ack timeout,
|
||||
and produces ``LoadResult`` for completed loads. The session coordinator
|
||||
parses wire messages and dispatches typed arguments here; this module
|
||||
never touches ``ControlConnection`` directly — it emits via the ``send``
|
||||
callback injected by the coordinator (which gates on ConnectAck).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
AbortFetchMsg,
|
||||
FetchMsg,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.base import JobId
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_LOAD_TIMEOUT_S = 30.0
|
||||
_ABORT_ACK_TIMEOUT_S = 10.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class _InboundRequestState:
|
||||
"""Client-role state for a single load request."""
|
||||
|
||||
job_id: int # opaque ID assigned by the manager to this load request
|
||||
kv_request_id: str
|
||||
submitted_at: float
|
||||
aborted_at: float | None = None
|
||||
|
||||
|
||||
class LoadResult(NamedTuple):
|
||||
"""Result from a session poll, client side."""
|
||||
|
||||
job_id: int
|
||||
kv_request_id: str
|
||||
success: bool
|
||||
|
||||
|
||||
class ClientRole:
|
||||
"""Client-side load state machine for one peer session.
|
||||
|
||||
The coordinator owns the connection and the send-gating; this role
|
||||
is given a ``send`` callback and a ``peer_id`` for log messages and
|
||||
is otherwise self-contained.
|
||||
"""
|
||||
|
||||
def __init__(self, peer_id: str, send: Callable[[dict], None]) -> None:
|
||||
self._peer_id = peer_id
|
||||
self._send = send
|
||||
self._inbound: dict[str, _InboundRequestState] = {}
|
||||
self._completed_loads: list[LoadResult] = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def request_blocks(
|
||||
self,
|
||||
job_id: JobId,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[bytes],
|
||||
block_ids: Sequence[int],
|
||||
send_ready: bool,
|
||||
) -> None:
|
||||
"""Register a load request and send the FetchMsg."""
|
||||
logger.debug(
|
||||
"P2PSession %s: request_blocks job_id=%d kv_request_id=%s "
|
||||
"blocks=%d ready=%s",
|
||||
self._peer_id,
|
||||
job_id,
|
||||
kv_request_id,
|
||||
len(block_ids),
|
||||
send_ready,
|
||||
)
|
||||
self._inbound[kv_request_id] = _InboundRequestState(
|
||||
job_id=job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
submitted_at=time.monotonic(),
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: FetchMsg.TYPE,
|
||||
FetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
FetchMsg.BLOCK_HASHES: list(keys),
|
||||
FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids],
|
||||
}
|
||||
)
|
||||
|
||||
def cancel(self, kv_request_id: str) -> None:
|
||||
"""Cancel a pending load. Sends AbortFetchMsg if still active."""
|
||||
req = self._inbound.pop(kv_request_id, None)
|
||||
if req is not None and req.aborted_at is None:
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: kv_request_id,
|
||||
}
|
||||
)
|
||||
|
||||
def on_transfer_done(self, kv_request_id: str, success: bool) -> None:
|
||||
"""Handle a TransferDoneMsg from the peer."""
|
||||
req = self._inbound.pop(kv_request_id, None)
|
||||
if req is not None:
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=req.job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
success=success,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# No matching _inbound entry: either a duplicate
|
||||
# transfer_done from the peer (protocol violation) or a
|
||||
# benign race with a local cancel/abort/timeout that
|
||||
# already popped the entry. We don't track terminated ids,
|
||||
# so we can't tell — log so it's findable.
|
||||
logger.warning(
|
||||
"P2PSession %s: transfer_done for unknown kv_request_id=%s "
|
||||
"(duplicate from peer, or raced with local cancel/timeout)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
)
|
||||
|
||||
def on_abort_ack(self, kv_request_id: str) -> None:
|
||||
"""Handle an AbortAckMsg from the peer."""
|
||||
req = self._inbound.pop(kv_request_id, None)
|
||||
if req is not None:
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=req.job_id,
|
||||
kv_request_id=kv_request_id,
|
||||
success=False,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# See on_transfer_done: same ambiguity (duplicate ack
|
||||
# vs. raced with local cancel/timeout that already popped).
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack for unknown kv_request_id=%s "
|
||||
"(duplicate from peer, or raced with local cancel/timeout)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
)
|
||||
|
||||
def collect_results(self) -> list[LoadResult]:
|
||||
"""Walk timeouts and drain completed loads.
|
||||
|
||||
Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg
|
||||
sent and enter the aborting phase. Aborting requests past
|
||||
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads.
|
||||
"""
|
||||
now = time.monotonic()
|
||||
to_remove: list[str] = []
|
||||
for req_id, req in self._inbound.items():
|
||||
if req.aborted_at is None:
|
||||
if now - req.submitted_at >= _LOAD_TIMEOUT_S:
|
||||
req.aborted_at = now
|
||||
logger.warning(
|
||||
"P2PSession %s: %s timed out, sending abort",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortFetchMsg.TYPE,
|
||||
AbortFetchMsg.KV_REQUEST_ID: req_id,
|
||||
}
|
||||
)
|
||||
else:
|
||||
if now - req.aborted_at >= _ABORT_ACK_TIMEOUT_S:
|
||||
to_remove.append(req_id)
|
||||
self._completed_loads.append(
|
||||
LoadResult(
|
||||
job_id=req.job_id,
|
||||
kv_request_id=req_id,
|
||||
success=False,
|
||||
)
|
||||
)
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_ack timed out for kv_request_id=%s",
|
||||
self._peer_id,
|
||||
req_id,
|
||||
)
|
||||
for req_id in to_remove:
|
||||
self._inbound.pop(req_id)
|
||||
|
||||
results = self._completed_loads
|
||||
self._completed_loads = []
|
||||
return results
|
||||
|
||||
def close(self) -> list[tuple[int, str]]:
|
||||
"""Tear down. Returns ``(job_id, kv_request_id)`` for pending loads."""
|
||||
failed = [(req.job_id, req.kv_request_id) for req in self._inbound.values()]
|
||||
self._inbound.clear()
|
||||
self._completed_loads.clear()
|
||||
return failed
|
||||
@@ -0,0 +1,247 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2P KV cache sharing protocol constants and documentation.
|
||||
|
||||
Protocol Overview
|
||||
=================
|
||||
|
||||
Two session types communicate over a bidirectional message channel:
|
||||
- P2PClientSession (requests blocks from a server)
|
||||
- P2PServerSession (serves blocks to a client)
|
||||
|
||||
Connection Lifecycle
|
||||
--------------------
|
||||
|
||||
1. Client opens a connection and sends ConnectMsg with its identity,
|
||||
RDMA metadata, memory layout, and config fingerprint.
|
||||
2. Server validates block_len and config_fingerprint, registers the
|
||||
RDMA peer, and replies with ConnectAckMsg.
|
||||
3. Client receives ConnectAckMsg and transitions to ready state
|
||||
(flushes any queued messages).
|
||||
4. Either side may send DisconnectMsg to gracefully close.
|
||||
|
||||
Block Transfer Flow (happy path)
|
||||
---------------------------------
|
||||
|
||||
1. Client sends FetchMsg with a kv_request_id and lists of
|
||||
block keys + remote indexes where it wants the data written.
|
||||
2. Server matches requested blocks against locally stored blocks:
|
||||
- Blocks already available are transferred immediately via RDMA.
|
||||
- Blocks not yet available are recorded as "demanded" and
|
||||
transferred when the server later stores them.
|
||||
3. When all blocks for a kv_request_id are transferred, the server
|
||||
sends TransferDoneMsg (success=True) to the client.
|
||||
4. Client reports the load job as complete.
|
||||
|
||||
Abort Flow (timeout path)
|
||||
--------------------------
|
||||
|
||||
1. If the client times out waiting for TransferDoneMsg, it sends
|
||||
AbortFetchMsg to cancel the request.
|
||||
2. Server cancels inflight transfers for that kv_request_id and
|
||||
replies with AbortAckMsg.
|
||||
3. Client receives AbortAckMsg and reports the load job as failed.
|
||||
4. If AbortAckMsg itself times out, the client fails the job anyway.
|
||||
|
||||
Message Format
|
||||
--------------
|
||||
|
||||
All messages are dicts serialized with msgpack. Every message has a
|
||||
TYPE_KEY key identifying its type. Additional fields depend on the
|
||||
message type (see per-message class docstrings below).
|
||||
|
||||
Security
|
||||
--------
|
||||
|
||||
Sessions wrap all incoming message handling in try/except to guard
|
||||
against malformed messages from adversarial or buggy peers. Invalid
|
||||
messages are logged and dropped without crashing the session.
|
||||
|
||||
The config_fingerprint field in ConnectMsg ensures peers have
|
||||
compatible model configurations (model, dtype, block sizes). Mismatches
|
||||
are rejected during the handshake.
|
||||
"""
|
||||
|
||||
TYPE_KEY = "type"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validation helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require(msg: dict, key: str, typ: type, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is missing or not isinstance(typ)."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, typ):
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected {typ.__name__}, got {type(val).__name__}")
|
||||
|
||||
|
||||
def _require_pos_int(msg: dict, key: str, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is not a positive int."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, int) or val <= 0:
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected positive int, got {val!r}")
|
||||
|
||||
|
||||
def _require_non_neg_int(msg: dict, key: str, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is not a non-negative int."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, int) or val < 0:
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected non-negative int, got {val!r}")
|
||||
|
||||
|
||||
def _require_list(msg: dict, key: str, *, name: str = "") -> None:
|
||||
"""Raise ValueError if msg[key] is not a list."""
|
||||
val = msg.get(key)
|
||||
if not isinstance(val, list):
|
||||
label = name or key
|
||||
raise ValueError(f"{label}: expected list, got {type(val).__name__}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Message classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ConnectMsg:
|
||||
"""Client → Server: initial handshake request.
|
||||
|
||||
Fields:
|
||||
PEER_ID: Local peer identity string.
|
||||
AGENT_METADATA: RDMA agent metadata (opaque bytes).
|
||||
BASE_ADDR: Base memory address of the KV block region.
|
||||
NUM_BLOCKS: Number of blocks in the KV block region.
|
||||
BLOCK_LEN: Size in bytes of each block (must match between peers).
|
||||
CONFIG_FINGERPRINT: SHA-256 prefix of the model configuration.
|
||||
Peers with different fingerprints are incompatible.
|
||||
"""
|
||||
|
||||
TYPE = "connect"
|
||||
PEER_ID = "peer_id"
|
||||
AGENT_METADATA = "agent_metadata"
|
||||
BASE_ADDR = "base_addr"
|
||||
NUM_BLOCKS = "num_blocks"
|
||||
BLOCK_LEN = "block_len"
|
||||
CONFIG_FINGERPRINT = "config_fingerprint"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, ConnectMsg.PEER_ID, str)
|
||||
_require(msg, ConnectMsg.AGENT_METADATA, bytes)
|
||||
_require_non_neg_int(msg, ConnectMsg.BASE_ADDR)
|
||||
_require_pos_int(msg, ConnectMsg.NUM_BLOCKS)
|
||||
_require_pos_int(msg, ConnectMsg.BLOCK_LEN)
|
||||
|
||||
|
||||
class ConnectAckMsg:
|
||||
"""Server → Client: handshake acknowledgement.
|
||||
|
||||
Fields:
|
||||
PEER_ID: Server's peer identity string.
|
||||
"""
|
||||
|
||||
TYPE = "connect_ack"
|
||||
PEER_ID = "peer_id"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, ConnectAckMsg.PEER_ID, str)
|
||||
|
||||
|
||||
class DisconnectMsg:
|
||||
"""Either → Either: graceful connection close.
|
||||
|
||||
No additional fields beyond TYPE_KEY.
|
||||
"""
|
||||
|
||||
TYPE = "disconnect"
|
||||
|
||||
|
||||
class FetchMsg:
|
||||
"""Client → Server: request blocks by key.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: Identifies this block transfer request.
|
||||
BLOCK_HASHES: List of block keys (OffloadKey bytes).
|
||||
BLOCK_INDEXES: List of remote block indexes (same length as BLOCK_HASHES).
|
||||
"""
|
||||
|
||||
TYPE = "fetch"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
BLOCK_HASHES = "block_hashes"
|
||||
BLOCK_INDEXES = "block_indexes"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, FetchMsg.KV_REQUEST_ID, str)
|
||||
_require_list(msg, FetchMsg.BLOCK_HASHES)
|
||||
_require_list(msg, FetchMsg.BLOCK_INDEXES)
|
||||
hashes = msg[FetchMsg.BLOCK_HASHES]
|
||||
indexes = msg[FetchMsg.BLOCK_INDEXES]
|
||||
if len(hashes) != len(indexes):
|
||||
raise ValueError(
|
||||
f"block_hashes/block_indexes length mismatch: "
|
||||
f"{len(hashes)} vs {len(indexes)}"
|
||||
)
|
||||
for idx in indexes:
|
||||
if not isinstance(idx, int) or idx < 0:
|
||||
raise ValueError(f"block_indexes: invalid index {idx!r}")
|
||||
|
||||
|
||||
class TransferDoneMsg:
|
||||
"""Server → Client: all blocks transferred for a request.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request that completed.
|
||||
SUCCESS: Whether the transfer completed successfully.
|
||||
"""
|
||||
|
||||
TYPE = "transfer_done"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
SUCCESS = "success"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, TransferDoneMsg.KV_REQUEST_ID, str)
|
||||
_require(msg, TransferDoneMsg.SUCCESS, bool)
|
||||
|
||||
|
||||
class AbortFetchMsg:
|
||||
"""Client → Server: cancel a pending request.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request to cancel.
|
||||
"""
|
||||
|
||||
TYPE = "abort_fetch"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, AbortFetchMsg.KV_REQUEST_ID, str)
|
||||
|
||||
|
||||
class AbortAckMsg:
|
||||
"""Server → Client: acknowledge cancellation.
|
||||
|
||||
Fields:
|
||||
KV_REQUEST_ID: The request that was cancelled.
|
||||
"""
|
||||
|
||||
TYPE = "abort_ack"
|
||||
KV_REQUEST_ID = "kv_request_id"
|
||||
|
||||
@staticmethod
|
||||
def validate(msg: dict) -> None:
|
||||
"""Raise ValueError if any field has an invalid type or value."""
|
||||
_require(msg, AbortAckMsg.KV_REQUEST_ID, str)
|
||||
@@ -0,0 +1,601 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Server-role state machine for a single peer session.
|
||||
|
||||
Owns block matching (supply vs. demand), inflight RDMA transfers, store-job
|
||||
timeouts, abort-drain, and produces ``StoreResult`` for completed stores.
|
||||
The session coordinator parses wire messages and dispatches typed
|
||||
arguments here; this module never touches ``ControlConnection`` directly
|
||||
— it emits via the ``send`` callback injected by the coordinator (which
|
||||
gates on ConnectAck).
|
||||
|
||||
Protocol violations the role can detect (today: duplicate ``FetchMsg``
|
||||
for the same ``kv_request_id``) are surfaced as ``ValueError`` so the
|
||||
coordinator's ``_dispatch_message`` can reuse its existing
|
||||
``_protocol_error`` path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
AbortAckMsg,
|
||||
TransferDoneMsg,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.base import JobId
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_STORE_TIMEOUT_S = 30.0
|
||||
_CANCEL_DRAIN_TIMEOUT_S = 10.0
|
||||
|
||||
|
||||
class StoreResult(NamedTuple):
|
||||
"""Result from a session poll, server side."""
|
||||
|
||||
job_id: int
|
||||
success: bool
|
||||
|
||||
|
||||
class _InflightXfer(NamedTuple):
|
||||
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
|
||||
|
||||
kv_request_id: str
|
||||
block_count: int
|
||||
# The set of store job IDs that contributed blocks to this transfer.
|
||||
job_ids: set[int]
|
||||
|
||||
|
||||
class _MatchResult(NamedTuple):
|
||||
"""Result of block matching: pairs ready for transfer."""
|
||||
|
||||
local_idxs: list[int]
|
||||
remote_idxs: list[int]
|
||||
# The set of store job IDs that contributed blocks
|
||||
job_ids: set[int]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _OutboundRequestState:
|
||||
"""Server-role state for a single peer fetch request.
|
||||
|
||||
The owning ``kv_request_id`` is the dict key in
|
||||
``ServerRole._outbound`` and is not duplicated on the value.
|
||||
"""
|
||||
|
||||
demand_received: bool = False
|
||||
available: dict[OffloadKey, tuple[int, int]] = field(
|
||||
default_factory=dict
|
||||
) # key → (job_id, local_block_idx): blocks we have, awaiting demand
|
||||
demanded: dict[OffloadKey, int] = field(
|
||||
default_factory=dict
|
||||
) # key → remote_block_idx: blocks peer wants, awaiting supply
|
||||
remaining: int = 0 # blocks that need to be transferred to client
|
||||
finishing: bool = False # Signal finish request ASAP
|
||||
# Job IDs that submit_store'd blocks for this request and have not
|
||||
# yet emitted a StoreResult. The terminal-finalize helper drains
|
||||
# this set; poll-done and poll-failed discard entries as their
|
||||
# StoreResults fire.
|
||||
pending_job_ids: set[int] = field(default_factory=set)
|
||||
|
||||
def add_stored_blocks(
|
||||
self,
|
||||
block_hashes: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: int,
|
||||
) -> _MatchResult:
|
||||
"""Add locally-stored blocks. Returns matched pairs."""
|
||||
self.pending_job_ids.add(job_id)
|
||||
local_idxs: list[int] = []
|
||||
remote_idxs: list[int] = []
|
||||
for block_hash, local_idx in zip(block_hashes, block_ids):
|
||||
remote_idx = self.demanded.pop(block_hash, None)
|
||||
if remote_idx is not None:
|
||||
local_idxs.append(local_idx)
|
||||
remote_idxs.append(remote_idx)
|
||||
else:
|
||||
self.available[block_hash] = (job_id, local_idx)
|
||||
return _MatchResult(
|
||||
local_idxs=local_idxs,
|
||||
remote_idxs=remote_idxs,
|
||||
job_ids={job_id} if local_idxs else set(),
|
||||
)
|
||||
|
||||
def add_fetch_demand(
|
||||
self,
|
||||
block_hashes: Sequence[OffloadKey],
|
||||
block_indexes: Sequence[int],
|
||||
) -> _MatchResult:
|
||||
"""Register the peer's fetch demand. Returns matched pairs."""
|
||||
self.demand_received = True
|
||||
self.remaining = len(block_hashes)
|
||||
|
||||
local_idxs: list[int] = []
|
||||
remote_idxs: list[int] = []
|
||||
job_ids: set[int] = set()
|
||||
for block_hash, remote_idx in zip(block_hashes, block_indexes):
|
||||
stored_entry = self.available.pop(block_hash, None)
|
||||
if stored_entry is not None:
|
||||
stored_job_id, local_idx = stored_entry
|
||||
local_idxs.append(local_idx)
|
||||
remote_idxs.append(remote_idx)
|
||||
job_ids.add(stored_job_id)
|
||||
else:
|
||||
self.demanded[block_hash] = remote_idx
|
||||
return _MatchResult(
|
||||
local_idxs=local_idxs,
|
||||
remote_idxs=remote_idxs,
|
||||
job_ids=job_ids,
|
||||
)
|
||||
|
||||
|
||||
class ServerRole:
|
||||
"""Server-side store/serve state machine for one peer session.
|
||||
|
||||
The coordinator owns the connection and the send-gating; this role
|
||||
is given a ``send`` callback, the ``DataTransport``, and the
|
||||
``peer_id`` for transport calls and log messages.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
peer_id: str,
|
||||
transport: DataTransport,
|
||||
send: Callable[[dict], None],
|
||||
) -> None:
|
||||
self._peer_id = peer_id
|
||||
self._transport = transport
|
||||
self._send = send
|
||||
|
||||
self._outbound: dict[str, _OutboundRequestState] = {}
|
||||
# transfer_id → xfer. Mutate ONLY via _inflight_add / _inflight_pop
|
||||
# so the per-request count below stays in sync.
|
||||
self._inflight: dict[int, _InflightXfer] = {}
|
||||
# kv_request_id → number of entries in _inflight for that id.
|
||||
# Kept in sync with _inflight; entries that hit zero are removed
|
||||
# so `kv_request_id in self._inflight_per_req` is an exact
|
||||
# "has any inflight transfer" predicate (O(1) replacement for
|
||||
# the previous O(N) scan).
|
||||
self._inflight_per_req: dict[str, int] = {}
|
||||
self._store_jobs: dict[int, float] = {} # job_id → submitted_at
|
||||
self._pending_aborts: dict[str, float] = {} # kv_request_id → start
|
||||
# StoreResults queued by _finalize_outbound for the next poll
|
||||
# tick to surface. Mirrors the deferred-result pattern used for
|
||||
# load timeouts.
|
||||
self._pending_store_results: list[StoreResult] = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def add_stored_blocks(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: JobId,
|
||||
) -> None:
|
||||
"""New blocks stored locally — match against pending fetch demand."""
|
||||
self._store_jobs[job_id] = time.monotonic()
|
||||
req = self._outbound.setdefault(kv_request_id, _OutboundRequestState())
|
||||
result = req.add_stored_blocks(keys, block_ids, job_id)
|
||||
if result.local_idxs and req.demand_received:
|
||||
self._submit_transfer(kv_request_id, result)
|
||||
|
||||
def on_fetch(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
block_hashes: Sequence[OffloadKey],
|
||||
block_indexes: Sequence[int],
|
||||
) -> None:
|
||||
"""Handle a FetchMsg from the peer.
|
||||
|
||||
Raises ``ValueError`` on a duplicate fetch for the same
|
||||
``kv_request_id``; the coordinator's dispatch loop turns that
|
||||
into a protocol-error disconnect.
|
||||
"""
|
||||
logger.debug(
|
||||
"P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(block_hashes),
|
||||
)
|
||||
existing = self._outbound.get(kv_request_id)
|
||||
if existing is not None and existing.demand_received:
|
||||
# A second fetch for the same kv_request_id would overwrite
|
||||
# `remaining` and leak inflight bookkeeping. Treat as a
|
||||
# protocol violation.
|
||||
raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}")
|
||||
req = self._outbound.setdefault(kv_request_id, _OutboundRequestState())
|
||||
result = req.add_fetch_demand(block_hashes, block_indexes)
|
||||
if result.local_idxs:
|
||||
self._submit_transfer(kv_request_id, result)
|
||||
# Prefiller-first mode: finish_request may have run before
|
||||
# fetch arrived. If so, finalize once we know what was
|
||||
# demanded — fully satisfied → success, else early-fail.
|
||||
if req.finishing and not self._has_inflight_for(kv_request_id):
|
||||
self._finalize_outbound(kv_request_id)
|
||||
|
||||
def on_abort_fetch(self, kv_request_id: str) -> None:
|
||||
"""Handle an AbortFetchMsg from the peer."""
|
||||
# Abort for an unknown id may be a benign race/duplicate or a
|
||||
# real protocol violation; we don't track completed ids, so warn.
|
||||
if kv_request_id not in self._outbound and not self._has_inflight_for(
|
||||
kv_request_id
|
||||
):
|
||||
logger.warning(
|
||||
"P2PSession %s: abort_fetch for unknown kv_request_id=%s "
|
||||
"(no outbound or inflight state); benign race or stale",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
)
|
||||
# Idempotent: receiving AbortFetchMsg again before we've sent the
|
||||
# ack just triggers another drain attempt without resetting the
|
||||
# deadline.
|
||||
self._pending_aborts.setdefault(kv_request_id, time.monotonic())
|
||||
self._drain_abort(kv_request_id)
|
||||
|
||||
def finish(self, kv_request_id: str) -> None:
|
||||
"""Mark an outbound request finishing.
|
||||
|
||||
No more submit_store calls will arrive for this id. Any blocks
|
||||
the peer demanded but we never stored will never come; tell the
|
||||
peer to stop waiting (TransferDoneMsg success=False) instead of
|
||||
letting it hit _LOAD_TIMEOUT_S.
|
||||
|
||||
If the decoder hasn't sent fetch yet (no demand received),
|
||||
defer — on_fetch will finalize once demand arrives.
|
||||
|
||||
If inflight transfers exist for this id, defer — the last
|
||||
completing transfer in collect_results will fire the message.
|
||||
"""
|
||||
req = self._outbound.get(kv_request_id)
|
||||
if req is None:
|
||||
return
|
||||
req.finishing = True
|
||||
if not req.demand_received:
|
||||
return
|
||||
if self._has_inflight_for(kv_request_id):
|
||||
return
|
||||
# Remaining > 0 here: if it had hit 0, the poll-done success
|
||||
# branch would have already popped _outbound and we'd have
|
||||
# returned at `req is None` above. Helper derives success from
|
||||
# remaining and emits StoreResult(success=False) for any
|
||||
# leftover pending jobs.
|
||||
self._finalize_outbound(kv_request_id)
|
||||
|
||||
def collect_results(self) -> list[StoreResult]:
|
||||
"""Drain timeouts, deferred results, and transport completions."""
|
||||
results: list[StoreResult] = self._timeout_pending_store_jobs()
|
||||
|
||||
if self._pending_store_results:
|
||||
results.extend(self._pending_store_results)
|
||||
self._pending_store_results.clear()
|
||||
|
||||
poll_result = self._transport.poll()
|
||||
|
||||
for tid in poll_result.done:
|
||||
xfer = self._inflight_pop(tid)
|
||||
if xfer is None:
|
||||
# Bug signal: transport reported a transfer we have no
|
||||
# bookkeeping for. Likely a double-completion in the
|
||||
# transport or a stale removal in the session. The
|
||||
# attached job(s) still live in _store_jobs and will be
|
||||
# surfaced as failures by _timeout_pending_store_jobs
|
||||
# after _STORE_TIMEOUT_S, but log loudly so the
|
||||
# underlying bug is findable.
|
||||
logger.error(
|
||||
"P2PSession %s: transport reported done for unknown "
|
||||
"transfer_id=%d; attached job(s) will fail via "
|
||||
"store-timeout instead of completing now",
|
||||
self._peer_id,
|
||||
tid,
|
||||
)
|
||||
continue
|
||||
req = self._outbound.get(xfer.kv_request_id)
|
||||
for job_id in xfer.job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
# Already reported (timeout, cancellation, etc.) —
|
||||
# don't double-emit a contradictory success result.
|
||||
continue
|
||||
results.append(StoreResult(job_id=job_id, success=True))
|
||||
if req is not None:
|
||||
req.pending_job_ids.discard(job_id)
|
||||
if req is not None and req.demand_received:
|
||||
req.remaining -= xfer.block_count
|
||||
assert req.remaining >= 0, (
|
||||
f"remaining went negative for kv_request_id={xfer.kv_request_id}"
|
||||
)
|
||||
if req.remaining == 0:
|
||||
self._finalize_outbound(xfer.kv_request_id, success=True)
|
||||
elif req.finishing and not self._has_inflight_for(xfer.kv_request_id):
|
||||
self._finalize_outbound(xfer.kv_request_id, success=False)
|
||||
|
||||
failed_kv_request_ids: set[str] | None = None
|
||||
for tid in poll_result.failed:
|
||||
xfer = self._inflight_pop(tid)
|
||||
if xfer is None:
|
||||
# See the matching error log in the done branch above.
|
||||
logger.error(
|
||||
"P2PSession %s: transport reported failed for unknown "
|
||||
"transfer_id=%d; attached job(s) will fail via "
|
||||
"store-timeout instead of completing now",
|
||||
self._peer_id,
|
||||
tid,
|
||||
)
|
||||
continue
|
||||
if failed_kv_request_ids is None:
|
||||
failed_kv_request_ids = set()
|
||||
failed_kv_request_ids.add(xfer.kv_request_id)
|
||||
req_for_xfer = self._outbound.get(xfer.kv_request_id)
|
||||
for job_id in xfer.job_ids:
|
||||
if self._store_jobs.pop(job_id, None) is None:
|
||||
# Already reported (timeout, cancellation, etc.) —
|
||||
# don't double-emit.
|
||||
continue
|
||||
results.append(StoreResult(job_id=job_id, success=False))
|
||||
if req_for_xfer is not None:
|
||||
req_for_xfer.pending_job_ids.discard(job_id)
|
||||
req = self._outbound.pop(xfer.kv_request_id, None)
|
||||
if req is not None and req.demand_received:
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: False,
|
||||
}
|
||||
)
|
||||
|
||||
# Cancel other inflight for the same failed kv_request_ids
|
||||
if failed_kv_request_ids:
|
||||
ids_to_cancel = [
|
||||
tid
|
||||
for tid, xfer in self._inflight.items()
|
||||
if xfer.kv_request_id in failed_kv_request_ids
|
||||
]
|
||||
for tid in ids_to_cancel:
|
||||
self._inflight_pop(tid)
|
||||
self._transport.cancel(ids_to_cancel)
|
||||
|
||||
return results
|
||||
|
||||
def collect_idle_timeouts(self) -> list[StoreResult]:
|
||||
"""Run only the store-job timeout sweep.
|
||||
|
||||
Used by the coordinator's no-conn poll path: a pending session
|
||||
cannot have inflight transfers (no peer registered yet), so we
|
||||
skip the transport poll and the deferred-result drain.
|
||||
"""
|
||||
return self._timeout_pending_store_jobs()
|
||||
|
||||
def drain_pending_aborts(self) -> None:
|
||||
"""Re-attempt every parked abort once per poll tick."""
|
||||
if not self._pending_aborts:
|
||||
return
|
||||
for kv_request_id in list(self._pending_aborts):
|
||||
self._drain_abort(kv_request_id)
|
||||
|
||||
def close(self) -> list[int]:
|
||||
"""Tear down. Cancels inflight, returns failed store job ids."""
|
||||
failed_stores = list(self._store_jobs.keys())
|
||||
self._store_jobs.clear()
|
||||
if self._inflight:
|
||||
self._transport.cancel(list(self._inflight.keys()))
|
||||
self._inflight.clear()
|
||||
self._inflight_per_req.clear()
|
||||
self._outbound.clear()
|
||||
self._pending_aborts.clear()
|
||||
self._pending_store_results.clear()
|
||||
return failed_stores
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — inflight bookkeeping
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _has_inflight_for(self, kv_request_id: str) -> bool:
|
||||
return kv_request_id in self._inflight_per_req
|
||||
|
||||
def _inflight_add(self, tid: int, xfer: _InflightXfer) -> None:
|
||||
"""Insert an inflight transfer and bump the per-request count."""
|
||||
self._inflight[tid] = xfer
|
||||
self._inflight_per_req[xfer.kv_request_id] = (
|
||||
self._inflight_per_req.get(xfer.kv_request_id, 0) + 1
|
||||
)
|
||||
|
||||
def _inflight_pop(self, tid: int) -> _InflightXfer | None:
|
||||
"""Pop an inflight transfer and decrement the per-request count.
|
||||
|
||||
Removes the per-request entry once the count hits zero so the
|
||||
dict stays bounded and `_has_inflight_for` remains exact.
|
||||
"""
|
||||
xfer = self._inflight.pop(tid, None)
|
||||
if xfer is None:
|
||||
return None
|
||||
new_count = self._inflight_per_req.get(xfer.kv_request_id, 0) - 1
|
||||
if new_count > 0:
|
||||
self._inflight_per_req[xfer.kv_request_id] = new_count
|
||||
else:
|
||||
self._inflight_per_req.pop(xfer.kv_request_id, None)
|
||||
return xfer
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — finalize / abort drain
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _finalize_outbound(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
success: bool | None = None,
|
||||
) -> None:
|
||||
"""Pop the outbound state and emit terminal results.
|
||||
|
||||
Called when no further work will happen for this kv_request_id
|
||||
on the server side: either request_finish has fired and there
|
||||
are no inflight transfers, or the last inflight just completed
|
||||
while finishing.
|
||||
|
||||
If ``success`` is None, derive it from ``req.remaining == 0``.
|
||||
The same flag is used for both the peer's TransferDoneMsg and
|
||||
the StoreResult(s) emitted for any leftover pending job_ids.
|
||||
"""
|
||||
req = self._outbound.pop(kv_request_id, None)
|
||||
if req is None:
|
||||
return
|
||||
if success is None:
|
||||
success = req.remaining == 0
|
||||
for job_id in req.pending_job_ids:
|
||||
self._store_jobs.pop(job_id, None)
|
||||
self._pending_store_results.append(
|
||||
StoreResult(job_id=job_id, success=success)
|
||||
)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: TransferDoneMsg.TYPE,
|
||||
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
|
||||
TransferDoneMsg.SUCCESS: success,
|
||||
}
|
||||
)
|
||||
|
||||
def _drain_abort(self, kv_request_id: str) -> None:
|
||||
"""One drain attempt for a pending abort.
|
||||
|
||||
Stops accepting more blocks for ``kv_request_id``, then asks the
|
||||
transport to cancel any matching inflight transfers in
|
||||
``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains
|
||||
inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to
|
||||
``mode="immediate"`` and acks anyway.
|
||||
"""
|
||||
self._outbound.pop(kv_request_id, None)
|
||||
ids = [
|
||||
tid
|
||||
for tid, xfer in self._inflight.items()
|
||||
if xfer.kv_request_id == kv_request_id
|
||||
]
|
||||
if not ids:
|
||||
self._finalize_abort(kv_request_id)
|
||||
return
|
||||
|
||||
started_at = self._pending_aborts.get(kv_request_id)
|
||||
expired = (
|
||||
started_at is not None
|
||||
and time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S
|
||||
)
|
||||
if expired:
|
||||
for tid in ids:
|
||||
self._inflight_pop(tid)
|
||||
self._transport.cancel(ids, mode="immediate")
|
||||
logger.warning(
|
||||
"P2PSession %s: cancel drain timed out for kv_request_id=%s,"
|
||||
" force-canceled %d transfers",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(ids),
|
||||
)
|
||||
self._finalize_abort(kv_request_id)
|
||||
return
|
||||
|
||||
still = self._transport.cancel(ids, mode="wait")
|
||||
# Tids the transport successfully released are gone from its
|
||||
# _inflight; mirror that in session bookkeeping so they don't
|
||||
# block the drain forever waiting for a poll() event that will
|
||||
# never come.
|
||||
still_set = set(still)
|
||||
for tid in ids:
|
||||
if tid not in still_set:
|
||||
self._inflight_pop(tid)
|
||||
if not still:
|
||||
self._finalize_abort(kv_request_id)
|
||||
|
||||
def _finalize_abort(self, kv_request_id: str) -> None:
|
||||
self._pending_aborts.pop(kv_request_id, None)
|
||||
self._send(
|
||||
{
|
||||
TYPE_KEY: AbortAckMsg.TYPE,
|
||||
AbortAckMsg.KV_REQUEST_ID: kv_request_id,
|
||||
}
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal — transfers and store-job timeouts
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None:
|
||||
logger.debug(
|
||||
"P2PSession %s: NIXL write_blocks CALL kv_request_id=%s "
|
||||
"local_idxs=%d remote_idxs=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(result.local_idxs),
|
||||
len(result.remote_idxs),
|
||||
)
|
||||
transfer_id = self._transport.write_blocks(
|
||||
self._peer_id, result.local_idxs, result.remote_idxs
|
||||
)
|
||||
if transfer_id is not None:
|
||||
logger.debug(
|
||||
"P2PSession %s: NIXL write_blocks SUBMITTED kv_request_id=%s "
|
||||
"transfer_id=%d blocks=%d",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
transfer_id,
|
||||
len(result.local_idxs),
|
||||
)
|
||||
self._inflight_add(
|
||||
transfer_id,
|
||||
_InflightXfer(
|
||||
kv_request_id=kv_request_id,
|
||||
block_count=len(result.local_idxs),
|
||||
job_ids=result.job_ids,
|
||||
),
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"P2PSession %s: write_blocks failed for %s (%d blocks)",
|
||||
self._peer_id,
|
||||
kv_request_id,
|
||||
len(result.local_idxs),
|
||||
)
|
||||
# The matched blocks were popped from req.demanded /
|
||||
# req.available, but no inflight will satisfy them, so
|
||||
# remaining will never reach 0 on its own. Mark the
|
||||
# request as finishing so the existing terminal paths
|
||||
# clean up: if other inflight is in flight, the last one
|
||||
# to drain will fire _finalize_outbound(success=False)
|
||||
# via the elif branch in collect_results. If
|
||||
# nothing else is in flight, finalize now so the peer
|
||||
# and the local store jobs don't wait for finish_request
|
||||
# or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S.
|
||||
req = self._outbound.get(kv_request_id)
|
||||
if req is not None:
|
||||
req.finishing = True
|
||||
if not self._has_inflight_for(kv_request_id):
|
||||
self._finalize_outbound(kv_request_id, success=False)
|
||||
|
||||
def _timeout_pending_store_jobs(self) -> list[StoreResult]:
|
||||
if not self._store_jobs:
|
||||
return []
|
||||
deadline = time.monotonic() - _STORE_TIMEOUT_S
|
||||
timed_out: list[int] | None = None
|
||||
for jid, submitted_at in self._store_jobs.items():
|
||||
if submitted_at <= deadline:
|
||||
if timed_out is None:
|
||||
timed_out = []
|
||||
timed_out.append(jid)
|
||||
if timed_out is None:
|
||||
return []
|
||||
results: list[StoreResult] = []
|
||||
for jid in timed_out:
|
||||
del self._store_jobs[jid]
|
||||
results.append(StoreResult(job_id=jid, success=False))
|
||||
logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid)
|
||||
return results
|
||||
@@ -0,0 +1,440 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
P2PSession — bidirectional session combining client + server roles.
|
||||
|
||||
A single P2PSession per remote peer handles BOTH directions of the P2P
|
||||
protocol on one ControlConnection: it can request blocks from the peer
|
||||
("client" role, in :mod:`.client`) AND serve blocks to the peer
|
||||
("server" role, in :mod:`.server`). This module is the thin coordinator
|
||||
that owns the connection, the handshake, send-gating, and the message
|
||||
dispatch — each parsed message is forwarded to the corresponding role.
|
||||
|
||||
Wire protocol is unchanged. Both sides advertise their NIXL metadata
|
||||
via ConnectMsg when their session is connected; the peer's ConnectMsg
|
||||
triggers transport.add_remote_peer; ConnectAckMsg confirms the peer
|
||||
received our ConnectMsg, after which queued outgoing messages are flushed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey
|
||||
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.client import ClientRole, LoadResult
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
|
||||
TYPE_KEY,
|
||||
AbortAckMsg,
|
||||
AbortFetchMsg,
|
||||
ConnectAckMsg,
|
||||
ConnectMsg,
|
||||
DisconnectMsg,
|
||||
FetchMsg,
|
||||
TransferDoneMsg,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.p2p.session.server import (
|
||||
ServerRole,
|
||||
StoreResult,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.kv_offload.tiering.base import JobId
|
||||
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Cap on consecutive non-protocol dispatch exceptions before we tear
|
||||
# down the session. Protocol violations (ValueError) disconnect on the
|
||||
# first occurrence; this threshold protects against repeated internal
|
||||
# bugs that may indicate a peer-induced bad state. Reset on any
|
||||
# successful dispatch.
|
||||
_MAX_CONSECUTIVE_DISPATCH_ERRORS = 5
|
||||
|
||||
|
||||
class SessionPollResult(NamedTuple):
|
||||
"""Result of one P2PSession.poll() tick.
|
||||
|
||||
`loads`/`stores` are the same per-role results the manager has always
|
||||
consumed. `new_fetch_ids` reports kv_request_ids whose FetchMsg
|
||||
arrived this tick — the manager uses them to bind kv_request_id →
|
||||
session and replay any submit_store batches parked while no peer had
|
||||
asked yet. Reporting (rather than calling back into the manager
|
||||
mid-dispatch) keeps the dependency strictly top-down.
|
||||
"""
|
||||
|
||||
loads: list[LoadResult]
|
||||
stores: list[StoreResult]
|
||||
new_fetch_ids: list[str]
|
||||
|
||||
|
||||
class P2PSession:
|
||||
"""Bidirectional session — coordinator over ClientRole + ServerRole.
|
||||
|
||||
Lifecycle:
|
||||
- Constructor with conn=None ⇒ pending. Accepts add_stored_blocks
|
||||
but cannot send (used by the prefiller to buffer blocks before
|
||||
the decoder connects).
|
||||
- Constructor with conn != None ⇒ connected. Sends our own ConnectMsg
|
||||
immediately; the peer's ConnectMsg arrives in poll() and is
|
||||
dispatched to _on_connect (which calls transport.add_remote_peer
|
||||
and replies with ConnectAckMsg). Outgoing sends are queued until
|
||||
ConnectAckMsg confirms our metadata reached the peer.
|
||||
- attach_connection(conn) on a pending session ⇒ same as above,
|
||||
starting from pending.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
peer_id: str,
|
||||
local_id: str,
|
||||
transport: DataTransport,
|
||||
local_block_len: int,
|
||||
conn: ControlConnection | None = None,
|
||||
) -> None:
|
||||
self.peer_id = peer_id
|
||||
self._local_id = local_id
|
||||
self._transport = transport
|
||||
self._local_block_len = local_block_len
|
||||
self._conn: ControlConnection | None = None
|
||||
|
||||
self._send_ready = False # True after the peer acked our ConnectMsg
|
||||
# Msgs waiting to be sent on connection establishment
|
||||
self._queued: list[dict] = []
|
||||
|
||||
# Consecutive non-protocol dispatch errors. Reset on success.
|
||||
self._dispatch_error_count: int = 0
|
||||
|
||||
# kv_request_ids whose FetchMsg arrived during the current poll
|
||||
# tick. Drained and returned in the next poll() result so the
|
||||
# manager can bind kv_request_id → session and replay any
|
||||
# submit_store batches parked before the binding existed.
|
||||
self._new_fetch_ids: list[str] = []
|
||||
|
||||
self._client = ClientRole(peer_id=peer_id, send=self._send)
|
||||
self._server = ServerRole(peer_id=peer_id, transport=transport, send=self._send)
|
||||
|
||||
if conn is not None:
|
||||
self.attach_connection(conn)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Properties
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
# Pending sessions (awaiting connection) are alive — only a
|
||||
# closed real connection counts as dead.
|
||||
return self._conn is None or self._conn.alive
|
||||
|
||||
@property
|
||||
def connected(self) -> bool:
|
||||
return self._conn is not None
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
"""True after the peer acked our ConnectMsg (we may send freely)."""
|
||||
return self._send_ready
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Connection lifecycle
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def attach_connection(self, conn: ControlConnection) -> None:
|
||||
"""Attach a connection to a pending session and announce ourselves.
|
||||
|
||||
Symmetric: every side advertises its NIXL metadata on connect, so
|
||||
whichever peer receives a session first can register the other.
|
||||
"""
|
||||
if self._conn is not None:
|
||||
raise ValueError(f"P2PSession {self.peer_id}: already connected")
|
||||
self._conn = conn
|
||||
self._send_connect()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def request_blocks(
|
||||
self,
|
||||
job_id: JobId,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[bytes],
|
||||
block_ids: Sequence[int],
|
||||
) -> None:
|
||||
"""Send fetch to the peer."""
|
||||
self._client.request_blocks(
|
||||
job_id, kv_request_id, keys, block_ids, send_ready=self._send_ready
|
||||
)
|
||||
|
||||
def add_stored_blocks(
|
||||
self,
|
||||
kv_request_id: str,
|
||||
keys: Sequence[OffloadKey],
|
||||
block_ids: Sequence[int],
|
||||
job_id: JobId,
|
||||
) -> None:
|
||||
"""New blocks stored locally — match against pending fetch demand."""
|
||||
self._server.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
|
||||
|
||||
def finish_request(self, kv_request_id: str) -> None:
|
||||
"""Called when the request is finishing locally.
|
||||
|
||||
Cancels any inbound load (client role) and finalizes any
|
||||
outbound serving (server role) for this id. Roles that aren't
|
||||
active for this id are silent no-ops.
|
||||
"""
|
||||
self._client.cancel(kv_request_id)
|
||||
self._server.finish(kv_request_id)
|
||||
|
||||
def poll(self) -> SessionPollResult:
|
||||
"""Process incoming messages, drive transfers, apply timeouts."""
|
||||
if self._conn is None:
|
||||
# Pending session — store-job timeouts still apply so buffered
|
||||
# jobs that never get picked up are surfaced as failures.
|
||||
return SessionPollResult(
|
||||
loads=[],
|
||||
stores=self._server.collect_idle_timeouts(),
|
||||
new_fetch_ids=[],
|
||||
)
|
||||
|
||||
for msg in self._conn.recv():
|
||||
self._on_message(msg)
|
||||
|
||||
loads = self._client.collect_results()
|
||||
stores = self._server.collect_results()
|
||||
self._server.drain_pending_aborts()
|
||||
|
||||
new_fetch_ids = self._new_fetch_ids
|
||||
self._new_fetch_ids = []
|
||||
return SessionPollResult(
|
||||
loads=loads, stores=stores, new_fetch_ids=new_fetch_ids
|
||||
)
|
||||
|
||||
def close(self) -> tuple[list[tuple[int, str]], list[int]]:
|
||||
"""Shut down. Returns (failed_loads, failed_stores).
|
||||
|
||||
failed_loads: list of (job_id, kv_request_id) pairs.
|
||||
failed_stores: list of job_ids.
|
||||
"""
|
||||
failed_loads = self._client.close()
|
||||
failed_stores = self._server.close()
|
||||
|
||||
if self._conn is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
return failed_loads, failed_stores
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Message dispatch
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _on_message(self, msg: dict) -> None:
|
||||
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else msg
|
||||
try:
|
||||
self._dispatch_message(msg)
|
||||
except ValueError as exc:
|
||||
# Protocol contract violation from the peer — *Msg.validate()
|
||||
# and handler-level checks raise ValueError. Retrying won't
|
||||
# help and may corrupt session state, so disconnect now.
|
||||
self._protocol_error(f"malformed {msg_type!r}: {exc}")
|
||||
return
|
||||
except Exception as exc:
|
||||
# Anything else is most likely an internal bug rather than a
|
||||
# peer fault. Log loudly with a traceback so it doesn't
|
||||
# disappear, but don't kill the session on a single hiccup.
|
||||
# Disconnect only if errors keep arriving — that pattern is
|
||||
# consistent with a peer wedging us into a broken state.
|
||||
self._dispatch_error_count += 1
|
||||
logger.exception(
|
||||
"P2PSession %s: error handling message %r (count=%d): %s",
|
||||
self.peer_id,
|
||||
msg_type,
|
||||
self._dispatch_error_count,
|
||||
exc,
|
||||
)
|
||||
if self._dispatch_error_count >= _MAX_CONSECUTIVE_DISPATCH_ERRORS:
|
||||
self._protocol_error(
|
||||
f"too many consecutive dispatch errors "
|
||||
f"({self._dispatch_error_count})"
|
||||
)
|
||||
return
|
||||
self._dispatch_error_count = 0
|
||||
|
||||
def _protocol_error(self, reason: str) -> None:
|
||||
"""Log a protocol violation and disconnect.
|
||||
|
||||
Best-effort sends ``DisconnectMsg`` so the peer learns why we're
|
||||
going away, then marks the connection dead. The manager reaps
|
||||
the session on the next poll via ``alive``.
|
||||
"""
|
||||
logger.error(
|
||||
"P2PSession %s: protocol error: %s — disconnecting",
|
||||
self.peer_id,
|
||||
reason,
|
||||
)
|
||||
if self._conn is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
|
||||
self._conn.mark_dead()
|
||||
|
||||
def _dispatch_message(self, msg: dict) -> None:
|
||||
# Drop messages buffered before disconnect: a poll batch can
|
||||
# contain msg-after-DisconnectMsg, and dispatching them would
|
||||
# mutate state on a dead session.
|
||||
if self._conn is not None and not self._conn.alive:
|
||||
return
|
||||
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else None
|
||||
if msg_type == ConnectMsg.TYPE:
|
||||
self._on_connect(msg)
|
||||
elif msg_type == ConnectAckMsg.TYPE:
|
||||
ConnectAckMsg.validate(msg)
|
||||
self._on_connect_ack()
|
||||
elif msg_type == FetchMsg.TYPE:
|
||||
FetchMsg.validate(msg)
|
||||
kv_request_id = msg[FetchMsg.KV_REQUEST_ID]
|
||||
block_hashes = [
|
||||
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
|
||||
for bh in msg[FetchMsg.BLOCK_HASHES]
|
||||
]
|
||||
block_indexes = msg[FetchMsg.BLOCK_INDEXES]
|
||||
# Run the server-role state machine inline as today —
|
||||
# add_fetch_demand records demand against any blocks we've
|
||||
# already seen in `available`. Report the kv_request_id so
|
||||
# the manager (after poll() returns) can replay any parked
|
||||
# submit_store batches; their add_stored_blocks calls hit
|
||||
# the demand recorded here and submit transfers immediately.
|
||||
self._server.on_fetch(kv_request_id, block_hashes, block_indexes)
|
||||
self._new_fetch_ids.append(kv_request_id)
|
||||
elif msg_type == AbortFetchMsg.TYPE:
|
||||
AbortFetchMsg.validate(msg)
|
||||
self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID])
|
||||
elif msg_type == TransferDoneMsg.TYPE:
|
||||
TransferDoneMsg.validate(msg)
|
||||
self._client.on_transfer_done(
|
||||
msg[TransferDoneMsg.KV_REQUEST_ID],
|
||||
msg[TransferDoneMsg.SUCCESS],
|
||||
)
|
||||
elif msg_type == AbortAckMsg.TYPE:
|
||||
AbortAckMsg.validate(msg)
|
||||
self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID])
|
||||
elif msg_type == DisconnectMsg.TYPE:
|
||||
if self._conn is not None:
|
||||
self._conn.mark_dead()
|
||||
else:
|
||||
logger.warning(
|
||||
"P2PSession %s: unknown message type %r", self.peer_id, msg_type
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Handshake
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _on_connect(self, msg: dict) -> None:
|
||||
# Validation failures here mean an incompatible or malicious peer.
|
||||
# Mark the connection dead so the manager reaps the session;
|
||||
# don't call add_remote_peer or send connect_ack.
|
||||
if self._send_ready:
|
||||
# We've already received connect_ack, so the handshake is
|
||||
# complete. A second connect from the peer is a protocol
|
||||
# violation — re-registering would corrupt transport state.
|
||||
self._protocol_error("duplicate connect after handshake")
|
||||
return
|
||||
try:
|
||||
ConnectMsg.validate(msg)
|
||||
if msg[ConnectMsg.BLOCK_LEN] != self._local_block_len:
|
||||
raise ValueError(
|
||||
f"block_len mismatch from {self.peer_id}: "
|
||||
f"remote={msg[ConnectMsg.BLOCK_LEN]}, "
|
||||
f"local={self._local_block_len}"
|
||||
)
|
||||
remote_fp = msg.get(ConnectMsg.CONFIG_FINGERPRINT, "")
|
||||
local_fp = self._transport.config_fingerprint
|
||||
if local_fp and remote_fp and remote_fp != local_fp:
|
||||
raise ValueError(
|
||||
f"config fingerprint mismatch from {self.peer_id}: "
|
||||
f"remote={remote_fp!r}, local={local_fp!r}"
|
||||
)
|
||||
self._transport.add_remote_peer(
|
||||
self.peer_id,
|
||||
agent_metadata=msg[ConnectMsg.AGENT_METADATA],
|
||||
base_addr=msg[ConnectMsg.BASE_ADDR],
|
||||
num_blocks=msg[ConnectMsg.NUM_BLOCKS],
|
||||
block_len=msg[ConnectMsg.BLOCK_LEN],
|
||||
)
|
||||
except ValueError as exc:
|
||||
logger.error("P2PSession %s: rejecting peer connect: %s", self.peer_id, exc)
|
||||
if self._conn is not None:
|
||||
self._conn.mark_dead()
|
||||
return
|
||||
|
||||
if self._conn is not None:
|
||||
self._conn.send(
|
||||
{
|
||||
TYPE_KEY: ConnectAckMsg.TYPE,
|
||||
ConnectAckMsg.PEER_ID: self._local_id,
|
||||
}
|
||||
)
|
||||
|
||||
def _on_connect_ack(self) -> None:
|
||||
if self._queued:
|
||||
logger.debug(
|
||||
"P2PSession %s: connect_ack received, flushing %d queued msg(s)",
|
||||
self.peer_id,
|
||||
len(self._queued),
|
||||
)
|
||||
self._send_ready = True
|
||||
for queued in self._queued:
|
||||
self._do_send(queued)
|
||||
self._queued.clear()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Send helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _send_connect(self) -> None:
|
||||
"""Send our ConnectMsg announcing local NIXL metadata."""
|
||||
assert self._conn is not None
|
||||
self._conn.send(
|
||||
{
|
||||
TYPE_KEY: ConnectMsg.TYPE,
|
||||
ConnectMsg.PEER_ID: self._local_id,
|
||||
ConnectMsg.AGENT_METADATA: self._transport.get_agent_metadata(),
|
||||
ConnectMsg.BASE_ADDR: self._transport.base_addr,
|
||||
ConnectMsg.NUM_BLOCKS: self._transport.num_blocks,
|
||||
ConnectMsg.BLOCK_LEN: self._transport.block_len,
|
||||
ConnectMsg.CONFIG_FINGERPRINT: self._transport.config_fingerprint,
|
||||
}
|
||||
)
|
||||
|
||||
def _send(self, msg: dict) -> None:
|
||||
if self._conn is None or not self._send_ready:
|
||||
logger.debug(
|
||||
"P2PSession %s: queueing %s (ready=%s queue_depth=%d)",
|
||||
self.peer_id,
|
||||
msg.get(TYPE_KEY),
|
||||
self._send_ready,
|
||||
len(self._queued) + 1,
|
||||
)
|
||||
self._queued.append(msg)
|
||||
return
|
||||
self._do_send(msg)
|
||||
|
||||
def _do_send(self, msg: dict) -> None:
|
||||
if self._conn is None:
|
||||
return
|
||||
try:
|
||||
self._conn.send(msg)
|
||||
logger.debug("P2PSession %s: sent %s", self.peer_id, msg.get(TYPE_KEY))
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"P2PSession %s: failed to send %s",
|
||||
self.peer_id,
|
||||
msg.get(TYPE_KEY),
|
||||
)
|
||||
Reference in New Issue
Block a user