Secondary tier implementation for PD disaggregation (#42285)

Signed-off-by: Liran Schour <lirans@il.ibm.com>
Signed-off-by: liranschour <liranschour@users.noreply.github.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
liranschour
2026-06-30 07:51:44 +03:00
committed by GitHub
co-authored by Or Ozeri Or Ozeri
parent b5c9e1ac33
commit bec232a914
22 changed files with 7471 additions and 0 deletions
+14
View File
@@ -120,6 +120,20 @@ To enable KV cache sharing between multiple vLLM instances using the same `root_
PYTHONHASHSEED=0 vllm serve ...
```
### P2P (Including P/D)
The P2P tier (`type: "p2p"`) shares completed KV blocks between vLLM instances over RDMA via NIXL. Each instance binds a control socket on `host:port` and exchanges blocks directly with peers — no shared filesystem required.
| Key | Required | Default | Notes |
| --- | --- | --- | --- |
| `type` | yes | — | Must be `p2p`. |
| `host` | no | `0.0.0.0` | Address the control socket binds to. |
| `port` | no | `7777` | Port for the control socket. Must be reachable from peers. |
| `backends` | no | `["UCX"]` | NIXL transport backends. See [NixlConnector Usage Guide](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin) for available backends and selection guidance. |
| `num_threads` | no | `4` | NIXL agent worker threads. Only used when `backends` is UCX-only; ignored when any non-UCX backend is requested. |
The `backends` and `num_threads` options mirror the conditional logic used by [`NixlConnector`](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin): when any non-UCX backend is configured, NIXL is initialised with `backends=...`; otherwise it falls back to a UCX-only agent with the configured `num_threads`. This lets the P2P tier use a different transport (e.g. `MOONCAKE`, `GDS_MT`, `LIBFABRIC`) than the main `NixlConnector` running in the same process.
## Tuning Tips
- `cpu_bytes_to_use`: a bigger CPU tier means fewer trips to slower secondary tiers and a higher hit rate. The value is total across all workers, not per-worker. Leave headroom for the rest of the host workload.
@@ -0,0 +1,331 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
P2PConnector proxy server for OffloadingConnector + TieringOffloadingSpec.
Unlike NixlConnector (which returns remote_host/remote_port in the prefill
response), OffloadingConnector does not embed connector coordinates in its
response. This proxy injects the prefiller's P2PConnector address into
kv_transfer_params before forwarding the decode request so the decoder knows
where to pull KV blocks from.
Usage:
.venv/bin/python p2p_connector_proxy.py \
--port 8192 \
--prefiller-host 127.0.0.1 --prefiller-port 8100 \
--decoder-host 127.0.0.1 --decoder-port 8200 \
--p2p-connector-host 127.0.0.1 --p2p-connector-port 7777
"""
import argparse
import asyncio
import itertools
import logging
import os
import uuid
from contextlib import asynccontextmanager
import httpx
from fastapi import FastAPI, Request
from fastapi.responses import StreamingResponse
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@asynccontextmanager
async def lifespan(app: FastAPI):
app.state.prefill_clients = []
app.state.decode_clients = []
for i, (host, port) in enumerate(global_args.prefiller_instances):
app.state.prefill_clients.append(
{
"client": httpx.AsyncClient(
timeout=None,
base_url=f"http://{host}:{port}/v1",
limits=httpx.Limits(
max_connections=None, max_keepalive_connections=None
),
),
"host": host,
"port": port,
"id": i,
}
)
for i, (host, port) in enumerate(global_args.decoder_instances):
app.state.decode_clients.append(
{
"client": httpx.AsyncClient(
timeout=None,
base_url=f"http://{host}:{port}/v1",
limits=httpx.Limits(
max_connections=None, max_keepalive_connections=None
),
),
"host": host,
"port": port,
"id": i,
}
)
app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients)))
app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients)))
mode = "decoder-first" if global_args.decoder_first else "prefiller-first"
pd_host = global_args.p2p_connector_host
pd_port = global_args.p2p_connector_port
print(
f"Proxy ready [{mode}]: "
f"{len(app.state.prefill_clients)} prefiller(s), "
f"{len(app.state.decode_clients)} decoder(s). "
f"P2PConnector at {pd_host}:{pd_port}"
)
yield
for ci in app.state.prefill_clients:
await ci["client"].aclose()
for ci in app.state.decode_clients:
await ci["client"].aclose()
app = FastAPI(lifespan=lifespan)
def parse_args():
p = argparse.ArgumentParser()
p.add_argument("--port", type=int, default=8192)
p.add_argument("--host", type=str, default="127.0.0.1")
p.add_argument("--prefiller-hosts", type=str, nargs="+", default=["127.0.0.1"])
p.add_argument("--prefiller-ports", type=int, nargs="+", default=[8100])
p.add_argument("--decoder-hosts", type=str, nargs="+", default=["127.0.0.1"])
p.add_argument("--decoder-ports", type=int, nargs="+", default=[8200])
# P2PConnector coordinates of the prefiller — injected into decode requests.
p.add_argument(
"--p2p-connector-host",
type=str,
default="127.0.0.1",
help="Host of the prefiller's P2PConnector ZMQ socket",
)
p.add_argument(
"--p2p-connector-port",
type=int,
default=7777,
help="Port of the prefiller's P2PConnector ZMQ socket",
)
# P2PConnector coordinates of the decoder — injected into prefill requests
# so the prefiller's submit_store can resolve the peer to push KV to.
p.add_argument(
"--decoder-p2p-connector-host",
type=str,
default="127.0.0.1",
help="Host of the decoder's P2PConnector ZMQ socket",
)
p.add_argument(
"--decoder-p2p-connector-port",
type=int,
default=7778,
help="Port of the decoder's P2PConnector ZMQ socket",
)
p.add_argument(
"--decoder-first",
action="store_true",
help="Send decode request before prefill so decoder is already "
"waiting when KV blocks arrive (decoder-first mode)",
)
args = p.parse_args()
if len(args.prefiller_hosts) != len(args.prefiller_ports):
raise ValueError("Prefiller host/port count mismatch")
if len(args.decoder_hosts) != len(args.decoder_ports):
raise ValueError("Decoder host/port count mismatch")
args.prefiller_instances = list(zip(args.prefiller_hosts, args.prefiller_ports))
args.decoder_instances = list(zip(args.decoder_hosts, args.decoder_ports))
return args
def _get_next(app, service: str):
if service == "prefill":
return app.state.prefill_clients[next(app.state.prefill_iterator)]
return app.state.decode_clients[next(app.state.decode_iterator)]
def _auth_headers(request_id: str) -> dict:
headers: dict = {"X-Request-Id": request_id}
api_key = os.environ.get("OPENAI_API_KEY", "")
if api_key:
headers["Authorization"] = f"Bearer {api_key}"
return headers
async def _prefill(client_info, endpoint, req_data, request_id):
"""Send a prefill-only request (max_tokens=1) to the prefiller."""
data = req_data.copy()
data["kv_transfer_params"] = {
"decode": {
"kv_request_id": request_id,
},
}
data["stream"] = False
data["max_tokens"] = 1
data.pop("max_completion_tokens", None)
data.pop("stream_options", None)
data.pop("min_tokens", None)
data.pop("min_completion_tokens", None)
headers = _auth_headers(request_id)
resp = await client_info["client"].post(endpoint, json=data, headers=headers)
resp.raise_for_status()
await resp.aread()
return resp
async def _stream_decode(client_info, endpoint, req_data, request_id):
headers = _auth_headers(request_id)
async with client_info["client"].stream(
"POST", endpoint, json=req_data, headers=headers
) as resp:
resp.raise_for_status()
async for chunk in resp.aiter_bytes():
yield chunk
async def _handle_completions(api: str, request: Request):
try:
req_data = await request.json()
request_id = str(uuid.uuid4())
prefill_client = _get_next(request.app, "prefill")
await _prefill(prefill_client, api, req_data, request_id)
# Inject the prefiller's P2PConnector address so the decoder can pull
# KV blocks from it via the P2PConnector transport.
req_data["kv_transfer_params"] = {
"prefill": {
"kv_request_id": request_id,
"remote_host": global_args.p2p_connector_host,
"remote_port": global_args.p2p_connector_port,
},
}
decode_client = _get_next(request.app, "decode")
logger.debug("prefill=%s decode=%s", prefill_client, decode_client)
async def generate():
async for chunk in _stream_decode(decode_client, api, req_data, request_id):
yield chunk
return StreamingResponse(generate(), media_type="application/json")
except Exception as e:
import sys
import traceback
print(f"Proxy error on {api}: {e}")
print("".join(traceback.format_exception(*sys.exc_info())))
raise
async def _handle_completions_decoder_first(api: str, request: Request):
"""Decoder-first mode: send decode request before prefill.
The decoder establishes its request and starts polling for KV blocks
immediately. The prefill is then sent so the prefiller computes and
pushes blocks to the already-waiting decoder.
"""
try:
req_data = await request.json()
request_id = str(uuid.uuid4())
prefill_client = _get_next(request.app, "prefill")
decode_client = _get_next(request.app, "decode")
decode_data = req_data.copy()
decode_data["kv_transfer_params"] = {
"prefill": {
"kv_request_id": request_id,
"remote_host": global_args.p2p_connector_host,
"remote_port": global_args.p2p_connector_port,
},
}
async def generate():
queue: asyncio.Queue = asyncio.Queue()
async def _run_decode():
try:
async for chunk in _stream_decode(
decode_client, api, decode_data, request_id
):
await queue.put(("data", chunk))
except Exception as exc:
await queue.put(("error", exc))
finally:
await queue.put(("done", None))
# 1. Start decode request — decoder is now waiting for KV blocks
asyncio.create_task(_run_decode())
# 2. Send prefill — blocks are computed and pushed to the decoder
try:
await _prefill(prefill_client, api, req_data, request_id)
except Exception as exc:
logger.warning("decoder-first: prefill failed: %s", exc)
logger.debug(
"decoder-first: prefill done, streaming decode prefill=%s decode=%s",
prefill_client,
decode_client,
)
# 3. Stream the decode response
while True:
kind, value = await queue.get()
if kind == "done":
break
if kind == "error":
raise value # type: ignore[misc]
yield value
return StreamingResponse(generate(), media_type="application/json")
except Exception as e:
import sys
import traceback
print(f"Proxy error on {api}: {e}")
print("".join(traceback.format_exception(*sys.exc_info())))
raise
def _route_handler(api: str):
if global_args.decoder_first:
return lambda req: _handle_completions_decoder_first(api, req)
return lambda req: _handle_completions(api, req)
@app.post("/v1/completions")
async def completions(request: Request):
return await _route_handler("/completions")(request)
@app.post("/v1/chat/completions")
async def chat_completions(request: Request):
return await _route_handler("/chat/completions")(request)
@app.get("/healthcheck")
async def healthcheck():
return {
"status": "ok",
"prefill_instances": len(app.state.prefill_clients),
"decode_instances": len(app.state.decode_clients),
}
if __name__ == "__main__":
global global_args
global_args = parse_args()
import uvicorn
uvicorn.run(app, host=global_args.host, port=global_args.port)
+312
View File
@@ -0,0 +1,312 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Accuracy test driver for the p2p connector
# (OffloadingConnector + TieringOffloadingSpec + p2p tier).
#
# Mirrors tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh:
# brings up N prefillers + M decoders on the local host, fronts them
# with p2p_connector_proxy.py, then runs the connector-agnostic
# test_accuracy.py (gsm8k via lm_eval) against the proxy.
#
# Knobs (env vars unless flagged otherwise):
# MODEL_NAMES space-separated model list (default: Llama-3.2-1B-Instruct)
# NUM_PREFILL_INSTANCES default 1
# NUM_DECODE_INSTANCES default 1
# PREFILLER_TP_SIZE default 1
# DECODER_TP_SIZE default 1
# GPU_MEMORY_UTILIZATION default 0.45
# MAX_MODEL_LEN default 512
# PREFILL_BLOCK_SIZE default 128
# DECODE_BLOCK_SIZE default 128
# CPU_BYTES default 209715200 (200 MB)
# VLLM_SERVE_EXTRA_ARGS comma-separated extra args for vllm serve
# --decoder-first toggle decoder-first proxy mode
#
# Examples:
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
# NUM_PREFILL_INSTANCES=2 NUM_DECODE_INSTANCES=2 \
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh
# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh --decoder-first
set -xe
# ---------------------------------------------------------------------------
# Args
# ---------------------------------------------------------------------------
DECODER_FIRST="false"
while [[ $# -gt 0 ]]; do
case $1 in
--decoder-first)
DECODER_FIRST="true"
shift 1
;;
*)
echo "Unknown option $1"
echo "Usage: $0 [--decoder-first]"
exit 1
;;
esac
done
# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------
MODEL_NAMES=${MODEL_NAMES:-}
if [[ -n "$MODEL_NAMES" ]]; then
# shellcheck disable=SC2206
MODELS=($MODEL_NAMES)
else
MODELS=(
"meta-llama/Llama-3.2-1B-Instruct"
)
fi
# ---------------------------------------------------------------------------
# Defaults
# ---------------------------------------------------------------------------
NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1}
NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1}
PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.45}
MAX_MODEL_LEN=${MAX_MODEL_LEN:-512}
PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128}
DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128}
CPU_BYTES=${CPU_BYTES:-209715200}
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
# Base ports — per-instance offsets layered on top.
PREFILL_HTTP_BASE=8100
DECODE_HTTP_BASE=8200
PREFILL_PD_BASE=7777
DECODE_PD_BASE=$((PREFILL_PD_BASE + NUM_PREFILL_INSTANCES))
PROXY_PORT=8192
P2P_HOST=127.0.0.1
# ---------------------------------------------------------------------------
# Resolve repo root + venv (works in .venv and /workspace/venv pods)
# ---------------------------------------------------------------------------
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../../.." && pwd -P)}"
if [[ -z "${VLLM_BIN:-}" ]]; then
if [[ -x "${GIT_ROOT}/.venv/bin/vllm" ]]; then
VLLM_BIN="${GIT_ROOT}/.venv/bin/vllm"
elif [[ -x "/workspace/venv/bin/vllm" ]]; then
VLLM_BIN="/workspace/venv/bin/vllm"
else
VLLM_BIN="$(command -v vllm)"
fi
fi
if [[ -z "${PYTHON_BIN:-}" ]]; then
if [[ -x "${GIT_ROOT}/.venv/bin/python" ]]; then
PYTHON_BIN="${GIT_ROOT}/.venv/bin/python"
elif [[ -x "/workspace/venv/bin/python" ]]; then
PYTHON_BIN="/workspace/venv/bin/python"
else
PYTHON_BIN="$(command -v python3 || command -v python)"
fi
fi
echo "Using vllm: ${VLLM_BIN}"
echo "Using python: ${PYTHON_BIN}"
SMI_BIN=$(command -v nvidia-smi || command -v rocm-smi || echo "")
# Trap SIGINT/SIGTERM/EXIT to kill background jobs.
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
wait_for_server() {
local port=$1
timeout 1200 bash -c "
until curl -s localhost:${port}/v1/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
cleanup_instances() {
echo "Cleaning up any running vLLM / proxy instances..."
pkill -f "vllm serve" || true
pkill -f "p2p_connector_proxy.py" || true
sleep 2
}
get_num_gpus() {
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
elif [[ "$SMI_BIN" == *"rocm"* ]]; then
$SMI_BIN -l | grep -c GPU
else
echo "1"
fi
}
# Build the OffloadingConnector kv-transfer-config for a given PD port.
# Mirrors deploy_local.sh:131.
build_kv_config() {
local pd_port=$1
printf '{"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":%s,"secondary_tiers":[{"type":"p2p","host":"%s","port":%s}]}}' \
"${CPU_BYTES}" "${P2P_HOST}" "${pd_port}"
}
# ---------------------------------------------------------------------------
# Per-model run
# ---------------------------------------------------------------------------
run_tests_for_model() {
local model_name=$1
echo "================================"
echo "Testing model: $model_name"
echo " prefillers=${NUM_PREFILL_INSTANCES} (tp=${PREFILLER_TP_SIZE})"
echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})"
echo " decoder_first=${DECODER_FIRST}"
echo "================================"
PREFILL_HOSTS=()
PREFILL_PORTS=()
PREFILL_PD_PORTS=()
DECODE_HOSTS=()
DECODE_PORTS=()
DECODE_PD_PORTS=()
local num_gpus
num_gpus=$(get_num_gpus)
local next_gpu=0
# ---- Prefillers ----
for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do
local gpu_id=$((i * PREFILLER_TP_SIZE % num_gpus))
local cuda_devs="${gpu_id}"
for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
done
next_gpu=$(((gpu_id + PREFILLER_TP_SIZE) % num_gpus))
local http_port=$((PREFILL_HTTP_BASE + i))
local pd_port=$((PREFILL_PD_BASE + i))
local kv_cfg
kv_cfg=$(build_kv_config "${pd_port}")
echo "Prefiller $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
PYTHONHASHSEED=42 \
${VLLM_BIN} serve ${model_name} \
--port ${http_port} \
--enforce-eager \
--block-size ${PREFILL_BLOCK_SIZE} \
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
--max-model-len ${MAX_MODEL_LEN} \
--tensor-parallel-size ${PREFILLER_TP_SIZE} \
--kv-transfer-config '${kv_cfg}'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
BASE_CMD="${BASE_CMD} $arg"
done
fi
eval "${BASE_CMD} &"
PREFILL_HOSTS+=("${P2P_HOST}")
PREFILL_PORTS+=("${http_port}")
PREFILL_PD_PORTS+=("${pd_port}")
done
# ---- Decoders ----
for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do
local gpu_id=$(((next_gpu + i * DECODER_TP_SIZE) % num_gpus))
local cuda_devs="${gpu_id}"
for (( j=1; j < DECODER_TP_SIZE; j++ )); do
cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))"
done
local http_port=$((DECODE_HTTP_BASE + i))
local pd_port=$((DECODE_PD_BASE + i))
local kv_cfg
kv_cfg=$(build_kv_config "${pd_port}")
echo "Decoder $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}"
BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \
PYTHONHASHSEED=42 \
${VLLM_BIN} serve ${model_name} \
--port ${http_port} \
--enforce-eager \
--block-size ${DECODE_BLOCK_SIZE} \
--gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
--max-model-len ${MAX_MODEL_LEN} \
--tensor-parallel-size ${DECODER_TP_SIZE} \
--kv-transfer-config '${kv_cfg}'"
if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then
IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS"
for arg in "${extra_args[@]}"; do
BASE_CMD="${BASE_CMD} $arg"
done
fi
eval "${BASE_CMD} &"
DECODE_HOSTS+=("${P2P_HOST}")
DECODE_PORTS+=("${http_port}")
DECODE_PD_PORTS+=("${pd_port}")
done
# ---- Wait for HTTP readiness ----
for port in "${PREFILL_PORTS[@]}"; do
echo "Waiting for prefill instance on port $port to start..."
wait_for_server "$port"
done
for port in "${DECODE_PORTS[@]}"; do
echo "Waiting for decode instance on port $port to start..."
wait_for_server "$port"
done
# ---- Proxy ----
# The proxy currently advertises a single prefiller PD address to decoders.
# For the 1xM and matched NxM common cases the first prefiller's PD coords
# are the right pick; multi-prefiller PD round-robin is a follow-up.
PROXY_CMD="${PYTHON_BIN} ${SCRIPT_DIR}/p2p_connector_proxy.py \
--port ${PROXY_PORT} \
--host ${P2P_HOST} \
--prefiller-hosts ${PREFILL_HOSTS[*]} \
--prefiller-ports ${PREFILL_PORTS[*]} \
--decoder-hosts ${DECODE_HOSTS[*]} \
--decoder-ports ${DECODE_PORTS[*]} \
--p2p-connector-host ${P2P_HOST} \
--p2p-connector-port ${PREFILL_PD_PORTS[0]} \
--decoder-p2p-connector-host ${P2P_HOST} \
--decoder-p2p-connector-port ${DECODE_PD_PORTS[0]}"
if [[ "${DECODER_FIRST}" == "true" ]]; then
PROXY_CMD="${PROXY_CMD} --decoder-first"
fi
echo "Starting proxy: ${PROXY_CMD}"
eval "${PROXY_CMD} &"
sleep 5
# ---- Run accuracy test (reused from nixl_integration) ----
echo "Running tests for $model_name"
TEST_MODEL=$model_name "${PYTHON_BIN}" -m pytest -s -x \
"${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py"
cleanup_instances
sleep 3
}
# ---------------------------------------------------------------------------
# Drive
# ---------------------------------------------------------------------------
for model in "${MODELS[@]}"; do
run_tests_for_model "$model"
done
echo "All tests completed!"
@@ -0,0 +1,330 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for DataTransport base class and NixlTransport."""
from __future__ import annotations
import ctypes
from unittest.mock import MagicMock, patch
import numpy as np
from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
# ---------------------------------------------------------------------------
# DataTransport base class tests
# ---------------------------------------------------------------------------
class TestDataTransportBase:
"""Tests for the DataTransport abstract base properties."""
def _make_view(self, num_blocks: int = 8, block_len: int = 1024) -> memoryview:
"""Create a memoryview with the given shape."""
buf = np.zeros((num_blocks, block_len), dtype=np.uint8)
return memoryview(buf)
def test_properties(self):
"""base_addr, num_blocks, block_len are set from memoryview shape."""
view = self._make_view(num_blocks=4, block_len=2048)
# Use NixlTransport (concrete) with NIXL mocked away
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view)
assert transport.num_blocks == 4
assert transport.block_len == 2048
assert transport.base_addr == ctypes.addressof(ctypes.c_char.from_buffer(view))
def test_config_fingerprint_empty_when_no_fields(self):
"""No config fields → empty fingerprint."""
view = self._make_view()
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view, config_fields=None)
assert transport.config_fingerprint == ""
def test_config_fingerprint_deterministic(self):
"""Same config fields → same fingerprint."""
view = self._make_view()
fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1}
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
t1 = NixlTransport("test:1", view, config_fields=fields)
t2 = NixlTransport("test:2", view, config_fields=fields)
assert t1.config_fingerprint == t2.config_fingerprint
assert len(t1.config_fingerprint) == 16
def test_config_fingerprint_differs_for_different_fields(self):
"""Different config fields → different fingerprint."""
view = self._make_view()
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
t1 = NixlTransport("test:1", view, config_fields={"model": "a"})
t2 = NixlTransport("test:2", view, config_fields={"model": "b"})
assert t1.config_fingerprint != t2.config_fingerprint
# ---------------------------------------------------------------------------
# NixlTransport tests (with mocked NIXL agent)
# ---------------------------------------------------------------------------
class TestNixlTransportWithMockedAgent:
"""Tests for NixlTransport logic with a mocked NIXL agent."""
def _make_transport(self) -> NixlTransport:
"""Create a NixlTransport with mocked NIXL internals."""
view = memoryview(np.zeros((8, 1024), dtype=np.uint8))
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view)
# Manually set up a mock agent after construction
agent = MagicMock()
agent.add_remote_agent.return_value = "nixl-peer-name"
agent.get_xfer_descs.return_value = MagicMock()
agent.prep_xfer_dlist.return_value = MagicMock()
agent.make_prepped_xfer.return_value = MagicMock(name="handle")
agent.transfer.return_value = None
agent.check_xfer_state.return_value = "PROC"
agent.get_agent_metadata.return_value = b"test-metadata"
transport._agent = agent
transport._local_dlist = MagicMock()
return transport
def test_available_false_without_nixl(self):
"""Without NIXL installed, available is False."""
view = memoryview(np.zeros((4, 512), dtype=np.uint8))
with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None):
transport = NixlTransport("test:1", view)
assert transport.available is False
def test_available_true_with_agent(self):
transport = self._make_transport()
assert transport.available is True
def test_get_agent_metadata(self):
transport = self._make_transport()
assert transport.get_agent_metadata() == b"test-metadata"
def test_write_blocks_returns_none_for_unknown_peer(self):
"""write_blocks returns None if peer not registered."""
transport = self._make_transport()
result = transport.write_blocks("unknown:1", [0, 1], [2, 3])
assert result is None
def test_write_blocks_returns_transfer_id(self):
"""write_blocks returns an integer transfer_id on success."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0, 1], [2, 3])
assert tid is not None
assert isinstance(tid, int)
def test_write_blocks_increments_transfer_id(self):
"""Each write_blocks call gets a unique transfer_id."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid1 = transport.write_blocks("peer:1", [0], [1])
tid2 = transport.write_blocks("peer:1", [2], [3])
assert tid1 != tid2
def test_poll_empty_when_no_inflight(self):
"""poll returns empty when nothing is inflight."""
transport = self._make_transport()
result = transport.poll()
assert result == PollResult(done=(), failed=())
def test_poll_returns_done_when_transfer_completes(self):
"""Completed transfer appears in poll().done."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
# Simulate completion
transport._agent.check_xfer_state.return_value = "DONE"
result = transport.poll()
assert tid in result.done
assert result.failed == ()
# Handle released
transport._agent.release_xfer_handle.assert_called()
def test_poll_returns_failed_for_error_state(self):
"""Transfer in error state appears in poll().failed."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = "ERR"
result = transport.poll()
assert result.done == ()
assert tid in result.failed
def test_poll_ignores_in_progress(self):
"""Transfers in PROC/PEND state stay inflight."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
transport.write_blocks("peer:1", [0], [1])
transport._agent.check_xfer_state.return_value = "PROC"
result = transport.poll()
assert result.done == ()
assert result.failed == ()
transport._agent.check_xfer_state.return_value = "PEND"
result = transport.poll()
assert result.done == ()
assert result.failed == ()
def test_cancel_removes_inflight(self):
"""cancel removes transfers and releases handles."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
assert tid in transport._inflight
result = transport.cancel([tid])
assert result == []
assert tid not in transport._inflight
transport._agent.release_xfer_handle.assert_called()
def test_cancel_ignores_unknown_ids(self):
"""cancel with unknown IDs doesn't crash."""
transport = self._make_transport()
assert transport.cancel([999, 1000]) == []
assert transport.cancel([999, 1000], mode="wait") == []
def test_cancel_wait_release_succeeds(self):
"""wait-mode cancel that succeeds pops the entry and returns []."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
assert tid in transport._inflight
result = transport.cancel([tid], mode="wait")
assert result == []
assert tid not in transport._inflight
transport._agent.release_xfer_handle.assert_called_once()
def test_cancel_wait_release_raises(self):
"""wait-mode cancel keeps the entry and returns the tid on raise."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.release_xfer_handle.side_effect = RuntimeError(
"NIXL_ERR_REPOST_ACTIVE"
)
result = transport.cancel([tid], mode="wait")
assert result == [tid]
assert tid in transport._inflight
def test_cancel_wait_then_poll_completes(self):
"""A wait-cancel that left a tid pending later completes via poll."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
tid = transport.write_blocks("peer:1", [0], [1])
transport._agent.release_xfer_handle.side_effect = RuntimeError("busy")
assert transport.cancel([tid], mode="wait") == [tid]
assert tid in transport._inflight
transport._agent.release_xfer_handle.side_effect = None
transport._agent.check_xfer_state.return_value = "DONE"
result = transport.poll()
assert tid in result.done
assert tid not in transport._inflight
def test_add_and_remove_remote_peer(self):
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
assert "peer:1" in transport._remote_dlists
transport.remove_remote_peer("peer:1")
assert "peer:1" not in transport._remote_dlists
transport._agent.release_dlist_handle.assert_called()
transport._agent.remove_remote_agent.assert_called()
def test_close_releases_everything(self):
"""close releases all handles and clears state."""
transport = self._make_transport()
transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024)
transport.write_blocks("peer:1", [0], [1])
transport.close()
assert transport._agent is None
assert transport._inflight == {}
assert transport._remote_dlists == {}
# ---------------------------------------------------------------------------
# NIXL agent-config selection
# ---------------------------------------------------------------------------
class TestNixlAgentConfigSelection:
"""Tests that backends/num_threads pick the right nixl_agent_config call.
Mirrors the conditional in
vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py:325-329.
"""
def _make_view(self) -> memoryview:
return memoryview(np.zeros((4, 512), dtype=np.uint8))
def test_non_ucx_backends_passes_backends_kwarg(self):
"""When any non-UCX backend is requested, pass backends + telemetry."""
agent_cls = MagicMock()
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
with (
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
patch(
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
),
):
NixlTransport("test:1", self._make_view(), backends=["MOONCAKE"])
config_fn.assert_called_once_with(backends=["MOONCAKE"], capture_telemetry=True)
# num_threads must NOT be passed on the non-UCX branch.
assert "num_threads" not in config_fn.call_args.kwargs
def test_ucx_only_passes_num_threads(self):
"""UCX-only configuration passes num_threads + telemetry, no backends."""
agent_cls = MagicMock()
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
with (
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
patch(
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
),
):
NixlTransport("test:1", self._make_view(), num_threads=8)
config_fn.assert_called_once_with(num_threads=8, capture_telemetry=True)
assert "backends" not in config_fn.call_args.kwargs
def test_default_backends_is_ucx_only(self):
"""No backends arg → defaults to UCX-only branch."""
agent_cls = MagicMock()
config_fn = MagicMock(return_value=MagicMock(name="cfg"))
with (
patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls),
patch(
"vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn
),
):
NixlTransport("test:1", self._make_view())
# Default num_threads=4, no backends kwarg.
config_fn.assert_called_once_with(num_threads=4, capture_telemetry=True)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for vllm.v1.kv_offload.tiering.p2p.control.zmq."""
from __future__ import annotations
import socket
import time
import pytest
import zmq
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
ZmqConnection,
ZmqTransport,
_Sockets,
)
def _free_port() -> int:
"""Find a free TCP port."""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return s.getsockname()[1]
def _make_transport(host: str = "127.0.0.1", attempts: int = 8):
"""Construct a ZmqTransport on a fresh port, retrying on bind collisions.
Why: _free_port() releases the probe socket before ZmqTransport binds the
same port — a parallel test run can steal it in between. Retrying on
ZMQError/OSError closes that race without a production change.
"""
last_err: Exception | None = None
for _ in range(attempts):
port = _free_port()
try:
return ZmqTransport(f"{host}:{port}", host, port), port
except (zmq.ZMQError, OSError) as e:
last_err = e
assert last_err is not None
raise last_err
def _wait_for_inbound(transport: ZmqTransport, deadline: float = 2.0):
"""Poll until at least one new inbound connection is accepted, or fail."""
end = time.monotonic() + deadline
while time.monotonic() < end:
new = transport.poll()
if new:
return new
time.sleep(0.005)
raise AssertionError(f"no inbound connection within {deadline}s")
def _wait_for_messages(
transport: ZmqTransport,
conn: ZmqConnection,
n: int,
deadline: float = 2.0,
) -> list[dict]:
"""Poll until `conn` has received at least `n` messages, then return them."""
end = time.monotonic() + deadline
msgs: list[dict] = []
while time.monotonic() < end:
transport.poll()
msgs.extend(conn.recv())
if len(msgs) >= n:
return msgs
time.sleep(0.005)
raise AssertionError(f"got {len(msgs)}/{n} messages within {deadline}s")
def _make_mock_connection(peer_id: str = "test:1234") -> ZmqConnection:
"""Create a ZmqConnection with mock sockets for unit testing."""
from unittest.mock import MagicMock
sockets = _Sockets(dealer=MagicMock(), monitor=MagicMock())
return ZmqConnection(peer_id, sockets)
class TestZmqConnection:
"""Tests for ZmqConnection in isolation (no real sockets)."""
def test_enqueue_and_recv(self):
"""Messages enqueued are returned by recv() in order."""
conn = _make_mock_connection()
conn.enqueue({"type": "a"})
conn.enqueue({"type": "b"})
msgs = conn.recv()
assert list(msgs) == [{"type": "a"}, {"type": "b"}]
# Second recv is empty
assert not conn.recv()
def test_recv_returns_empty_initially(self):
conn = _make_mock_connection()
assert not conn.recv()
def test_alive_initially_true(self):
conn = _make_mock_connection()
assert conn.alive is True
def test_mark_dead(self):
conn = _make_mock_connection()
conn.mark_dead()
assert conn.alive is False
def test_send_raises_when_closed(self):
conn = _make_mock_connection()
conn.mark_dead()
with pytest.raises(RuntimeError, match="closed connection"):
conn.send({"type": "test"})
class TestZmqTransportConnectivity:
"""Integration tests for ZmqTransport with real ZMQ sockets."""
def test_connect_and_send_message(self):
"""Two transports can connect and exchange messages."""
transport_a, port_a = _make_transport()
transport_b, port_b = _make_transport()
try:
peer_a_id = f"127.0.0.1:{port_a}"
conn_b_to_a = transport_b.connect(peer_a_id)
conn_b_to_a.send({"type": "hello", "data": 42})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
conn_a_from_b = new_conns[0]
assert conn_a_from_b.peer_id == f"127.0.0.1:{port_b}"
msgs = _wait_for_messages(transport_a, conn_a_from_b, 1)
assert msgs == [{"type": "hello", "data": 42}]
finally:
transport_a.close()
transport_b.close()
def test_bidirectional_messaging(self):
"""Both sides can send and receive after connection."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"type": "connect", "from": "b"})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
conn_a = new_conns[0]
conn_a.send({"type": "reply", "from": "a"})
msgs = _wait_for_messages(transport_b, conn_b, 1)
assert msgs == [{"type": "reply", "from": "a"}]
finally:
transport_a.close()
transport_b.close()
def test_poll_returns_empty_when_no_connections(self):
transport, _ = _make_transport()
try:
assert not transport.poll()
finally:
transport.close()
def test_multiple_messages(self):
"""Multiple messages are buffered and returned together."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"seq": 1})
conn_b.send({"seq": 2})
conn_b.send({"seq": 3})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
conn_a = new_conns[0]
msgs = _wait_for_messages(transport_a, conn_a, 3)
assert [m["seq"] for m in msgs] == [1, 2, 3]
finally:
transport_a.close()
transport_b.close()
def test_duplicate_connect_asserts(self):
"""Connecting to the same peer twice raises AssertionError."""
# port_a is never bound — we just need a syntactically-valid peer id.
port_a = _free_port()
transport_b, _ = _make_transport()
try:
transport_b.connect(f"127.0.0.1:{port_a}")
with pytest.raises(AssertionError, match="already exists"):
transport_b.connect(f"127.0.0.1:{port_a}")
finally:
transport_b.close()
def test_dead_connection_removed_on_poll(self):
"""Dead connections are cleaned up during poll."""
transport_a, port_a = _make_transport()
transport_b, _ = _make_transport()
try:
conn_b = transport_b.connect(f"127.0.0.1:{port_a}")
conn_b.send({"type": "hello"})
new_conns = _wait_for_inbound(transport_a)
assert len(new_conns) == 1
# Mark the inbound connection dead manually.
new_conns[0].mark_dead()
# Pruning is synchronous within poll().
transport_a.poll()
assert len(transport_a._connections) == 0
finally:
transport_a.close()
transport_b.close()
def test_close_is_idempotent(self):
"""Calling close() twice doesn't raise."""
transport, _ = _make_transport()
transport.close()
transport.close() # should not raise
+6
View File
@@ -66,6 +66,12 @@ SecondaryTierFactory.register_tier(
"FileSystemTierManager",
)
SecondaryTierFactory.register_tier(
"p2p",
"vllm.v1.kv_offload.tiering.p2p.manager",
"P2PSecondaryTierManager",
)
SecondaryTierFactory.register_tier(
"obj",
"vllm.v1.kv_offload.tiering.obj.manager",
@@ -0,0 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.kv_offload.tiering.p2p.control.base import (
ControlConnection,
ControlTransport,
)
from vllm.v1.kv_offload.tiering.p2p.control.zmq import (
ZmqConnection,
ZmqTransport,
)
__all__ = [
"ControlConnection",
"ControlTransport",
"ZmqConnection",
"ZmqTransport",
]
@@ -0,0 +1,166 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base classes for the P2P control-plane transport.
The control plane handles peer discovery, connection lifecycle, and
message routing. It is message-content agnostic — it moves opaque
dicts between peers without interpreting them.
Architecture
------------
ControlTransport (one per node)
├── listen for inbound connections
├── connect() to outbound peers
└── poll() → new ControlConnections
ControlConnection (one per peer)
├── send(msg) — enqueue a message to the peer
├── recv() — drain buffered inbound messages
├── mark_dead() — signal that the peer is gone
└── close() — tear down the connection
Threading model: all I/O is driven by the caller invoking poll().
No background threads. poll() must be called periodically to:
- receive messages (buffered per-connection)
- accept new inbound peers
- detect disconnections
Implementor contracts
---------------------
- ControlConnection.send() must not block. Messages are serialized
and queued for the next I/O pass.
- ControlConnection.recv() returns all messages received since the
last call (may be empty). Messages are dicts (already deserialized).
- ControlConnection.alive returns False after mark_dead() or close().
- ControlTransport.poll() returns newly accepted connections only —
not previously returned ones. Each connection appears exactly once.
- ControlTransport.connect() creates an outbound connection to a peer
identified by peer_id (format: "host:port"). Raises on failure.
- Messages may arrive from unknown peers (new inbound connections).
The transport creates a ControlConnection and returns it from poll()
with the first message(s) already in its recv() buffer.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import Sequence
class ControlConnection(ABC):
"""Bidirectional message channel to a single remote peer.
Lifecycle:
1. Created by ControlTransport (connect() or poll())
2. Used for send/recv by sessions
3. Marked dead on disconnect (mark_dead())
4. Cleaned up with close()
Once mark_dead() is called, alive becomes False and the owning
session should stop using this connection. close() releases
underlying resources (sockets, monitors).
"""
def __init__(self, peer_id: str) -> None:
self.peer_id = peer_id
@property
@abstractmethod
def alive(self) -> bool:
"""True if the connection is usable. False after mark_dead/close."""
...
@abstractmethod
def send(self, msg: dict) -> None:
"""Enqueue a message for delivery to the peer.
Must not block. Serialization happens internally.
Raises on closed connection.
"""
...
@abstractmethod
def recv(self) -> Sequence[dict]:
"""Drain and return all messages received since the last call.
Returns an empty sequence if no messages are pending.
The returned sequence is read-only — callers must not mutate it.
Messages are dicts deserialized from the wire format.
"""
...
@abstractmethod
def mark_dead(self) -> None:
"""Mark this connection as dead (peer disconnected).
After this call, alive returns False. The session should
stop using this connection and the transport will clean it up.
"""
...
@abstractmethod
def close(self) -> None:
"""Release all resources (sockets, monitors).
Idempotent. After close(), alive returns False.
"""
...
class ControlTransport(ABC):
"""Manages peer connections and drives all control-plane I/O.
Owns the listening socket and all active connections.
The caller must invoke poll() periodically to process I/O.
Lifecycle:
1. Constructed with a local identity and listen address
2. connect() to reach remote peers
3. poll() to accept inbound peers and process messages
4. close() to shut down
"""
@abstractmethod
def connect(self, peer_id: str) -> ControlConnection:
"""Create an outbound connection to a remote peer.
Args:
peer_id: Remote peer identity (format: "host:port").
Returns:
A new ControlConnection ready for send/recv.
The connection's send queue is live immediately — messages
sent before the remote peer's poll() will be buffered.
"""
...
@abstractmethod
def poll(self) -> Sequence[ControlConnection]:
"""Process all pending I/O and return newly accepted connections.
This is the main I/O driver. Each call:
- Receives messages from all connected peers (buffered in
each connection's recv() queue)
- Accepts new inbound peers and creates connections for them
(first message already in recv() buffer)
- Detects disconnections and marks connections dead
Returns:
Newly accepted inbound connections (not previously returned).
The returned sequence is read-only — callers must not mutate
it. The caller is responsible for creating sessions for
these connections.
"""
...
@abstractmethod
def close(self) -> None:
"""Shut down the transport and all connections.
Closes the listening socket and all active connections.
Idempotent.
"""
...
@@ -0,0 +1,307 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ZMQ-based transport layer for P2P KV cache sharing.
Provides ZmqConnection (per-peer messaging) and ZmqTransport (connection
management). Message-content agnostic.
"""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
import msgspec
import zmq
import zmq.utils.monitor
from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.p2p.control.base import (
ControlConnection,
ControlTransport,
)
logger = init_logger(__name__)
_HEARTBEAT_IVL_MS = 2000
_HEARTBEAT_TIMEOUT_MS = 10000
_HEARTBEAT_TTL_MS = 10000
# Shared sentinels returned when there is nothing to report.
_EMPTY_INBOX: tuple[dict, ...] = ()
_EMPTY_NEW_CONNECTIONS: tuple[ControlConnection, ...] = ()
def _tcp_addr(host: str, port: int | str) -> str:
return f"tcp://{host}:{port}"
def _apply_heartbeat(sock: zmq.Socket) -> None:
sock.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
sock.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
sock.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
@dataclass
class _Sockets:
dealer: zmq.Socket
monitor: zmq.Socket
class ZmqConnection(ControlConnection):
"""Bidirectional message channel to a single remote peer."""
def __init__(self, peer_id: str, sockets: _Sockets) -> None:
super().__init__(peer_id)
self._sockets = sockets
self._closed = False
self._inbox: list[dict] = []
def send(self, msg: dict) -> None:
"""Send a msgpack-encoded message to this peer."""
if self._closed:
raise RuntimeError(
f"ZmqConnection: send on closed connection to {self.peer_id}"
)
data = msgspec.msgpack.encode(msg)
self._sockets.dealer.send(data)
def recv(self) -> Sequence[dict]:
"""Drain and return all buffered incoming messages."""
if not self._inbox:
return _EMPTY_INBOX
msgs = self._inbox
self._inbox = []
return msgs
@property
def alive(self) -> bool:
return not self._closed
def close(self) -> None:
if self._closed:
return
self._closed = True
logger.info("ZmqConnection: closing connection to %s", self.peer_id)
self._sockets.monitor.close()
self._sockets.dealer.close()
def enqueue(self, msg: dict) -> None:
"""Buffer an incoming message."""
self._inbox.append(msg)
def mark_dead(self) -> None:
"""Mark connection as disconnected."""
self._closed = True
@property
def monitor_socket(self) -> zmq.Socket:
"""Monitor socket for disconnect detection (used by ZmqTransport)."""
return self._sockets.monitor
class ZmqTransport(ControlTransport):
"""ZMQ implementation of ControlTransport.
Manages a ROUTER socket for accepting connections and DEALER sockets
for outbound connections. Message-content agnostic.
"""
def __init__(self, local_id: str, host: str, port: int) -> None:
self._local_id = local_id
self._closed = False
self._connections: dict[str, ZmqConnection] = {}
self._pending_inbound: list[tuple[str, dict]] = []
self._zmq_ctx = zmq.Context()
self._router: zmq.Socket = self._zmq_ctx.socket(zmq.ROUTER)
_apply_heartbeat(self._router)
bind_addr = _tcp_addr(host, port)
self._router.bind(bind_addr)
logger.info("ZmqTransport %s: ROUTER bound on %s", self._local_id, bind_addr)
# ------------------------------------------------------------------
# ZmqConnection lifecycle
# ------------------------------------------------------------------
def connect(self, peer_id: str) -> ZmqConnection:
"""Open an outbound connection to a remote peer."""
assert peer_id not in self._connections, (
f"ZmqConnection to {peer_id} already exists"
)
logger.info(
"ZmqTransport %s: opening OUTBOUND connection to %s",
self._local_id,
peer_id,
)
return self._open_connection(peer_id, direction="outbound")
def poll(self) -> Sequence[ControlConnection]:
"""Process all pending I/O. Returns newly accepted connections.
- Receives messages (buffered in each connection's inbox)
- Creates connections for new inbound peers (connect msg in inbox)
- Checks monitors for disconnections
- Removes and closes dead connections
"""
self._recv_router()
self._check_monitors()
# Create connections for new inbound peers
new_connections: list[ControlConnection] | None = None
for sender_id, msg in self._pending_inbound:
conn = self._connections.get(sender_id)
if conn is None:
logger.info(
"ZmqTransport %s: accepting INBOUND connection from %s",
self._local_id,
sender_id,
)
conn = self._open_connection(sender_id, direction="inbound")
if new_connections is None:
new_connections = []
new_connections.append(conn)
conn.enqueue(msg)
self._pending_inbound.clear()
# Remove dead connections
for pid in [p for p, c in self._connections.items() if not c.alive]:
self._connections.pop(pid).close()
return (
new_connections if new_connections is not None else _EMPTY_NEW_CONNECTIONS
)
def close(self) -> None:
if self._closed:
return
self._closed = True
for conn in self._connections.values():
conn.close()
self._connections.clear()
self._router.setsockopt(zmq.LINGER, 0)
self._router.close()
self._zmq_ctx.destroy(linger=0)
# ------------------------------------------------------------------
# Internal
# ------------------------------------------------------------------
def _open_connection(
self, peer_id: str, direction: str = "outbound"
) -> ZmqConnection:
"""Create a DEALER socket + monitor and register the connection."""
host, port_str = peer_id.rsplit(":", 1)
dealer_addr = _tcp_addr(host, port_str)
logger.debug(
"ZmqTransport %s: creating DEALER for %s peer %s -> %s",
self._local_id,
direction,
peer_id,
dealer_addr,
)
dealer = self._zmq_ctx.socket(zmq.DEALER)
_apply_heartbeat(dealer)
dealer.identity = self._local_id.encode()
safe_id = peer_id.replace(":", "-").replace("/", "-")
monitor_addr = f"inproc://p2p-monitor-{safe_id}"
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
monitor_sock = self._zmq_ctx.socket(zmq.PAIR)
monitor_sock.connect(monitor_addr)
dealer.connect(dealer_addr)
sockets = _Sockets(dealer=dealer, monitor=monitor_sock)
conn = ZmqConnection(peer_id, sockets)
self._connections[peer_id] = conn
logger.info(
"ZmqTransport %s: %s connection established to %s (active connections: %d)",
self._local_id,
direction,
peer_id,
len(self._connections),
)
return conn
def _recv_router(self) -> None:
"""Non-blocking: receive all pending messages from ROUTER."""
while True:
try:
frames = self._router.recv_multipart(zmq.NOBLOCK)
except zmq.Again:
break
except zmq.ZMQError as exc:
logger.warning("ZmqTransport %s: recv error: %s", self._local_id, exc)
break
if len(frames) != 2:
logger.warning(
"ZmqTransport %s: dropping message with %d frames (expected 2)",
self._local_id,
len(frames),
)
continue
identity, data = frames
sender_id = identity.decode()
logger.debug(
"ZmqTransport %s: ROUTER recv from %s (%d bytes)",
self._local_id,
sender_id,
len(data),
)
try:
msg = msgspec.msgpack.decode(data)
except Exception as exc:
logger.warning(
"ZmqTransport %s: failed to decode message from %s: %s",
self._local_id,
sender_id,
exc,
)
continue
conn = self._connections.get(sender_id)
if conn is not None:
conn.enqueue(msg)
else:
self._pending_inbound.append((sender_id, msg))
def _check_monitors(self) -> None:
"""Non-blocking: check all monitor sockets for disconnection."""
for conn in self._connections.values():
if not conn.alive:
continue
try:
event = zmq.utils.monitor.recv_monitor_message(
conn.monitor_socket, zmq.NOBLOCK
)
except zmq.Again:
continue
except zmq.ZMQError as exc:
logger.warning(
"ZmqTransport %s: monitor error for peer %s: %s",
self._local_id,
conn.peer_id,
exc,
)
continue
if event["event"] == zmq.EVENT_DISCONNECTED:
logger.debug(
"ZmqTransport %s: peer %s disconnected",
self._local_id,
conn.peer_id,
)
conn.mark_dead()
@@ -0,0 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.kv_offload.tiering.p2p.data.base import DataTransport, PollResult
from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport
__all__ = [
"DataTransport",
"NixlTransport",
"PollResult",
]
+259
View File
@@ -0,0 +1,259 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract base class for the P2P data-plane transport.
The data plane handles RDMA (or similar) block transfers between peers.
It is independent of the control plane — the control plane establishes
who can talk to whom, then the data plane moves blocks at wire speed.
Architecture
------------
DataTransport (one per node)
├── owns the local KV block memory region
├── registers remote peers (add_remote_peer / remove_remote_peer)
├── submits block writes (write_blocks → transfer_id)
├── polls for completion (poll → done/failed IDs)
└── cancels inflight transfers (cancel)
Memory model
------------
The local node exposes a contiguous block region:
base_addr ──► ┌─────────────┐ block 0
├─────────────┤ block 1
├─────────────┤ ...
└─────────────┘ block (num_blocks - 1)
Each block is block_len bytes.
Remote peers expose the same layout. write_blocks() copies local
blocks to a remote peer's block region by index:
write_blocks("peer:1", local_idxs=[0, 3], remote_idxs=[5, 7])
→ writes local block 0 → remote block 5
local block 3 → remote block 7
Transfer lifecycle
------------------
1. Register remote peer: add_remote_peer(peer_id, metadata, ...)
- Provides the remote memory layout so transfers can target it
2. Submit: write_blocks(peer_id, local_idxs, remote_idxs) → int
- Returns a transfer_id (opaque int) for tracking
- Returns None if peer not registered or submission fails
3. Poll: poll() → PollResult(done=[...], failed=[...])
- Returns transfer_ids that completed or failed since last poll
- Completed transfers are automatically cleaned up
4. Cancel: cancel(transfer_ids, mode="immediate" | "wait")
- Best-effort cancellation of inflight transfers
- mode="wait" returns ids still in PROC/PEND so the caller can poll
them to completion
Implementor contracts
---------------------
- write_blocks() must not block. The transfer runs asynchronously.
- poll() must be called periodically. It drives completion checking.
- transfer_ids are unique across the lifetime of the transport.
- add_remote_peer() must be called before write_blocks() to that peer.
- get_agent_metadata() returns opaque bytes that the remote peer
needs to call add_remote_peer() (e.g., RDMA connection info).
- config_fingerprint is a content hash of the model configuration.
Peers with different fingerprints are incompatible and must not
exchange blocks (validated during the control-plane handshake).
- close() releases all resources (memory registrations, handles).
After close(), no other methods may be called.
Threading model: no background threads. All I/O driven by poll().
"""
from __future__ import annotations
import ctypes
import hashlib
import json
from abc import ABC, abstractmethod
from collections.abc import Iterable, Sequence
from typing import Literal, NamedTuple
CancelMode = Literal["immediate", "wait"]
class PollResult(NamedTuple):
"""Result of polling inflight transfers.
Attributes:
done: Transfer IDs that completed successfully.
failed: Transfer IDs that failed (error, timeout, etc.).
"""
done: Sequence[int]
failed: Sequence[int]
class DataTransport(ABC):
"""Abstract data-plane transport for RDMA-style block transfers.
Owns the local KV block memory region and manages transfers to/from
registered remote peers.
Construction:
view: A 2D memoryview (num_blocks × block_len bytes) over the
local KV cache block storage.
config_fields: Dict of model config values used to compute the
compatibility fingerprint. None → empty fingerprint
(compatible with any peer).
"""
def __init__(self, view: memoryview, config_fields: dict | None = None) -> None:
assert view.shape is not None
self._view = view
self._base_addr = ctypes.addressof(ctypes.c_char.from_buffer(view))
self._num_blocks = view.shape[0]
self._block_len = view.shape[1]
self._config_fingerprint = self._compute_fingerprint(config_fields)
@property
def base_addr(self) -> int:
"""Base address of the local block memory region."""
return self._base_addr
@property
def num_blocks(self) -> int:
"""Number of blocks in the local region."""
return self._num_blocks
@property
def block_len(self) -> int:
"""Size of each block in bytes."""
return self._block_len
@property
def config_fingerprint(self) -> str:
"""Content-hash of the model configuration (hex string).
Peers must have matching fingerprints to exchange blocks.
Empty string means no fingerprint (always compatible).
"""
return self._config_fingerprint
@staticmethod
def _compute_fingerprint(config_fields: dict | None) -> str:
if not config_fields:
return ""
canonical = json.dumps(config_fields, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode()).hexdigest()[:16]
@abstractmethod
def get_agent_metadata(self) -> bytes:
"""Return opaque metadata needed by remote peers to connect.
The returned bytes are sent during the control-plane handshake
and passed to the remote peer's add_remote_peer().
"""
...
@abstractmethod
def add_remote_peer(
self,
peer_id: str,
agent_metadata: bytes,
base_addr: int,
num_blocks: int,
block_len: int,
) -> None:
"""Register a remote peer for block transfers.
Must be called before write_blocks() to this peer.
Args:
peer_id: Unique identifier for the remote peer.
agent_metadata: Opaque bytes from the peer's get_agent_metadata().
base_addr: Base address of the peer's block memory region.
num_blocks: Number of blocks in the peer's region.
block_len: Size of each block (must match local block_len).
"""
...
@abstractmethod
def remove_remote_peer(self, peer_id: str) -> None:
"""Unregister a remote peer and release associated resources.
Inflight transfers to this peer should be cancelled first.
"""
...
@abstractmethod
def write_blocks(
self,
peer_id: str,
local_idxs: list[int],
remote_idxs: list[int],
) -> int | None:
"""Submit a WRITE transfer: local blocks → remote peer's blocks.
Args:
peer_id: Target peer (must be registered via add_remote_peer).
local_idxs: Indexes of local blocks to read from.
remote_idxs: Indexes of remote blocks to write to.
Must be same length as local_idxs.
Returns:
A unique transfer_id (int) to track this transfer, or
None if the peer is not registered or submission failed.
"""
...
@abstractmethod
def poll(self) -> PollResult:
"""Poll all inflight transfers for completion.
Returns:
PollResult with lists of completed and failed transfer_ids.
Completed/failed transfers are removed from the inflight set.
Must be called periodically to drive progress checking.
"""
...
@abstractmethod
def cancel(
self,
transfer_ids: Iterable[int],
mode: CancelMode = "immediate",
) -> list[int]:
"""Cancel inflight transfers by their IDs.
Best-effort: transfers that already completed are ignored.
Args:
transfer_ids: IDs to cancel. Unknown IDs are ignored.
mode:
"immediate" (default): pop and release each handle and
return []. Matches the legacy fire-and-forget
behavior — the caller does not wait for the
underlying transfer to drain.
"wait": attempt to release each handle. If the release
cannot complete because the transfer is still
PROC/PEND, the entry stays in the inflight set and
its id is included in the returned list. The
caller is expected to keep calling poll() until
every returned id surfaces in done/failed.
Returns:
For mode="wait", the subset of *transfer_ids* still
tracked as inflight after the cancel attempt. For
mode="immediate", always [].
"""
...
@abstractmethod
def close(self) -> None:
"""Release all resources (registrations, handles, memory).
Cancels any remaining inflight transfers. Idempotent.
After close(), no other methods may be called.
"""
...
+297
View File
@@ -0,0 +1,297 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL.
"""
from __future__ import annotations
import itertools
from collections.abc import Iterable
from typing import Any
from vllm.distributed.nixl_utils import NixlWrapper as _NixlAgent
from vllm.distributed.nixl_utils import nixl_agent_config as _NixlAgentConfig
from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.p2p.data.base import (
CancelMode,
DataTransport,
PollResult,
)
logger = init_logger(__name__)
# Shared sentinel returned by poll() in the steady state (no inflight, or
# no transfer changed state since the last poll). Tuples make it immutable;
# callers only iterate / membership-test / equality-check.
_EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=())
class NixlTransport(DataTransport):
"""Manages a NIXL agent, memory registration, and block transfers.
Wraps the NIXL C library behind a Python interface so the rest of the
P2P tier code never touches NIXL types directly. Tracks inflight
handles internally and returns completed/failed tags on poll.
"""
def __init__(
self,
local_id: str,
view: memoryview,
config_fields: dict | None = None,
backends: list[str] | None = None,
num_threads: int = 4,
) -> None:
super().__init__(view, config_fields=config_fields)
self._local_id = local_id
self._backends = list(backends) if backends else ["UCX"]
self._num_threads = num_threads
self._agent: Any = None
self._reg: Any = None
self._local_dlist: Any = None
self._remote_dlists: dict[str, object] = {}
self._peer_nixl_names: dict[str, str] = {}
self._inflight: dict[int, object] = {} # transfer_id → handle
self._next_id = itertools.count()
self._init(view)
@property
def available(self) -> bool:
return self._agent is not None
def _init(self, view: memoryview) -> None:
if _NixlAgent is None:
return
non_ucx_backends = [b for b in self._backends if b != "UCX"]
if non_ucx_backends:
cfg = _NixlAgentConfig(backends=self._backends, capture_telemetry=True)
logger.info(
"NixlTransport %s: NIXL backends=%s",
self._local_id,
self._backends,
)
else:
cfg = _NixlAgentConfig(
num_threads=self._num_threads, capture_telemetry=True
)
logger.info(
"NixlTransport %s: NIXL backends=[UCX] num_threads=%d",
self._local_id,
self._num_threads,
)
self._agent = _NixlAgent(self._local_id, cfg)
total_size = self._num_blocks * self._block_len
reg_descs = [(self._base_addr, total_size, 0, "")]
self._reg = self._agent.register_memory(reg_descs, mem_type="DRAM")
block_tuples = [
(self._base_addr + i * self._block_len, self._block_len, 0)
for i in range(self._num_blocks)
]
xfer_dlist = self._agent.get_xfer_descs(block_tuples, mem_type="DRAM")
self._local_dlist = self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", xfer_dlist)
logger.info(
"NixlTransport %s: registered %d blocks", self._local_id, self._num_blocks
)
def get_agent_metadata(self) -> bytes:
assert self._agent is not None
return self._agent.get_agent_metadata()
# ------------------------------------------------------------------
# Peer management
# ------------------------------------------------------------------
def add_remote_peer(
self,
peer_id: str,
agent_metadata: bytes,
base_addr: int,
num_blocks: int,
block_len: int,
) -> None:
nixl_name = self._agent.add_remote_agent(agent_metadata)
block_descs = [
(base_addr + i * block_len, block_len, 0) for i in range(num_blocks)
]
xfer_dlist = self._agent.get_xfer_descs(block_descs, mem_type="DRAM")
remote_dlist = self._agent.prep_xfer_dlist(nixl_name, xfer_dlist)
self._peer_nixl_names[peer_id] = nixl_name
self._remote_dlists[peer_id] = remote_dlist
def remove_remote_peer(self, peer_id: str) -> None:
nixl_name = self._peer_nixl_names.pop(peer_id, None)
dlist = self._remote_dlists.pop(peer_id, None)
if self._agent is not None:
if dlist is not None:
self._agent.release_dlist_handle(dlist)
if nixl_name:
self._agent.remove_remote_agent(nixl_name)
# ------------------------------------------------------------------
# Transfer submission and polling
# ------------------------------------------------------------------
def write_blocks(
self,
peer_id: str,
local_idxs: list[int],
remote_idxs: list[int],
) -> int | None:
"""Submit a WRITE transfer to *peer_id*.
Returns a transfer ID, or None if the peer is not registered.
The ID is returned via poll() when the transfer completes or fails.
"""
remote_dlist = self._remote_dlists.get(peer_id)
if remote_dlist is None:
logger.warning(
"NixlTransport %s: write_blocks NO REMOTE DLIST for peer=%s "
"(known peers=%s)",
self._local_id,
peer_id,
list(self._remote_dlists.keys()),
)
return None
logger.debug(
"NixlTransport %s: write_blocks NIXL.transfer peer=%s blocks=%d",
self._local_id,
peer_id,
len(local_idxs),
)
handle = self._agent.make_prepped_xfer(
"WRITE",
self._local_dlist,
local_idxs,
remote_dlist,
remote_idxs,
)
self._agent.transfer(handle)
transfer_id = next(self._next_id)
self._inflight[transfer_id] = handle
return transfer_id
def poll(self) -> PollResult:
"""Poll all inflight transfers.
Returns PollResult(done=..., failed=...) with transfer IDs.
Completed handles are released automatically.
"""
if not self._inflight:
return _EMPTY_POLL_RESULT
done_ids: list[int] | None = None
failed_ids: list[int] | None = None
for transfer_id, handle in self._inflight.items():
try:
state = self._agent.check_xfer_state(handle)
except Exception as exc:
logger.warning(
"NixlTransport %s: check_xfer_state failed for transfer_id=%d: %s",
self._local_id,
transfer_id,
exc,
)
continue
if state == "DONE":
if done_ids is None:
done_ids = []
done_ids.append(transfer_id)
elif state not in ("PROC", "PEND"):
if failed_ids is None:
failed_ids = []
failed_ids.append(transfer_id)
if done_ids is None and failed_ids is None:
return _EMPTY_POLL_RESULT
handles_to_release = []
for tid in done_ids or ():
handles_to_release.append(self._inflight.pop(tid))
for tid in failed_ids or ():
handles_to_release.append(self._inflight.pop(tid))
self._release_handles(handles_to_release)
return PollResult(
done=done_ids if done_ids is not None else _EMPTY_POLL_RESULT.done,
failed=failed_ids if failed_ids is not None else _EMPTY_POLL_RESULT.failed,
)
def cancel(
self,
transfer_ids: Iterable[int],
mode: CancelMode = "immediate",
) -> list[int]:
"""Cancel inflight transfers by their IDs.
See ``DataTransport.cancel`` for the contract. In "wait" mode,
transfers whose ``release_xfer_handle`` raises (NIXL could not
complete the abort because the backend is still draining) stay
in ``self._inflight`` so a later ``poll()`` will observe them.
"""
if mode == "immediate":
handles = [
self._inflight.pop(tid) for tid in transfer_ids if tid in self._inflight
]
self._release_handles(handles)
return []
still_inflight: list[int] = []
for tid in transfer_ids:
handle = self._inflight.get(tid)
if handle is None:
continue
try:
self._agent.release_xfer_handle(handle)
except Exception as exc:
logger.debug(
"NixlTransport %s: cancel pending for transfer_id=%d: %s",
self._local_id,
tid,
exc,
)
still_inflight.append(tid)
continue
del self._inflight[tid]
return still_inflight
# ------------------------------------------------------------------
# Lifecycle
# ------------------------------------------------------------------
def close(self) -> None:
if self._agent is None:
return
self._release_handles(list(self._inflight.values()))
self._inflight.clear()
for peer_id in list(self._remote_dlists):
self.remove_remote_peer(peer_id)
if self._local_dlist is not None:
self._agent.release_dlist_handle(self._local_dlist)
self._local_dlist = None
if self._reg is not None:
self._agent.deregister_memory(self._reg)
self._reg = None
self._agent = None
# ------------------------------------------------------------------
# Internal
# ------------------------------------------------------------------
def _release_handles(self, handles: list[object]) -> None:
if self._agent is None:
return
for handle in handles:
try:
self._agent.release_xfer_handle(handle)
except Exception as exc:
logger.warning(
"NixlTransport %s: release_xfer_handle failed: %s",
self._local_id,
exc,
)
+664
View File
@@ -0,0 +1,664 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
P2PSecondaryTierManager: Secondary tier for P2P KV cache sharing.
Owns transports and a single bidirectional P2PSession per remote peer.
"""
from __future__ import annotations
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
from typing_extensions import override
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
ReqContext,
RequestOffloadingContext,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport
from vllm.v1.kv_offload.tiering.p2p.session import P2PSession
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
logger = init_logger(__name__)
# Reap unbound store batches that have been parked without a FetchMsg
# binding them to a session for longer than this. Protects against the
# prefiller buffering blocks for a decoder that never asks (decoder died,
# network partition, lost kv_request_id). Must be longer than the per-store
# deadline so the store-timeout path fires first for individual jobs.
_UNBOUND_STORE_TIMEOUT_S = 60.0
# Time we wait during shutdown for inflight transfers to drain via
# cancel(mode="wait") before falling back to mode="immediate". Bounded
# so a wedged peer can't hang shutdown.
_SHUTDOWN_DRAIN_TIMEOUT_S = 3.0
# Sleep between iterations of the bounded drain loops in drain_jobs() and
# _drain_inflight_for_shutdown(). Short enough to keep latency low, long
# enough to avoid busy-spinning the scheduler thread.
_DRAIN_SLEEP_S = 0.001
def _prefill_params(kv_params: dict | None) -> dict | None:
"""Return the ``prefill`` sub-dict, or None if absent.
Set on decoder requests; carries kv_request_id, remote_host, remote_port.
"""
if not kv_params:
return None
return kv_params.get("prefill")
def _decode_params(kv_params: dict | None) -> dict | None:
"""Return the ``decode`` sub-dict, or None if absent.
Set on prefiller requests; carries kv_request_id.
"""
if not kv_params:
return None
return kv_params.get("decode")
@dataclass
class _UnboundStoreBatch:
"""A submit_store batch parked at the manager before any peer has fetched.
Indexed by kv_request_id only — the prefiller no longer learns the peer
identity at store time. When a FetchMsg(kv_request_id) arrives on some
session, the manager binds the kv_request_id to that session and replays
every parked batch into ServerRole via session.add_stored_blocks.
"""
job_id: int
keys: list[OffloadKey]
block_ids: Sequence[int]
submitted_at: float = field(default_factory=time.monotonic)
class P2PSecondaryTierManager(SecondaryTierManager):
"""Secondary tier for P2P KV cache sharing.
A single P2PSession per remote peer handles both client-role (loading
blocks from the peer) and server-role (serving blocks to the peer)
over the same control connection.
Single-threaded: every public method runs on the scheduler thread, and
the engine drives polling via ``get_finished_jobs()`` once per step.
``has_pending_work()`` keeps the engine ticking so the control transport
and existing sessions are polled even when no requests are scheduled.
"""
def __init__(
self,
offloading_spec: OffloadingSpec,
primary_kv_view: memoryview,
tier_type: str = "p2p",
host: str = "0.0.0.0",
port: int = 7777,
backends: list[str] | None = None,
num_threads: int = 4,
**kwargs,
) -> None:
"""Initialize the P2P secondary tier manager.
All keyword arguments after ``primary_kv_view`` come from the
``secondary_tiers`` entry in ``kv_connector_extra_config``. See
``docs/features/kv_offloading_usage.md`` for the user-facing
configuration reference.
Args:
offloading_spec: Owning ``OffloadingSpec`` (provides
``vllm_config`` and the offloaded block layout).
primary_kv_view: Memoryview over the CPU primary tier; the
NIXL agent registers this region for RDMA transfers.
tier_type: Tier identifier (defaults to ``"p2p"``).
host: Address the ZMQ control socket binds to.
port: Port for the ZMQ control socket. Must be reachable
from peers.
backends: NIXL transport backends (e.g. ``["UCX"]``,
``["MOONCAKE"]``, ``["LIBFABRIC"]``). Defaults to
``["UCX"]``. When any non-UCX backend is requested, the
NIXL agent is initialized with ``backends=...``;
otherwise it falls back to a UCX-only agent with
``num_threads`` threads.
num_threads: NIXL agent worker threads for the UCX-only
branch. Ignored when ``backends`` contains a non-UCX
entry.
**kwargs: Reserved for future tier-specific options.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
port = int(port)
self._local_id = f"{host}:{port}"
config_fields = FileMapper.from_offloading_spec(
root_dir="",
offloading_spec=offloading_spec,
gpu_blocks_per_file=offloading_spec.block_size_factor,
parallel_agnostic=True,
).get_run_config()
self._data: DataTransport = NixlTransport(
self._local_id,
primary_kv_view,
config_fields=config_fields,
backends=backends,
num_threads=int(num_threads),
)
self._control: ControlTransport = ZmqTransport(self._local_id, host, port)
self._sessions: dict[str, P2PSession] = {}
# kv_request_id → session, set when the bound session has received
# FetchMsg for that id. submit_store after binding routes directly
# to the session; before binding, batches are parked in
# _unbound_stores below. Stays in sync with _sessions: entries
# pointing to a reaped session are purged in _reap_dead_sessions.
self._kv_to_session: dict[str, P2PSession] = {}
# kv_request_id → list of batches submit_store'd before any peer
# asked for that id. Drained into a session by _on_session_fetch
# when the corresponding FetchMsg arrives, or surfaced as failures
# by _reap_unbound_stores after _UNBOUND_STORE_TIMEOUT_S.
self._unbound_stores: dict[str, list[_UnboundStoreBatch]] = {}
self._finished_jobs: list[JobResult] = []
# kv_request_ids that hit a transport/session failure; On load lookup()
# rejects them so the request falls back to local prefill.
self._failed_req_ids: set[str] = set()
# ------------------------------------------------------------------
# SecondaryTierManager interface
# ------------------------------------------------------------------
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
prefill = _prefill_params(req_context.kv_transfer_params)
if (
not prefill
or not prefill.get("remote_host")
or not prefill.get("remote_port")
or not prefill.get("kv_request_id")
):
return LookupResult.MISS
kv_request_id = prefill["kv_request_id"]
if kv_request_id in self._failed_req_ids:
return LookupResult.MISS
return LookupResult.HIT
@override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
"""Open the outbound session toward the producer if needed.
On the decoder side (``prefill`` set), open a session toward the
producer at remote_host:remote_port so submit_load can issue
FetchMsg as soon as it fires. On the prefiller side, sessions
are created when the consumer's inbound connection arrives in
_accept_new_peers — submit_store no longer pre-creates anything.
"""
prefill = _prefill_params(req_context.kv_transfer_params)
if prefill:
peer_id = self._remote_id_from_params(prefill)
if peer_id:
self._get_or_create_session(peer_id)
return RequestOffloadingContext()
@override
def on_request_finished(self, req_context: ReqContext) -> None:
"""Cancels pending loads and prunes session-scoped state.
Decoder side (``prefill`` set): looks up the session by peer_id
because the producer's address is what addresses the client-role
load to cancel. Prefiller side (``decode`` set): looks up via
kv_request_id because peer_id is no longer carried on store-time
kv_transfer_params; if a session has bound the id, finish it. If
no session has bound the id yet, this is a no-op: parked batches
in `_unbound_stores` are left in place and cleaned up only by
`_reap_unbound_stores` after `_UNBOUND_STORE_TIMEOUT_S`.
"""
kv_params = req_context.kv_transfer_params
if not kv_params:
return
prefill = _prefill_params(kv_params)
decode = _decode_params(kv_params)
kv_request_id = (prefill or decode or {}).get("kv_request_id")
if not kv_request_id:
return
self._failed_req_ids.discard(kv_request_id)
if prefill:
peer_id = self._remote_id_from_params(prefill)
if peer_id:
session = self._sessions.get(peer_id)
if session is not None:
session.finish_request(kv_request_id)
return
# Prefiller-side finish: identify the session via kv_request_id.
session = self._kv_to_session.pop(kv_request_id, None)
if session is not None:
session.finish_request(kv_request_id)
return
@override
def submit_store(self, job_metadata: JobMetadata) -> None:
job_id = job_metadata.job_id
keys = list(job_metadata.keys)
block_ids = job_metadata.block_ids
assert len(keys) == len(block_ids)
kv_params = job_metadata.req_context.kv_transfer_params
decode = _decode_params(kv_params)
logger.debug(
"P2P %s: submit_store ENTRY job_id=%d blocks=%d decode=%s kv_request_id=%s",
self._local_id,
job_id,
len(block_ids),
decode is not None,
(decode or {}).get("kv_request_id"),
)
# Absent ``decode`` block => not a remote-decode request: succeed
# locally without parking. An empty/malformed dict is still a
# remote-decode signal and must fail the missing-id check below.
if decode is None:
self._finished_jobs.append(JobResult(job_id=job_id, success=True))
return
kv_request_id = decode.get("kv_request_id")
if not kv_request_id:
logger.warning(
"P2P %s: submit_store missing kv_request_id",
self._local_id,
)
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
return
# Fast path: a session has already received FetchMsg for this id,
# so we can route the batch straight into its ServerRole.
session = self._kv_to_session.get(kv_request_id)
if session is not None:
session.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
return
# No session bound yet — park the batch keyed by kv_request_id.
# _on_session_fetch drains it on the first FetchMsg; if no peer
# ever asks, _reap_unbound_stores surfaces the job as failed.
self._unbound_stores.setdefault(kv_request_id, []).append(
_UnboundStoreBatch(
job_id=job_id,
keys=keys,
block_ids=block_ids,
)
)
logger.debug(
"P2P %s: parked submit_store kv_request_id=%s job_id=%d blocks=%d",
self._local_id,
kv_request_id,
job_id,
len(block_ids),
)
@override
def submit_load(self, job_metadata: JobMetadata) -> None:
job_id = job_metadata.job_id
keys = list(job_metadata.keys)
block_ids = job_metadata.block_ids
prefill = _prefill_params(job_metadata.req_context.kv_transfer_params)
logger.debug(
"P2P %s: submit_load ENTRY job_id=%d blocks=%d kv_request_id=%s peer=%s",
self._local_id,
job_id,
len(block_ids),
(prefill or {}).get("kv_request_id"),
self._remote_id_from_params(prefill or {}),
)
if (
not prefill
or not prefill.get("remote_host")
or not prefill.get("remote_port")
or not prefill.get("kv_request_id")
):
logger.debug(
"P2P %s: submit_load job_id=%d FAILED missing prefill params",
self._local_id,
job_id,
)
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
return
kv_request_id = prefill["kv_request_id"]
peer_id = self._remote_id_from_params(prefill)
assert peer_id is not None # guaranteed by prefill checks above
if not keys:
logger.debug(
"P2P %s: submit_load job_id=%d short-circuit success (no keys)",
self._local_id,
job_id,
)
self._finished_jobs.append(JobResult(job_id=job_id, success=True))
return
session = self._sessions.get(peer_id)
if session is None:
logger.warning(
"P2P %s: submit_load job_id=%d NO SESSION for peer=%s",
self._local_id,
job_id,
peer_id,
)
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
self._failed_req_ids.add(kv_request_id)
return
logger.debug(
"P2P %s: submit_load job_id=%d -> request_blocks peer=%s "
"kv_request_id=%s blocks=%d session_ready=%s",
self._local_id,
job_id,
peer_id,
kv_request_id,
len(block_ids),
session.ready,
)
session.request_blocks(job_id, kv_request_id, keys, block_ids)
@override
def get_finished_jobs(self) -> Iterable[JobResult]:
# Drive one polling sweep on the scheduler thread, then hand off
# whatever has accumulated. The engine calls this once per step
# (and keeps stepping while has_pending_work() is True).
self._poll_once()
result = self._finished_jobs
self._finished_jobs = []
return result
@override
def has_pending_work(self) -> bool:
# The engine tick is the only driver of _control.poll() and
# session.poll(); without it we miss new peer connects and
# inbound fetch messages on existing sessions. Keep the engine
# ticking for the lifetime of this manager.
return True
@override
def drain_jobs(self) -> None:
"""Block until every submitted load/store job has completed or failed.
Loops calling ``_poll_once()`` until no session has outstanding
inbound loads or in-flight outbound stores. Mid-flight transfers
are NOT cancelled — the caller (``TieringOffloadingManager.reset_cache``)
needs the primary memoryview to be quiescent, not aborted. Results
accumulate in ``_finished_jobs`` and are surfaced by the next
``get_finished_jobs()`` call.
"""
start = time.monotonic()
warned = False
while True:
self._poll_once()
pending = any(
s._client._inbound or s._server._inflight
for s in self._sessions.values()
)
if not pending:
return
if not warned and time.monotonic() - start > 5.0:
logger.warning(
"P2PSecondaryTierManager.drain_jobs: still draining "
"after 5s; a stuck transfer will block the engine.",
)
warned = True
time.sleep(_DRAIN_SLEEP_S)
@override
def on_schedule_end(self) -> None:
return
# ------------------------------------------------------------------
# Internal
# ------------------------------------------------------------------
@staticmethod
def _remote_id_from_params(role_params: dict) -> str | None:
"""Build peer_id from a role-scoped sub-dict (``prefill``/``p2p``)."""
host = role_params.get("remote_host")
port = role_params.get("remote_port")
if host and port:
return f"{host}:{port}"
return None
def _get_or_create_session(self, peer_id: str) -> P2PSession:
"""Return the existing session for peer_id, or open one outbound.
Decoder-side helper for on_new_request: when ``prefill`` is set,
the consumer must reach the producer at peer_id. If we already
have a session toward that peer (from a prior load or a
peer-initiated inbound), reuse it; otherwise open an outbound
ControlConnection and build a connected session.
"""
session = self._sessions.get(peer_id)
if session is not None:
return session
conn = self._control.connect(peer_id)
session = P2PSession(
peer_id=peer_id,
local_id=self._local_id,
transport=self._data,
local_block_len=self._data.block_len,
conn=conn,
)
self._sessions[peer_id] = session
return session
def _accept_new_peers(self, new_connections: Sequence[ControlConnection]) -> None:
for conn in new_connections:
logger.info(
"P2P %s: accepting incoming connection from %s",
self._local_id,
conn.peer_id,
)
try:
existing = self._sessions.get(conn.peer_id)
if existing is not None:
raise ValueError(f"duplicate connection from {conn.peer_id}")
self._sessions[conn.peer_id] = P2PSession(
peer_id=conn.peer_id,
local_id=self._local_id,
transport=self._data,
local_block_len=self._data.block_len,
conn=conn,
)
logger.info(
"P2P %s: created connected session for %s",
self._local_id,
conn.peer_id,
)
except (ValueError, KeyError, TypeError, AssertionError) as exc:
logger.error("P2P %s: rejecting peer: %s", self._local_id, exc)
conn.close()
def _reap_dead_sessions(self) -> None:
# Reap connected sessions whose connection died — peer is gone.
# Stranded prefiller-side stores are no longer tracked through a
# session (they live in _unbound_stores keyed by kv_request_id);
# _reap_unbound_stores handles their timeout independently.
dead: list[str] | None = None
for pid, s in self._sessions.items():
if s.connected and not s.alive:
if dead is None:
dead = []
dead.append(pid)
if dead is None:
return
for pid in dead:
session = self._sessions.pop(pid)
# Purge any kv_request_id → session entries pointing at this
# session so subsequent submit_stores fall back to the unbound
# path (which will time out into failure if no peer rebinds).
stale_kv_ids = [
kid for kid, s in self._kv_to_session.items() if s is session
]
for kid in stale_kv_ids:
del self._kv_to_session[kid]
failed_loads, failed_stores = session.close()
for job_id, kv_request_id in failed_loads:
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
self._failed_req_ids.add(kv_request_id)
for job_id in failed_stores:
self._finished_jobs.append(JobResult(job_id=job_id, success=False))
self._data.remove_remote_peer(pid)
logger.warning("P2P %s: peer %s down", self._local_id, pid)
def _reap_unbound_stores(self) -> None:
"""Time out submit_store batches that no peer has ever fetched.
Walks `_unbound_stores` for entries whose oldest batch is older
than `_UNBOUND_STORE_TIMEOUT_S`. Drops the kv_request_id, surfaces
every batched job as failed, and adds the id to `_failed_req_ids`
so a late inbound FetchMsg short-circuits to a clean rejection.
"""
if not self._unbound_stores:
return
deadline = time.monotonic() - _UNBOUND_STORE_TIMEOUT_S
expired: list[str] | None = None
for kid, batches in self._unbound_stores.items():
# Batches are appended in arrival order, so the head is oldest.
if batches and batches[0].submitted_at <= deadline:
if expired is None:
expired = []
expired.append(kid)
if expired is None:
return
for kid in expired:
batches = self._unbound_stores.pop(kid)
self._failed_req_ids.add(kid)
for batch in batches:
self._finished_jobs.append(
JobResult(job_id=batch.job_id, success=False)
)
logger.warning(
"P2P %s: unbound store kv_request_id=%s timed out after %.0fs "
"without a fetch — failing %d job(s)",
self._local_id,
kid,
_UNBOUND_STORE_TIMEOUT_S,
len(batches),
)
# ------------------------------------------------------------------
# Polling
# ------------------------------------------------------------------
def _poll_once(self) -> None:
"""One sweep of the polling work.
Drains the control transport, polls every session, accumulates
their results into ``_finished_jobs``, and reaps any dead sessions.
Runs on the scheduler thread.
"""
new_connections = self._control.poll()
if new_connections:
logger.info(
"P2P %s: _poll_once got %d new connection(s): %s",
self._local_id,
len(new_connections),
[c.peer_id for c in new_connections],
)
self._accept_new_peers(new_connections)
for session in self._sessions.values():
result = session.poll()
for lr in result.loads:
self._finished_jobs.append(
JobResult(job_id=lr.job_id, success=lr.success)
)
if not lr.success:
self._failed_req_ids.add(lr.kv_request_id)
for sr in result.stores:
self._finished_jobs.append(
JobResult(job_id=sr.job_id, success=sr.success)
)
# Bind kv_request_id → session for any FetchMsg this tick and
# replay any submit_store batches parked while no peer was
# asking. ServerRole.on_fetch already recorded the demand
# inline in dispatch, so the replayed add_stored_blocks calls
# match that demand and submit transfers immediately.
for kv_request_id in result.new_fetch_ids:
self._kv_to_session[kv_request_id] = session
for batch in self._unbound_stores.pop(kv_request_id, ()):
session.add_stored_blocks(
kv_request_id, batch.keys, batch.block_ids, batch.job_id
)
self._reap_dead_sessions()
self._reap_unbound_stores()
# ------------------------------------------------------------------
# Lifecycle
# ------------------------------------------------------------------
@override
def shutdown(self) -> None:
self._drain_inflight_for_shutdown()
for session in self._sessions.values():
session.close()
self._sessions.clear()
self._kv_to_session.clear()
# Surface buffered store jobs as failed so the engine doesn't
# leak them; the manager is going away after this call.
for batches in self._unbound_stores.values():
for batch in batches:
self._finished_jobs.append(
JobResult(job_id=batch.job_id, success=False)
)
self._unbound_stores.clear()
self._control.close()
self._data.close()
def _drain_inflight_for_shutdown(self) -> None:
"""Best-effort drain of inflight transfers before closing _data.
Mirrors session._drain_abort but as a single bounded loop. Collects
inflight transfer_ids from each session, repeatedly calls
_data.cancel(..., mode="wait") and _data.poll() so handles can
surface as done/failed, and falls back to mode="immediate" once
_SHUTDOWN_DRAIN_TIMEOUT_S elapses so a wedged peer can't hang us.
"""
ids = [tid for s in self._sessions.values() for tid in s._server._inflight]
if not ids:
return
deadline = time.monotonic() + _SHUTDOWN_DRAIN_TIMEOUT_S
still: list[int] = ids
while still and time.monotonic() < deadline:
still = list(self._data.cancel(still, mode="wait"))
if not still:
break
# poll() advances NIXL handle state so the next wait-cancel
# has a chance to release the handles.
self._data.poll()
time.sleep(_DRAIN_SLEEP_S)
if still:
logger.warning(
"P2P %s: shutdown drain timed out after %.1fs with %d "
"transfers still inflight — force-cancelling",
self._local_id,
_SHUTDOWN_DRAIN_TIMEOUT_S,
len(still),
)
self._data.cancel(still, mode="immediate")
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.v1.kv_offload.tiering.p2p.session.session import (
LoadResult,
P2PSession,
SessionPollResult,
StoreResult,
)
__all__ = [
"LoadResult",
"P2PSession",
"SessionPollResult",
"StoreResult",
]
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Client-role state machine for a single peer session.
Handles outgoing fetch requests, abort-on-timeout, abort-ack timeout,
and produces ``LoadResult`` for completed loads. The session coordinator
parses wire messages and dispatches typed arguments here; this module
never touches ``ControlConnection`` directly — it emits via the ``send``
callback injected by the coordinator (which gates on ConnectAck).
"""
from __future__ import annotations
import time
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, NamedTuple
from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
TYPE_KEY,
AbortFetchMsg,
FetchMsg,
)
if TYPE_CHECKING:
from vllm.v1.kv_offload.tiering.base import JobId
logger = init_logger(__name__)
_LOAD_TIMEOUT_S = 30.0
_ABORT_ACK_TIMEOUT_S = 10.0
@dataclass
class _InboundRequestState:
"""Client-role state for a single load request."""
job_id: int # opaque ID assigned by the manager to this load request
kv_request_id: str
submitted_at: float
aborted_at: float | None = None
class LoadResult(NamedTuple):
"""Result from a session poll, client side."""
job_id: int
kv_request_id: str
success: bool
class ClientRole:
"""Client-side load state machine for one peer session.
The coordinator owns the connection and the send-gating; this role
is given a ``send`` callback and a ``peer_id`` for log messages and
is otherwise self-contained.
"""
def __init__(self, peer_id: str, send: Callable[[dict], None]) -> None:
self._peer_id = peer_id
self._send = send
self._inbound: dict[str, _InboundRequestState] = {}
self._completed_loads: list[LoadResult] = []
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def request_blocks(
self,
job_id: JobId,
kv_request_id: str,
keys: Sequence[bytes],
block_ids: Sequence[int],
send_ready: bool,
) -> None:
"""Register a load request and send the FetchMsg."""
logger.debug(
"P2PSession %s: request_blocks job_id=%d kv_request_id=%s "
"blocks=%d ready=%s",
self._peer_id,
job_id,
kv_request_id,
len(block_ids),
send_ready,
)
self._inbound[kv_request_id] = _InboundRequestState(
job_id=job_id,
kv_request_id=kv_request_id,
submitted_at=time.monotonic(),
)
self._send(
{
TYPE_KEY: FetchMsg.TYPE,
FetchMsg.KV_REQUEST_ID: kv_request_id,
FetchMsg.BLOCK_HASHES: list(keys),
FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids],
}
)
def cancel(self, kv_request_id: str) -> None:
"""Cancel a pending load. Sends AbortFetchMsg if still active."""
req = self._inbound.pop(kv_request_id, None)
if req is not None and req.aborted_at is None:
self._send(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: kv_request_id,
}
)
def on_transfer_done(self, kv_request_id: str, success: bool) -> None:
"""Handle a TransferDoneMsg from the peer."""
req = self._inbound.pop(kv_request_id, None)
if req is not None:
self._completed_loads.append(
LoadResult(
job_id=req.job_id,
kv_request_id=kv_request_id,
success=success,
)
)
else:
# No matching _inbound entry: either a duplicate
# transfer_done from the peer (protocol violation) or a
# benign race with a local cancel/abort/timeout that
# already popped the entry. We don't track terminated ids,
# so we can't tell — log so it's findable.
logger.warning(
"P2PSession %s: transfer_done for unknown kv_request_id=%s "
"(duplicate from peer, or raced with local cancel/timeout)",
self._peer_id,
kv_request_id,
)
def on_abort_ack(self, kv_request_id: str) -> None:
"""Handle an AbortAckMsg from the peer."""
req = self._inbound.pop(kv_request_id, None)
if req is not None:
self._completed_loads.append(
LoadResult(
job_id=req.job_id,
kv_request_id=kv_request_id,
success=False,
)
)
else:
# See on_transfer_done: same ambiguity (duplicate ack
# vs. raced with local cancel/timeout that already popped).
logger.warning(
"P2PSession %s: abort_ack for unknown kv_request_id=%s "
"(duplicate from peer, or raced with local cancel/timeout)",
self._peer_id,
kv_request_id,
)
def collect_results(self) -> list[LoadResult]:
"""Walk timeouts and drain completed loads.
Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg
sent and enter the aborting phase. Aborting requests past
``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads.
"""
now = time.monotonic()
to_remove: list[str] = []
for req_id, req in self._inbound.items():
if req.aborted_at is None:
if now - req.submitted_at >= _LOAD_TIMEOUT_S:
req.aborted_at = now
logger.warning(
"P2PSession %s: %s timed out, sending abort",
self._peer_id,
req_id,
)
self._send(
{
TYPE_KEY: AbortFetchMsg.TYPE,
AbortFetchMsg.KV_REQUEST_ID: req_id,
}
)
else:
if now - req.aborted_at >= _ABORT_ACK_TIMEOUT_S:
to_remove.append(req_id)
self._completed_loads.append(
LoadResult(
job_id=req.job_id,
kv_request_id=req_id,
success=False,
)
)
logger.warning(
"P2PSession %s: abort_ack timed out for kv_request_id=%s",
self._peer_id,
req_id,
)
for req_id in to_remove:
self._inbound.pop(req_id)
results = self._completed_loads
self._completed_loads = []
return results
def close(self) -> list[tuple[int, str]]:
"""Tear down. Returns ``(job_id, kv_request_id)`` for pending loads."""
failed = [(req.job_id, req.kv_request_id) for req in self._inbound.values()]
self._inbound.clear()
self._completed_loads.clear()
return failed
@@ -0,0 +1,247 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
P2P KV cache sharing protocol constants and documentation.
Protocol Overview
=================
Two session types communicate over a bidirectional message channel:
- P2PClientSession (requests blocks from a server)
- P2PServerSession (serves blocks to a client)
Connection Lifecycle
--------------------
1. Client opens a connection and sends ConnectMsg with its identity,
RDMA metadata, memory layout, and config fingerprint.
2. Server validates block_len and config_fingerprint, registers the
RDMA peer, and replies with ConnectAckMsg.
3. Client receives ConnectAckMsg and transitions to ready state
(flushes any queued messages).
4. Either side may send DisconnectMsg to gracefully close.
Block Transfer Flow (happy path)
---------------------------------
1. Client sends FetchMsg with a kv_request_id and lists of
block keys + remote indexes where it wants the data written.
2. Server matches requested blocks against locally stored blocks:
- Blocks already available are transferred immediately via RDMA.
- Blocks not yet available are recorded as "demanded" and
transferred when the server later stores them.
3. When all blocks for a kv_request_id are transferred, the server
sends TransferDoneMsg (success=True) to the client.
4. Client reports the load job as complete.
Abort Flow (timeout path)
--------------------------
1. If the client times out waiting for TransferDoneMsg, it sends
AbortFetchMsg to cancel the request.
2. Server cancels inflight transfers for that kv_request_id and
replies with AbortAckMsg.
3. Client receives AbortAckMsg and reports the load job as failed.
4. If AbortAckMsg itself times out, the client fails the job anyway.
Message Format
--------------
All messages are dicts serialized with msgpack. Every message has a
TYPE_KEY key identifying its type. Additional fields depend on the
message type (see per-message class docstrings below).
Security
--------
Sessions wrap all incoming message handling in try/except to guard
against malformed messages from adversarial or buggy peers. Invalid
messages are logged and dropped without crashing the session.
The config_fingerprint field in ConnectMsg ensures peers have
compatible model configurations (model, dtype, block sizes). Mismatches
are rejected during the handshake.
"""
TYPE_KEY = "type"
# ---------------------------------------------------------------------------
# Validation helpers
# ---------------------------------------------------------------------------
def _require(msg: dict, key: str, typ: type, *, name: str = "") -> None:
"""Raise ValueError if msg[key] is missing or not isinstance(typ)."""
val = msg.get(key)
if not isinstance(val, typ):
label = name or key
raise ValueError(f"{label}: expected {typ.__name__}, got {type(val).__name__}")
def _require_pos_int(msg: dict, key: str, *, name: str = "") -> None:
"""Raise ValueError if msg[key] is not a positive int."""
val = msg.get(key)
if not isinstance(val, int) or val <= 0:
label = name or key
raise ValueError(f"{label}: expected positive int, got {val!r}")
def _require_non_neg_int(msg: dict, key: str, *, name: str = "") -> None:
"""Raise ValueError if msg[key] is not a non-negative int."""
val = msg.get(key)
if not isinstance(val, int) or val < 0:
label = name or key
raise ValueError(f"{label}: expected non-negative int, got {val!r}")
def _require_list(msg: dict, key: str, *, name: str = "") -> None:
"""Raise ValueError if msg[key] is not a list."""
val = msg.get(key)
if not isinstance(val, list):
label = name or key
raise ValueError(f"{label}: expected list, got {type(val).__name__}")
# ---------------------------------------------------------------------------
# Message classes
# ---------------------------------------------------------------------------
class ConnectMsg:
"""Client → Server: initial handshake request.
Fields:
PEER_ID: Local peer identity string.
AGENT_METADATA: RDMA agent metadata (opaque bytes).
BASE_ADDR: Base memory address of the KV block region.
NUM_BLOCKS: Number of blocks in the KV block region.
BLOCK_LEN: Size in bytes of each block (must match between peers).
CONFIG_FINGERPRINT: SHA-256 prefix of the model configuration.
Peers with different fingerprints are incompatible.
"""
TYPE = "connect"
PEER_ID = "peer_id"
AGENT_METADATA = "agent_metadata"
BASE_ADDR = "base_addr"
NUM_BLOCKS = "num_blocks"
BLOCK_LEN = "block_len"
CONFIG_FINGERPRINT = "config_fingerprint"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, ConnectMsg.PEER_ID, str)
_require(msg, ConnectMsg.AGENT_METADATA, bytes)
_require_non_neg_int(msg, ConnectMsg.BASE_ADDR)
_require_pos_int(msg, ConnectMsg.NUM_BLOCKS)
_require_pos_int(msg, ConnectMsg.BLOCK_LEN)
class ConnectAckMsg:
"""Server → Client: handshake acknowledgement.
Fields:
PEER_ID: Server's peer identity string.
"""
TYPE = "connect_ack"
PEER_ID = "peer_id"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, ConnectAckMsg.PEER_ID, str)
class DisconnectMsg:
"""Either → Either: graceful connection close.
No additional fields beyond TYPE_KEY.
"""
TYPE = "disconnect"
class FetchMsg:
"""Client → Server: request blocks by key.
Fields:
KV_REQUEST_ID: Identifies this block transfer request.
BLOCK_HASHES: List of block keys (OffloadKey bytes).
BLOCK_INDEXES: List of remote block indexes (same length as BLOCK_HASHES).
"""
TYPE = "fetch"
KV_REQUEST_ID = "kv_request_id"
BLOCK_HASHES = "block_hashes"
BLOCK_INDEXES = "block_indexes"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, FetchMsg.KV_REQUEST_ID, str)
_require_list(msg, FetchMsg.BLOCK_HASHES)
_require_list(msg, FetchMsg.BLOCK_INDEXES)
hashes = msg[FetchMsg.BLOCK_HASHES]
indexes = msg[FetchMsg.BLOCK_INDEXES]
if len(hashes) != len(indexes):
raise ValueError(
f"block_hashes/block_indexes length mismatch: "
f"{len(hashes)} vs {len(indexes)}"
)
for idx in indexes:
if not isinstance(idx, int) or idx < 0:
raise ValueError(f"block_indexes: invalid index {idx!r}")
class TransferDoneMsg:
"""Server → Client: all blocks transferred for a request.
Fields:
KV_REQUEST_ID: The request that completed.
SUCCESS: Whether the transfer completed successfully.
"""
TYPE = "transfer_done"
KV_REQUEST_ID = "kv_request_id"
SUCCESS = "success"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, TransferDoneMsg.KV_REQUEST_ID, str)
_require(msg, TransferDoneMsg.SUCCESS, bool)
class AbortFetchMsg:
"""Client → Server: cancel a pending request.
Fields:
KV_REQUEST_ID: The request to cancel.
"""
TYPE = "abort_fetch"
KV_REQUEST_ID = "kv_request_id"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, AbortFetchMsg.KV_REQUEST_ID, str)
class AbortAckMsg:
"""Server → Client: acknowledge cancellation.
Fields:
KV_REQUEST_ID: The request that was cancelled.
"""
TYPE = "abort_ack"
KV_REQUEST_ID = "kv_request_id"
@staticmethod
def validate(msg: dict) -> None:
"""Raise ValueError if any field has an invalid type or value."""
_require(msg, AbortAckMsg.KV_REQUEST_ID, str)
@@ -0,0 +1,601 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Server-role state machine for a single peer session.
Owns block matching (supply vs. demand), inflight RDMA transfers, store-job
timeouts, abort-drain, and produces ``StoreResult`` for completed stores.
The session coordinator parses wire messages and dispatches typed
arguments here; this module never touches ``ControlConnection`` directly
— it emits via the ``send`` callback injected by the coordinator (which
gates on ConnectAck).
Protocol violations the role can detect (today: duplicate ``FetchMsg``
for the same ``kv_request_id``) are surfaced as ``ValueError`` so the
coordinator's ``_dispatch_message`` can reuse its existing
``_protocol_error`` path.
"""
from __future__ import annotations
import time
from collections.abc import Callable, Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, NamedTuple
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
TYPE_KEY,
AbortAckMsg,
TransferDoneMsg,
)
if TYPE_CHECKING:
from vllm.v1.kv_offload.tiering.base import JobId
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
logger = init_logger(__name__)
_STORE_TIMEOUT_S = 30.0
_CANCEL_DRAIN_TIMEOUT_S = 10.0
class StoreResult(NamedTuple):
"""Result from a session poll, server side."""
job_id: int
success: bool
class _InflightXfer(NamedTuple):
"""Metadata for a single inflight RDMA transfer, keyed by transfer_id."""
kv_request_id: str
block_count: int
# The set of store job IDs that contributed blocks to this transfer.
job_ids: set[int]
class _MatchResult(NamedTuple):
"""Result of block matching: pairs ready for transfer."""
local_idxs: list[int]
remote_idxs: list[int]
# The set of store job IDs that contributed blocks
job_ids: set[int]
@dataclass
class _OutboundRequestState:
"""Server-role state for a single peer fetch request.
The owning ``kv_request_id`` is the dict key in
``ServerRole._outbound`` and is not duplicated on the value.
"""
demand_received: bool = False
available: dict[OffloadKey, tuple[int, int]] = field(
default_factory=dict
) # key → (job_id, local_block_idx): blocks we have, awaiting demand
demanded: dict[OffloadKey, int] = field(
default_factory=dict
) # key → remote_block_idx: blocks peer wants, awaiting supply
remaining: int = 0 # blocks that need to be transferred to client
finishing: bool = False # Signal finish request ASAP
# Job IDs that submit_store'd blocks for this request and have not
# yet emitted a StoreResult. The terminal-finalize helper drains
# this set; poll-done and poll-failed discard entries as their
# StoreResults fire.
pending_job_ids: set[int] = field(default_factory=set)
def add_stored_blocks(
self,
block_hashes: Sequence[OffloadKey],
block_ids: Sequence[int],
job_id: int,
) -> _MatchResult:
"""Add locally-stored blocks. Returns matched pairs."""
self.pending_job_ids.add(job_id)
local_idxs: list[int] = []
remote_idxs: list[int] = []
for block_hash, local_idx in zip(block_hashes, block_ids):
remote_idx = self.demanded.pop(block_hash, None)
if remote_idx is not None:
local_idxs.append(local_idx)
remote_idxs.append(remote_idx)
else:
self.available[block_hash] = (job_id, local_idx)
return _MatchResult(
local_idxs=local_idxs,
remote_idxs=remote_idxs,
job_ids={job_id} if local_idxs else set(),
)
def add_fetch_demand(
self,
block_hashes: Sequence[OffloadKey],
block_indexes: Sequence[int],
) -> _MatchResult:
"""Register the peer's fetch demand. Returns matched pairs."""
self.demand_received = True
self.remaining = len(block_hashes)
local_idxs: list[int] = []
remote_idxs: list[int] = []
job_ids: set[int] = set()
for block_hash, remote_idx in zip(block_hashes, block_indexes):
stored_entry = self.available.pop(block_hash, None)
if stored_entry is not None:
stored_job_id, local_idx = stored_entry
local_idxs.append(local_idx)
remote_idxs.append(remote_idx)
job_ids.add(stored_job_id)
else:
self.demanded[block_hash] = remote_idx
return _MatchResult(
local_idxs=local_idxs,
remote_idxs=remote_idxs,
job_ids=job_ids,
)
class ServerRole:
"""Server-side store/serve state machine for one peer session.
The coordinator owns the connection and the send-gating; this role
is given a ``send`` callback, the ``DataTransport``, and the
``peer_id`` for transport calls and log messages.
"""
def __init__(
self,
peer_id: str,
transport: DataTransport,
send: Callable[[dict], None],
) -> None:
self._peer_id = peer_id
self._transport = transport
self._send = send
self._outbound: dict[str, _OutboundRequestState] = {}
# transfer_id → xfer. Mutate ONLY via _inflight_add / _inflight_pop
# so the per-request count below stays in sync.
self._inflight: dict[int, _InflightXfer] = {}
# kv_request_id → number of entries in _inflight for that id.
# Kept in sync with _inflight; entries that hit zero are removed
# so `kv_request_id in self._inflight_per_req` is an exact
# "has any inflight transfer" predicate (O(1) replacement for
# the previous O(N) scan).
self._inflight_per_req: dict[str, int] = {}
self._store_jobs: dict[int, float] = {} # job_id → submitted_at
self._pending_aborts: dict[str, float] = {} # kv_request_id → start
# StoreResults queued by _finalize_outbound for the next poll
# tick to surface. Mirrors the deferred-result pattern used for
# load timeouts.
self._pending_store_results: list[StoreResult] = []
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def add_stored_blocks(
self,
kv_request_id: str,
keys: Sequence[OffloadKey],
block_ids: Sequence[int],
job_id: JobId,
) -> None:
"""New blocks stored locally — match against pending fetch demand."""
self._store_jobs[job_id] = time.monotonic()
req = self._outbound.setdefault(kv_request_id, _OutboundRequestState())
result = req.add_stored_blocks(keys, block_ids, job_id)
if result.local_idxs and req.demand_received:
self._submit_transfer(kv_request_id, result)
def on_fetch(
self,
kv_request_id: str,
block_hashes: Sequence[OffloadKey],
block_indexes: Sequence[int],
) -> None:
"""Handle a FetchMsg from the peer.
Raises ``ValueError`` on a duplicate fetch for the same
``kv_request_id``; the coordinator's dispatch loop turns that
into a protocol-error disconnect.
"""
logger.debug(
"P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d",
self._peer_id,
kv_request_id,
len(block_hashes),
)
existing = self._outbound.get(kv_request_id)
if existing is not None and existing.demand_received:
# A second fetch for the same kv_request_id would overwrite
# `remaining` and leak inflight bookkeeping. Treat as a
# protocol violation.
raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}")
req = self._outbound.setdefault(kv_request_id, _OutboundRequestState())
result = req.add_fetch_demand(block_hashes, block_indexes)
if result.local_idxs:
self._submit_transfer(kv_request_id, result)
# Prefiller-first mode: finish_request may have run before
# fetch arrived. If so, finalize once we know what was
# demanded — fully satisfied → success, else early-fail.
if req.finishing and not self._has_inflight_for(kv_request_id):
self._finalize_outbound(kv_request_id)
def on_abort_fetch(self, kv_request_id: str) -> None:
"""Handle an AbortFetchMsg from the peer."""
# Abort for an unknown id may be a benign race/duplicate or a
# real protocol violation; we don't track completed ids, so warn.
if kv_request_id not in self._outbound and not self._has_inflight_for(
kv_request_id
):
logger.warning(
"P2PSession %s: abort_fetch for unknown kv_request_id=%s "
"(no outbound or inflight state); benign race or stale",
self._peer_id,
kv_request_id,
)
# Idempotent: receiving AbortFetchMsg again before we've sent the
# ack just triggers another drain attempt without resetting the
# deadline.
self._pending_aborts.setdefault(kv_request_id, time.monotonic())
self._drain_abort(kv_request_id)
def finish(self, kv_request_id: str) -> None:
"""Mark an outbound request finishing.
No more submit_store calls will arrive for this id. Any blocks
the peer demanded but we never stored will never come; tell the
peer to stop waiting (TransferDoneMsg success=False) instead of
letting it hit _LOAD_TIMEOUT_S.
If the decoder hasn't sent fetch yet (no demand received),
defer — on_fetch will finalize once demand arrives.
If inflight transfers exist for this id, defer — the last
completing transfer in collect_results will fire the message.
"""
req = self._outbound.get(kv_request_id)
if req is None:
return
req.finishing = True
if not req.demand_received:
return
if self._has_inflight_for(kv_request_id):
return
# Remaining > 0 here: if it had hit 0, the poll-done success
# branch would have already popped _outbound and we'd have
# returned at `req is None` above. Helper derives success from
# remaining and emits StoreResult(success=False) for any
# leftover pending jobs.
self._finalize_outbound(kv_request_id)
def collect_results(self) -> list[StoreResult]:
"""Drain timeouts, deferred results, and transport completions."""
results: list[StoreResult] = self._timeout_pending_store_jobs()
if self._pending_store_results:
results.extend(self._pending_store_results)
self._pending_store_results.clear()
poll_result = self._transport.poll()
for tid in poll_result.done:
xfer = self._inflight_pop(tid)
if xfer is None:
# Bug signal: transport reported a transfer we have no
# bookkeeping for. Likely a double-completion in the
# transport or a stale removal in the session. The
# attached job(s) still live in _store_jobs and will be
# surfaced as failures by _timeout_pending_store_jobs
# after _STORE_TIMEOUT_S, but log loudly so the
# underlying bug is findable.
logger.error(
"P2PSession %s: transport reported done for unknown "
"transfer_id=%d; attached job(s) will fail via "
"store-timeout instead of completing now",
self._peer_id,
tid,
)
continue
req = self._outbound.get(xfer.kv_request_id)
for job_id in xfer.job_ids:
if self._store_jobs.pop(job_id, None) is None:
# Already reported (timeout, cancellation, etc.) —
# don't double-emit a contradictory success result.
continue
results.append(StoreResult(job_id=job_id, success=True))
if req is not None:
req.pending_job_ids.discard(job_id)
if req is not None and req.demand_received:
req.remaining -= xfer.block_count
assert req.remaining >= 0, (
f"remaining went negative for kv_request_id={xfer.kv_request_id}"
)
if req.remaining == 0:
self._finalize_outbound(xfer.kv_request_id, success=True)
elif req.finishing and not self._has_inflight_for(xfer.kv_request_id):
self._finalize_outbound(xfer.kv_request_id, success=False)
failed_kv_request_ids: set[str] | None = None
for tid in poll_result.failed:
xfer = self._inflight_pop(tid)
if xfer is None:
# See the matching error log in the done branch above.
logger.error(
"P2PSession %s: transport reported failed for unknown "
"transfer_id=%d; attached job(s) will fail via "
"store-timeout instead of completing now",
self._peer_id,
tid,
)
continue
if failed_kv_request_ids is None:
failed_kv_request_ids = set()
failed_kv_request_ids.add(xfer.kv_request_id)
req_for_xfer = self._outbound.get(xfer.kv_request_id)
for job_id in xfer.job_ids:
if self._store_jobs.pop(job_id, None) is None:
# Already reported (timeout, cancellation, etc.) —
# don't double-emit.
continue
results.append(StoreResult(job_id=job_id, success=False))
if req_for_xfer is not None:
req_for_xfer.pending_job_ids.discard(job_id)
req = self._outbound.pop(xfer.kv_request_id, None)
if req is not None and req.demand_received:
self._send(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id,
TransferDoneMsg.SUCCESS: False,
}
)
# Cancel other inflight for the same failed kv_request_ids
if failed_kv_request_ids:
ids_to_cancel = [
tid
for tid, xfer in self._inflight.items()
if xfer.kv_request_id in failed_kv_request_ids
]
for tid in ids_to_cancel:
self._inflight_pop(tid)
self._transport.cancel(ids_to_cancel)
return results
def collect_idle_timeouts(self) -> list[StoreResult]:
"""Run only the store-job timeout sweep.
Used by the coordinator's no-conn poll path: a pending session
cannot have inflight transfers (no peer registered yet), so we
skip the transport poll and the deferred-result drain.
"""
return self._timeout_pending_store_jobs()
def drain_pending_aborts(self) -> None:
"""Re-attempt every parked abort once per poll tick."""
if not self._pending_aborts:
return
for kv_request_id in list(self._pending_aborts):
self._drain_abort(kv_request_id)
def close(self) -> list[int]:
"""Tear down. Cancels inflight, returns failed store job ids."""
failed_stores = list(self._store_jobs.keys())
self._store_jobs.clear()
if self._inflight:
self._transport.cancel(list(self._inflight.keys()))
self._inflight.clear()
self._inflight_per_req.clear()
self._outbound.clear()
self._pending_aborts.clear()
self._pending_store_results.clear()
return failed_stores
# ------------------------------------------------------------------
# Internal — inflight bookkeeping
# ------------------------------------------------------------------
def _has_inflight_for(self, kv_request_id: str) -> bool:
return kv_request_id in self._inflight_per_req
def _inflight_add(self, tid: int, xfer: _InflightXfer) -> None:
"""Insert an inflight transfer and bump the per-request count."""
self._inflight[tid] = xfer
self._inflight_per_req[xfer.kv_request_id] = (
self._inflight_per_req.get(xfer.kv_request_id, 0) + 1
)
def _inflight_pop(self, tid: int) -> _InflightXfer | None:
"""Pop an inflight transfer and decrement the per-request count.
Removes the per-request entry once the count hits zero so the
dict stays bounded and `_has_inflight_for` remains exact.
"""
xfer = self._inflight.pop(tid, None)
if xfer is None:
return None
new_count = self._inflight_per_req.get(xfer.kv_request_id, 0) - 1
if new_count > 0:
self._inflight_per_req[xfer.kv_request_id] = new_count
else:
self._inflight_per_req.pop(xfer.kv_request_id, None)
return xfer
# ------------------------------------------------------------------
# Internal — finalize / abort drain
# ------------------------------------------------------------------
def _finalize_outbound(
self,
kv_request_id: str,
success: bool | None = None,
) -> None:
"""Pop the outbound state and emit terminal results.
Called when no further work will happen for this kv_request_id
on the server side: either request_finish has fired and there
are no inflight transfers, or the last inflight just completed
while finishing.
If ``success`` is None, derive it from ``req.remaining == 0``.
The same flag is used for both the peer's TransferDoneMsg and
the StoreResult(s) emitted for any leftover pending job_ids.
"""
req = self._outbound.pop(kv_request_id, None)
if req is None:
return
if success is None:
success = req.remaining == 0
for job_id in req.pending_job_ids:
self._store_jobs.pop(job_id, None)
self._pending_store_results.append(
StoreResult(job_id=job_id, success=success)
)
self._send(
{
TYPE_KEY: TransferDoneMsg.TYPE,
TransferDoneMsg.KV_REQUEST_ID: kv_request_id,
TransferDoneMsg.SUCCESS: success,
}
)
def _drain_abort(self, kv_request_id: str) -> None:
"""One drain attempt for a pending abort.
Stops accepting more blocks for ``kv_request_id``, then asks the
transport to cancel any matching inflight transfers in
``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains
inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to
``mode="immediate"`` and acks anyway.
"""
self._outbound.pop(kv_request_id, None)
ids = [
tid
for tid, xfer in self._inflight.items()
if xfer.kv_request_id == kv_request_id
]
if not ids:
self._finalize_abort(kv_request_id)
return
started_at = self._pending_aborts.get(kv_request_id)
expired = (
started_at is not None
and time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S
)
if expired:
for tid in ids:
self._inflight_pop(tid)
self._transport.cancel(ids, mode="immediate")
logger.warning(
"P2PSession %s: cancel drain timed out for kv_request_id=%s,"
" force-canceled %d transfers",
self._peer_id,
kv_request_id,
len(ids),
)
self._finalize_abort(kv_request_id)
return
still = self._transport.cancel(ids, mode="wait")
# Tids the transport successfully released are gone from its
# _inflight; mirror that in session bookkeeping so they don't
# block the drain forever waiting for a poll() event that will
# never come.
still_set = set(still)
for tid in ids:
if tid not in still_set:
self._inflight_pop(tid)
if not still:
self._finalize_abort(kv_request_id)
def _finalize_abort(self, kv_request_id: str) -> None:
self._pending_aborts.pop(kv_request_id, None)
self._send(
{
TYPE_KEY: AbortAckMsg.TYPE,
AbortAckMsg.KV_REQUEST_ID: kv_request_id,
}
)
# ------------------------------------------------------------------
# Internal — transfers and store-job timeouts
# ------------------------------------------------------------------
def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None:
logger.debug(
"P2PSession %s: NIXL write_blocks CALL kv_request_id=%s "
"local_idxs=%d remote_idxs=%d",
self._peer_id,
kv_request_id,
len(result.local_idxs),
len(result.remote_idxs),
)
transfer_id = self._transport.write_blocks(
self._peer_id, result.local_idxs, result.remote_idxs
)
if transfer_id is not None:
logger.debug(
"P2PSession %s: NIXL write_blocks SUBMITTED kv_request_id=%s "
"transfer_id=%d blocks=%d",
self._peer_id,
kv_request_id,
transfer_id,
len(result.local_idxs),
)
self._inflight_add(
transfer_id,
_InflightXfer(
kv_request_id=kv_request_id,
block_count=len(result.local_idxs),
job_ids=result.job_ids,
),
)
else:
logger.warning(
"P2PSession %s: write_blocks failed for %s (%d blocks)",
self._peer_id,
kv_request_id,
len(result.local_idxs),
)
# The matched blocks were popped from req.demanded /
# req.available, but no inflight will satisfy them, so
# remaining will never reach 0 on its own. Mark the
# request as finishing so the existing terminal paths
# clean up: if other inflight is in flight, the last one
# to drain will fire _finalize_outbound(success=False)
# via the elif branch in collect_results. If
# nothing else is in flight, finalize now so the peer
# and the local store jobs don't wait for finish_request
# or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S.
req = self._outbound.get(kv_request_id)
if req is not None:
req.finishing = True
if not self._has_inflight_for(kv_request_id):
self._finalize_outbound(kv_request_id, success=False)
def _timeout_pending_store_jobs(self) -> list[StoreResult]:
if not self._store_jobs:
return []
deadline = time.monotonic() - _STORE_TIMEOUT_S
timed_out: list[int] | None = None
for jid, submitted_at in self._store_jobs.items():
if submitted_at <= deadline:
if timed_out is None:
timed_out = []
timed_out.append(jid)
if timed_out is None:
return []
results: list[StoreResult] = []
for jid in timed_out:
del self._store_jobs[jid]
results.append(StoreResult(job_id=jid, success=False))
logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid)
return results
@@ -0,0 +1,440 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
P2PSession — bidirectional session combining client + server roles.
A single P2PSession per remote peer handles BOTH directions of the P2P
protocol on one ControlConnection: it can request blocks from the peer
("client" role, in :mod:`.client`) AND serve blocks to the peer
("server" role, in :mod:`.server`). This module is the thin coordinator
that owns the connection, the handshake, send-gating, and the message
dispatch — each parsed message is forwarded to the corresponding role.
Wire protocol is unchanged. Both sides advertise their NIXL metadata
via ConnectMsg when their session is connected; the peer's ConnectMsg
triggers transport.add_remote_peer; ConnectAckMsg confirms the peer
received our ConnectMsg, after which queued outgoing messages are flushed.
"""
from __future__ import annotations
import contextlib
from collections.abc import Sequence
from typing import TYPE_CHECKING, NamedTuple
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey
from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection
from vllm.v1.kv_offload.tiering.p2p.session.client import ClientRole, LoadResult
from vllm.v1.kv_offload.tiering.p2p.session.protocol import (
TYPE_KEY,
AbortAckMsg,
AbortFetchMsg,
ConnectAckMsg,
ConnectMsg,
DisconnectMsg,
FetchMsg,
TransferDoneMsg,
)
from vllm.v1.kv_offload.tiering.p2p.session.server import (
ServerRole,
StoreResult,
)
if TYPE_CHECKING:
from vllm.v1.kv_offload.tiering.base import JobId
from vllm.v1.kv_offload.tiering.p2p.data import DataTransport
logger = init_logger(__name__)
# Cap on consecutive non-protocol dispatch exceptions before we tear
# down the session. Protocol violations (ValueError) disconnect on the
# first occurrence; this threshold protects against repeated internal
# bugs that may indicate a peer-induced bad state. Reset on any
# successful dispatch.
_MAX_CONSECUTIVE_DISPATCH_ERRORS = 5
class SessionPollResult(NamedTuple):
"""Result of one P2PSession.poll() tick.
`loads`/`stores` are the same per-role results the manager has always
consumed. `new_fetch_ids` reports kv_request_ids whose FetchMsg
arrived this tick — the manager uses them to bind kv_request_id →
session and replay any submit_store batches parked while no peer had
asked yet. Reporting (rather than calling back into the manager
mid-dispatch) keeps the dependency strictly top-down.
"""
loads: list[LoadResult]
stores: list[StoreResult]
new_fetch_ids: list[str]
class P2PSession:
"""Bidirectional session — coordinator over ClientRole + ServerRole.
Lifecycle:
- Constructor with conn=None ⇒ pending. Accepts add_stored_blocks
but cannot send (used by the prefiller to buffer blocks before
the decoder connects).
- Constructor with conn != None ⇒ connected. Sends our own ConnectMsg
immediately; the peer's ConnectMsg arrives in poll() and is
dispatched to _on_connect (which calls transport.add_remote_peer
and replies with ConnectAckMsg). Outgoing sends are queued until
ConnectAckMsg confirms our metadata reached the peer.
- attach_connection(conn) on a pending session ⇒ same as above,
starting from pending.
"""
def __init__(
self,
peer_id: str,
local_id: str,
transport: DataTransport,
local_block_len: int,
conn: ControlConnection | None = None,
) -> None:
self.peer_id = peer_id
self._local_id = local_id
self._transport = transport
self._local_block_len = local_block_len
self._conn: ControlConnection | None = None
self._send_ready = False # True after the peer acked our ConnectMsg
# Msgs waiting to be sent on connection establishment
self._queued: list[dict] = []
# Consecutive non-protocol dispatch errors. Reset on success.
self._dispatch_error_count: int = 0
# kv_request_ids whose FetchMsg arrived during the current poll
# tick. Drained and returned in the next poll() result so the
# manager can bind kv_request_id → session and replay any
# submit_store batches parked before the binding existed.
self._new_fetch_ids: list[str] = []
self._client = ClientRole(peer_id=peer_id, send=self._send)
self._server = ServerRole(peer_id=peer_id, transport=transport, send=self._send)
if conn is not None:
self.attach_connection(conn)
# ------------------------------------------------------------------
# Properties
# ------------------------------------------------------------------
@property
def alive(self) -> bool:
# Pending sessions (awaiting connection) are alive — only a
# closed real connection counts as dead.
return self._conn is None or self._conn.alive
@property
def connected(self) -> bool:
return self._conn is not None
@property
def ready(self) -> bool:
"""True after the peer acked our ConnectMsg (we may send freely)."""
return self._send_ready
# ------------------------------------------------------------------
# Connection lifecycle
# ------------------------------------------------------------------
def attach_connection(self, conn: ControlConnection) -> None:
"""Attach a connection to a pending session and announce ourselves.
Symmetric: every side advertises its NIXL metadata on connect, so
whichever peer receives a session first can register the other.
"""
if self._conn is not None:
raise ValueError(f"P2PSession {self.peer_id}: already connected")
self._conn = conn
self._send_connect()
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
def request_blocks(
self,
job_id: JobId,
kv_request_id: str,
keys: Sequence[bytes],
block_ids: Sequence[int],
) -> None:
"""Send fetch to the peer."""
self._client.request_blocks(
job_id, kv_request_id, keys, block_ids, send_ready=self._send_ready
)
def add_stored_blocks(
self,
kv_request_id: str,
keys: Sequence[OffloadKey],
block_ids: Sequence[int],
job_id: JobId,
) -> None:
"""New blocks stored locally — match against pending fetch demand."""
self._server.add_stored_blocks(kv_request_id, keys, block_ids, job_id)
def finish_request(self, kv_request_id: str) -> None:
"""Called when the request is finishing locally.
Cancels any inbound load (client role) and finalizes any
outbound serving (server role) for this id. Roles that aren't
active for this id are silent no-ops.
"""
self._client.cancel(kv_request_id)
self._server.finish(kv_request_id)
def poll(self) -> SessionPollResult:
"""Process incoming messages, drive transfers, apply timeouts."""
if self._conn is None:
# Pending session — store-job timeouts still apply so buffered
# jobs that never get picked up are surfaced as failures.
return SessionPollResult(
loads=[],
stores=self._server.collect_idle_timeouts(),
new_fetch_ids=[],
)
for msg in self._conn.recv():
self._on_message(msg)
loads = self._client.collect_results()
stores = self._server.collect_results()
self._server.drain_pending_aborts()
new_fetch_ids = self._new_fetch_ids
self._new_fetch_ids = []
return SessionPollResult(
loads=loads, stores=stores, new_fetch_ids=new_fetch_ids
)
def close(self) -> tuple[list[tuple[int, str]], list[int]]:
"""Shut down. Returns (failed_loads, failed_stores).
failed_loads: list of (job_id, kv_request_id) pairs.
failed_stores: list of job_ids.
"""
failed_loads = self._client.close()
failed_stores = self._server.close()
if self._conn is not None:
with contextlib.suppress(Exception):
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
self._conn.close()
self._conn = None
return failed_loads, failed_stores
# ------------------------------------------------------------------
# Message dispatch
# ------------------------------------------------------------------
def _on_message(self, msg: dict) -> None:
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else msg
try:
self._dispatch_message(msg)
except ValueError as exc:
# Protocol contract violation from the peer — *Msg.validate()
# and handler-level checks raise ValueError. Retrying won't
# help and may corrupt session state, so disconnect now.
self._protocol_error(f"malformed {msg_type!r}: {exc}")
return
except Exception as exc:
# Anything else is most likely an internal bug rather than a
# peer fault. Log loudly with a traceback so it doesn't
# disappear, but don't kill the session on a single hiccup.
# Disconnect only if errors keep arriving — that pattern is
# consistent with a peer wedging us into a broken state.
self._dispatch_error_count += 1
logger.exception(
"P2PSession %s: error handling message %r (count=%d): %s",
self.peer_id,
msg_type,
self._dispatch_error_count,
exc,
)
if self._dispatch_error_count >= _MAX_CONSECUTIVE_DISPATCH_ERRORS:
self._protocol_error(
f"too many consecutive dispatch errors "
f"({self._dispatch_error_count})"
)
return
self._dispatch_error_count = 0
def _protocol_error(self, reason: str) -> None:
"""Log a protocol violation and disconnect.
Best-effort sends ``DisconnectMsg`` so the peer learns why we're
going away, then marks the connection dead. The manager reaps
the session on the next poll via ``alive``.
"""
logger.error(
"P2PSession %s: protocol error: %s — disconnecting",
self.peer_id,
reason,
)
if self._conn is not None:
with contextlib.suppress(Exception):
self._conn.send({TYPE_KEY: DisconnectMsg.TYPE})
self._conn.mark_dead()
def _dispatch_message(self, msg: dict) -> None:
# Drop messages buffered before disconnect: a poll batch can
# contain msg-after-DisconnectMsg, and dispatching them would
# mutate state on a dead session.
if self._conn is not None and not self._conn.alive:
return
msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else None
if msg_type == ConnectMsg.TYPE:
self._on_connect(msg)
elif msg_type == ConnectAckMsg.TYPE:
ConnectAckMsg.validate(msg)
self._on_connect_ack()
elif msg_type == FetchMsg.TYPE:
FetchMsg.validate(msg)
kv_request_id = msg[FetchMsg.KV_REQUEST_ID]
block_hashes = [
OffloadKey(bh if isinstance(bh, bytes) else bytes(bh))
for bh in msg[FetchMsg.BLOCK_HASHES]
]
block_indexes = msg[FetchMsg.BLOCK_INDEXES]
# Run the server-role state machine inline as today —
# add_fetch_demand records demand against any blocks we've
# already seen in `available`. Report the kv_request_id so
# the manager (after poll() returns) can replay any parked
# submit_store batches; their add_stored_blocks calls hit
# the demand recorded here and submit transfers immediately.
self._server.on_fetch(kv_request_id, block_hashes, block_indexes)
self._new_fetch_ids.append(kv_request_id)
elif msg_type == AbortFetchMsg.TYPE:
AbortFetchMsg.validate(msg)
self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID])
elif msg_type == TransferDoneMsg.TYPE:
TransferDoneMsg.validate(msg)
self._client.on_transfer_done(
msg[TransferDoneMsg.KV_REQUEST_ID],
msg[TransferDoneMsg.SUCCESS],
)
elif msg_type == AbortAckMsg.TYPE:
AbortAckMsg.validate(msg)
self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID])
elif msg_type == DisconnectMsg.TYPE:
if self._conn is not None:
self._conn.mark_dead()
else:
logger.warning(
"P2PSession %s: unknown message type %r", self.peer_id, msg_type
)
# ------------------------------------------------------------------
# Handshake
# ------------------------------------------------------------------
def _on_connect(self, msg: dict) -> None:
# Validation failures here mean an incompatible or malicious peer.
# Mark the connection dead so the manager reaps the session;
# don't call add_remote_peer or send connect_ack.
if self._send_ready:
# We've already received connect_ack, so the handshake is
# complete. A second connect from the peer is a protocol
# violation — re-registering would corrupt transport state.
self._protocol_error("duplicate connect after handshake")
return
try:
ConnectMsg.validate(msg)
if msg[ConnectMsg.BLOCK_LEN] != self._local_block_len:
raise ValueError(
f"block_len mismatch from {self.peer_id}: "
f"remote={msg[ConnectMsg.BLOCK_LEN]}, "
f"local={self._local_block_len}"
)
remote_fp = msg.get(ConnectMsg.CONFIG_FINGERPRINT, "")
local_fp = self._transport.config_fingerprint
if local_fp and remote_fp and remote_fp != local_fp:
raise ValueError(
f"config fingerprint mismatch from {self.peer_id}: "
f"remote={remote_fp!r}, local={local_fp!r}"
)
self._transport.add_remote_peer(
self.peer_id,
agent_metadata=msg[ConnectMsg.AGENT_METADATA],
base_addr=msg[ConnectMsg.BASE_ADDR],
num_blocks=msg[ConnectMsg.NUM_BLOCKS],
block_len=msg[ConnectMsg.BLOCK_LEN],
)
except ValueError as exc:
logger.error("P2PSession %s: rejecting peer connect: %s", self.peer_id, exc)
if self._conn is not None:
self._conn.mark_dead()
return
if self._conn is not None:
self._conn.send(
{
TYPE_KEY: ConnectAckMsg.TYPE,
ConnectAckMsg.PEER_ID: self._local_id,
}
)
def _on_connect_ack(self) -> None:
if self._queued:
logger.debug(
"P2PSession %s: connect_ack received, flushing %d queued msg(s)",
self.peer_id,
len(self._queued),
)
self._send_ready = True
for queued in self._queued:
self._do_send(queued)
self._queued.clear()
# ------------------------------------------------------------------
# Send helpers
# ------------------------------------------------------------------
def _send_connect(self) -> None:
"""Send our ConnectMsg announcing local NIXL metadata."""
assert self._conn is not None
self._conn.send(
{
TYPE_KEY: ConnectMsg.TYPE,
ConnectMsg.PEER_ID: self._local_id,
ConnectMsg.AGENT_METADATA: self._transport.get_agent_metadata(),
ConnectMsg.BASE_ADDR: self._transport.base_addr,
ConnectMsg.NUM_BLOCKS: self._transport.num_blocks,
ConnectMsg.BLOCK_LEN: self._transport.block_len,
ConnectMsg.CONFIG_FINGERPRINT: self._transport.config_fingerprint,
}
)
def _send(self, msg: dict) -> None:
if self._conn is None or not self._send_ready:
logger.debug(
"P2PSession %s: queueing %s (ready=%s queue_depth=%d)",
self.peer_id,
msg.get(TYPE_KEY),
self._send_ready,
len(self._queued) + 1,
)
self._queued.append(msg)
return
self._do_send(msg)
def _do_send(self, msg: dict) -> None:
if self._conn is None:
return
try:
self._conn.send(msg)
logger.debug("P2PSession %s: sent %s", self.peer_id, msg.get(TYPE_KEY))
except Exception:
logger.warning(
"P2PSession %s: failed to send %s",
self.peer_id,
msg.get(TYPE_KEY),
)