From bec232a9146bfd61e2be83879b0b74b952818758 Mon Sep 17 00:00:00 2001 From: liranschour Date: Tue, 30 Jun 2026 07:51:44 +0300 Subject: [PATCH] Secondary tier implementation for PD disaggregation (#42285) Signed-off-by: Liran Schour Signed-off-by: liranschour Co-authored-by: Or Ozeri Co-authored-by: Or Ozeri --- docs/features/kv_offloading_usage.md | 14 + tests/v1/kv_offload/tiering/p2p/__init__.py | 0 .../tiering/p2p/p2p_connector_proxy.py | 331 ++++ .../tiering/p2p/run_accuracy_test.sh | 312 ++++ .../tiering/p2p/test_data_transport.py | 330 ++++ .../v1/kv_offload/tiering/p2p/test_manager.py | 1370 ++++++++++++++ .../kv_offload/tiering/p2p/test_sessions.py | 1645 +++++++++++++++++ .../tiering/p2p/test_zmq_transport.py | 230 +++ vllm/v1/kv_offload/tiering/factory.py | 6 + vllm/v1/kv_offload/tiering/p2p/__init__.py | 0 .../tiering/p2p/control/__init__.py | 17 + .../v1/kv_offload/tiering/p2p/control/base.py | 166 ++ vllm/v1/kv_offload/tiering/p2p/control/zmq.py | 307 +++ .../kv_offload/tiering/p2p/data/__init__.py | 10 + vllm/v1/kv_offload/tiering/p2p/data/base.py | 259 +++ vllm/v1/kv_offload/tiering/p2p/data/nixl.py | 297 +++ vllm/v1/kv_offload/tiering/p2p/manager.py | 664 +++++++ .../tiering/p2p/session/__init__.py | 15 + .../kv_offload/tiering/p2p/session/client.py | 210 +++ .../tiering/p2p/session/protocol.py | 247 +++ .../kv_offload/tiering/p2p/session/server.py | 601 ++++++ .../kv_offload/tiering/p2p/session/session.py | 440 +++++ 22 files changed, 7471 insertions(+) create mode 100644 tests/v1/kv_offload/tiering/p2p/__init__.py create mode 100644 tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py create mode 100755 tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh create mode 100644 tests/v1/kv_offload/tiering/p2p/test_data_transport.py create mode 100644 tests/v1/kv_offload/tiering/p2p/test_manager.py create mode 100644 tests/v1/kv_offload/tiering/p2p/test_sessions.py create mode 100644 tests/v1/kv_offload/tiering/p2p/test_zmq_transport.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/__init__.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/control/__init__.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/control/base.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/control/zmq.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/data/__init__.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/data/base.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/data/nixl.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/manager.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/session/__init__.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/session/client.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/session/protocol.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/session/server.py create mode 100644 vllm/v1/kv_offload/tiering/p2p/session/session.py diff --git a/docs/features/kv_offloading_usage.md b/docs/features/kv_offloading_usage.md index 8ef5d6c63d..cff65753d9 100644 --- a/docs/features/kv_offloading_usage.md +++ b/docs/features/kv_offloading_usage.md @@ -120,6 +120,20 @@ To enable KV cache sharing between multiple vLLM instances using the same `root_ PYTHONHASHSEED=0 vllm serve ... ``` +### P2P (Including P/D) + +The P2P tier (`type: "p2p"`) shares completed KV blocks between vLLM instances over RDMA via NIXL. Each instance binds a control socket on `host:port` and exchanges blocks directly with peers — no shared filesystem required. + +| Key | Required | Default | Notes | +| --- | --- | --- | --- | +| `type` | yes | — | Must be `p2p`. | +| `host` | no | `0.0.0.0` | Address the control socket binds to. | +| `port` | no | `7777` | Port for the control socket. Must be reachable from peers. | +| `backends` | no | `["UCX"]` | NIXL transport backends. See [NixlConnector Usage Guide](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin) for available backends and selection guidance. | +| `num_threads` | no | `4` | NIXL agent worker threads. Only used when `backends` is UCX-only; ignored when any non-UCX backend is requested. | + +The `backends` and `num_threads` options mirror the conditional logic used by [`NixlConnector`](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin): when any non-UCX backend is configured, NIXL is initialised with `backends=...`; otherwise it falls back to a UCX-only agent with the configured `num_threads`. This lets the P2P tier use a different transport (e.g. `MOONCAKE`, `GDS_MT`, `LIBFABRIC`) than the main `NixlConnector` running in the same process. + ## Tuning Tips - `cpu_bytes_to_use`: a bigger CPU tier means fewer trips to slower secondary tiers and a higher hit rate. The value is total across all workers, not per-worker. Leave headroom for the rest of the host workload. diff --git a/tests/v1/kv_offload/tiering/p2p/__init__.py b/tests/v1/kv_offload/tiering/p2p/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py b/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py new file mode 100644 index 0000000000..10566f40e6 --- /dev/null +++ b/tests/v1/kv_offload/tiering/p2p/p2p_connector_proxy.py @@ -0,0 +1,331 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +P2PConnector proxy server for OffloadingConnector + TieringOffloadingSpec. + +Unlike NixlConnector (which returns remote_host/remote_port in the prefill +response), OffloadingConnector does not embed connector coordinates in its +response. This proxy injects the prefiller's P2PConnector address into +kv_transfer_params before forwarding the decode request so the decoder knows +where to pull KV blocks from. + +Usage: + .venv/bin/python p2p_connector_proxy.py \ + --port 8192 \ + --prefiller-host 127.0.0.1 --prefiller-port 8100 \ + --decoder-host 127.0.0.1 --decoder-port 8200 \ + --p2p-connector-host 127.0.0.1 --p2p-connector-port 7777 +""" + +import argparse +import asyncio +import itertools +import logging +import os +import uuid +from contextlib import asynccontextmanager + +import httpx +from fastapi import FastAPI, Request +from fastapi.responses import StreamingResponse + +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.prefill_clients = [] + app.state.decode_clients = [] + + for i, (host, port) in enumerate(global_args.prefiller_instances): + app.state.prefill_clients.append( + { + "client": httpx.AsyncClient( + timeout=None, + base_url=f"http://{host}:{port}/v1", + limits=httpx.Limits( + max_connections=None, max_keepalive_connections=None + ), + ), + "host": host, + "port": port, + "id": i, + } + ) + + for i, (host, port) in enumerate(global_args.decoder_instances): + app.state.decode_clients.append( + { + "client": httpx.AsyncClient( + timeout=None, + base_url=f"http://{host}:{port}/v1", + limits=httpx.Limits( + max_connections=None, max_keepalive_connections=None + ), + ), + "host": host, + "port": port, + "id": i, + } + ) + + app.state.prefill_iterator = itertools.cycle(range(len(app.state.prefill_clients))) + app.state.decode_iterator = itertools.cycle(range(len(app.state.decode_clients))) + + mode = "decoder-first" if global_args.decoder_first else "prefiller-first" + pd_host = global_args.p2p_connector_host + pd_port = global_args.p2p_connector_port + print( + f"Proxy ready [{mode}]: " + f"{len(app.state.prefill_clients)} prefiller(s), " + f"{len(app.state.decode_clients)} decoder(s). " + f"P2PConnector at {pd_host}:{pd_port}" + ) + yield + + for ci in app.state.prefill_clients: + await ci["client"].aclose() + for ci in app.state.decode_clients: + await ci["client"].aclose() + + +app = FastAPI(lifespan=lifespan) + + +def parse_args(): + p = argparse.ArgumentParser() + p.add_argument("--port", type=int, default=8192) + p.add_argument("--host", type=str, default="127.0.0.1") + p.add_argument("--prefiller-hosts", type=str, nargs="+", default=["127.0.0.1"]) + p.add_argument("--prefiller-ports", type=int, nargs="+", default=[8100]) + p.add_argument("--decoder-hosts", type=str, nargs="+", default=["127.0.0.1"]) + p.add_argument("--decoder-ports", type=int, nargs="+", default=[8200]) + # P2PConnector coordinates of the prefiller — injected into decode requests. + p.add_argument( + "--p2p-connector-host", + type=str, + default="127.0.0.1", + help="Host of the prefiller's P2PConnector ZMQ socket", + ) + p.add_argument( + "--p2p-connector-port", + type=int, + default=7777, + help="Port of the prefiller's P2PConnector ZMQ socket", + ) + # P2PConnector coordinates of the decoder — injected into prefill requests + # so the prefiller's submit_store can resolve the peer to push KV to. + p.add_argument( + "--decoder-p2p-connector-host", + type=str, + default="127.0.0.1", + help="Host of the decoder's P2PConnector ZMQ socket", + ) + p.add_argument( + "--decoder-p2p-connector-port", + type=int, + default=7778, + help="Port of the decoder's P2PConnector ZMQ socket", + ) + p.add_argument( + "--decoder-first", + action="store_true", + help="Send decode request before prefill so decoder is already " + "waiting when KV blocks arrive (decoder-first mode)", + ) + args = p.parse_args() + if len(args.prefiller_hosts) != len(args.prefiller_ports): + raise ValueError("Prefiller host/port count mismatch") + if len(args.decoder_hosts) != len(args.decoder_ports): + raise ValueError("Decoder host/port count mismatch") + args.prefiller_instances = list(zip(args.prefiller_hosts, args.prefiller_ports)) + args.decoder_instances = list(zip(args.decoder_hosts, args.decoder_ports)) + return args + + +def _get_next(app, service: str): + if service == "prefill": + return app.state.prefill_clients[next(app.state.prefill_iterator)] + return app.state.decode_clients[next(app.state.decode_iterator)] + + +def _auth_headers(request_id: str) -> dict: + headers: dict = {"X-Request-Id": request_id} + api_key = os.environ.get("OPENAI_API_KEY", "") + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + return headers + + +async def _prefill(client_info, endpoint, req_data, request_id): + """Send a prefill-only request (max_tokens=1) to the prefiller.""" + data = req_data.copy() + data["kv_transfer_params"] = { + "decode": { + "kv_request_id": request_id, + }, + } + data["stream"] = False + data["max_tokens"] = 1 + data.pop("max_completion_tokens", None) + data.pop("stream_options", None) + data.pop("min_tokens", None) + data.pop("min_completion_tokens", None) + + headers = _auth_headers(request_id) + resp = await client_info["client"].post(endpoint, json=data, headers=headers) + resp.raise_for_status() + await resp.aread() + return resp + + +async def _stream_decode(client_info, endpoint, req_data, request_id): + headers = _auth_headers(request_id) + async with client_info["client"].stream( + "POST", endpoint, json=req_data, headers=headers + ) as resp: + resp.raise_for_status() + async for chunk in resp.aiter_bytes(): + yield chunk + + +async def _handle_completions(api: str, request: Request): + try: + req_data = await request.json() + request_id = str(uuid.uuid4()) + + prefill_client = _get_next(request.app, "prefill") + await _prefill(prefill_client, api, req_data, request_id) + + # Inject the prefiller's P2PConnector address so the decoder can pull + # KV blocks from it via the P2PConnector transport. + req_data["kv_transfer_params"] = { + "prefill": { + "kv_request_id": request_id, + "remote_host": global_args.p2p_connector_host, + "remote_port": global_args.p2p_connector_port, + }, + } + + decode_client = _get_next(request.app, "decode") + logger.debug("prefill=%s decode=%s", prefill_client, decode_client) + + async def generate(): + async for chunk in _stream_decode(decode_client, api, req_data, request_id): + yield chunk + + return StreamingResponse(generate(), media_type="application/json") + + except Exception as e: + import sys + import traceback + + print(f"Proxy error on {api}: {e}") + print("".join(traceback.format_exception(*sys.exc_info()))) + raise + + +async def _handle_completions_decoder_first(api: str, request: Request): + """Decoder-first mode: send decode request before prefill. + + The decoder establishes its request and starts polling for KV blocks + immediately. The prefill is then sent so the prefiller computes and + pushes blocks to the already-waiting decoder. + """ + try: + req_data = await request.json() + request_id = str(uuid.uuid4()) + + prefill_client = _get_next(request.app, "prefill") + decode_client = _get_next(request.app, "decode") + + decode_data = req_data.copy() + decode_data["kv_transfer_params"] = { + "prefill": { + "kv_request_id": request_id, + "remote_host": global_args.p2p_connector_host, + "remote_port": global_args.p2p_connector_port, + }, + } + + async def generate(): + queue: asyncio.Queue = asyncio.Queue() + + async def _run_decode(): + try: + async for chunk in _stream_decode( + decode_client, api, decode_data, request_id + ): + await queue.put(("data", chunk)) + except Exception as exc: + await queue.put(("error", exc)) + finally: + await queue.put(("done", None)) + + # 1. Start decode request — decoder is now waiting for KV blocks + asyncio.create_task(_run_decode()) + + # 2. Send prefill — blocks are computed and pushed to the decoder + try: + await _prefill(prefill_client, api, req_data, request_id) + except Exception as exc: + logger.warning("decoder-first: prefill failed: %s", exc) + + logger.debug( + "decoder-first: prefill done, streaming decode prefill=%s decode=%s", + prefill_client, + decode_client, + ) + + # 3. Stream the decode response + while True: + kind, value = await queue.get() + if kind == "done": + break + if kind == "error": + raise value # type: ignore[misc] + yield value + + return StreamingResponse(generate(), media_type="application/json") + + except Exception as e: + import sys + import traceback + + print(f"Proxy error on {api}: {e}") + print("".join(traceback.format_exception(*sys.exc_info()))) + raise + + +def _route_handler(api: str): + if global_args.decoder_first: + return lambda req: _handle_completions_decoder_first(api, req) + return lambda req: _handle_completions(api, req) + + +@app.post("/v1/completions") +async def completions(request: Request): + return await _route_handler("/completions")(request) + + +@app.post("/v1/chat/completions") +async def chat_completions(request: Request): + return await _route_handler("/chat/completions")(request) + + +@app.get("/healthcheck") +async def healthcheck(): + return { + "status": "ok", + "prefill_instances": len(app.state.prefill_clients), + "decode_instances": len(app.state.decode_clients), + } + + +if __name__ == "__main__": + global global_args + global_args = parse_args() + import uvicorn + + uvicorn.run(app, host=global_args.host, port=global_args.port) diff --git a/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh b/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh new file mode 100755 index 0000000000..82053ebfad --- /dev/null +++ b/tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh @@ -0,0 +1,312 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Accuracy test driver for the p2p connector +# (OffloadingConnector + TieringOffloadingSpec + p2p tier). +# +# Mirrors tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh: +# brings up N prefillers + M decoders on the local host, fronts them +# with p2p_connector_proxy.py, then runs the connector-agnostic +# test_accuracy.py (gsm8k via lm_eval) against the proxy. +# +# Knobs (env vars unless flagged otherwise): +# MODEL_NAMES space-separated model list (default: Llama-3.2-1B-Instruct) +# NUM_PREFILL_INSTANCES default 1 +# NUM_DECODE_INSTANCES default 1 +# PREFILLER_TP_SIZE default 1 +# DECODER_TP_SIZE default 1 +# GPU_MEMORY_UTILIZATION default 0.45 +# MAX_MODEL_LEN default 512 +# PREFILL_BLOCK_SIZE default 128 +# DECODE_BLOCK_SIZE default 128 +# CPU_BYTES default 209715200 (200 MB) +# VLLM_SERVE_EXTRA_ARGS comma-separated extra args for vllm serve +# --decoder-first toggle decoder-first proxy mode +# +# Examples: +# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh +# NUM_PREFILL_INSTANCES=2 NUM_DECODE_INSTANCES=2 \ +# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh +# bash tests/v1/kv_offload/tiering/p2p/run_accuracy_test.sh --decoder-first + +set -xe + +# --------------------------------------------------------------------------- +# Args +# --------------------------------------------------------------------------- +DECODER_FIRST="false" + +while [[ $# -gt 0 ]]; do + case $1 in + --decoder-first) + DECODER_FIRST="true" + shift 1 + ;; + *) + echo "Unknown option $1" + echo "Usage: $0 [--decoder-first]" + exit 1 + ;; + esac +done + +# --------------------------------------------------------------------------- +# Models +# --------------------------------------------------------------------------- +MODEL_NAMES=${MODEL_NAMES:-} +if [[ -n "$MODEL_NAMES" ]]; then + # shellcheck disable=SC2206 + MODELS=($MODEL_NAMES) +else + MODELS=( + "meta-llama/Llama-3.2-1B-Instruct" + ) +fi + +# --------------------------------------------------------------------------- +# Defaults +# --------------------------------------------------------------------------- +NUM_PREFILL_INSTANCES=${NUM_PREFILL_INSTANCES:-1} +NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} +PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} +DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} +GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.45} +MAX_MODEL_LEN=${MAX_MODEL_LEN:-512} +PREFILL_BLOCK_SIZE=${PREFILL_BLOCK_SIZE:-128} +DECODE_BLOCK_SIZE=${DECODE_BLOCK_SIZE:-128} +CPU_BYTES=${CPU_BYTES:-209715200} +VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} + +# Base ports — per-instance offsets layered on top. +PREFILL_HTTP_BASE=8100 +DECODE_HTTP_BASE=8200 +PREFILL_PD_BASE=7777 +DECODE_PD_BASE=$((PREFILL_PD_BASE + NUM_PREFILL_INSTANCES)) +PROXY_PORT=8192 +P2P_HOST=127.0.0.1 + +# --------------------------------------------------------------------------- +# Resolve repo root + venv (works in .venv and /workspace/venv pods) +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../../.." && pwd -P)}" + +if [[ -z "${VLLM_BIN:-}" ]]; then + if [[ -x "${GIT_ROOT}/.venv/bin/vllm" ]]; then + VLLM_BIN="${GIT_ROOT}/.venv/bin/vllm" + elif [[ -x "/workspace/venv/bin/vllm" ]]; then + VLLM_BIN="/workspace/venv/bin/vllm" + else + VLLM_BIN="$(command -v vllm)" + fi +fi +if [[ -z "${PYTHON_BIN:-}" ]]; then + if [[ -x "${GIT_ROOT}/.venv/bin/python" ]]; then + PYTHON_BIN="${GIT_ROOT}/.venv/bin/python" + elif [[ -x "/workspace/venv/bin/python" ]]; then + PYTHON_BIN="/workspace/venv/bin/python" + else + PYTHON_BIN="$(command -v python3 || command -v python)" + fi +fi +echo "Using vllm: ${VLLM_BIN}" +echo "Using python: ${PYTHON_BIN}" + +SMI_BIN=$(command -v nvidia-smi || command -v rocm-smi || echo "") + +# Trap SIGINT/SIGTERM/EXIT to kill background jobs. +trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +wait_for_server() { + local port=$1 + timeout 1200 bash -c " + until curl -s localhost:${port}/v1/completions > /dev/null; do + sleep 1 + done" && return 0 || return 1 +} + +cleanup_instances() { + echo "Cleaning up any running vLLM / proxy instances..." + pkill -f "vllm serve" || true + pkill -f "p2p_connector_proxy.py" || true + sleep 2 +} + +get_num_gpus() { + if [[ "$SMI_BIN" == *"nvidia"* ]]; then + $SMI_BIN --query-gpu=name --format=csv,noheader | wc -l + elif [[ "$SMI_BIN" == *"rocm"* ]]; then + $SMI_BIN -l | grep -c GPU + else + echo "1" + fi +} + +# Build the OffloadingConnector kv-transfer-config for a given PD port. +# Mirrors deploy_local.sh:131. +build_kv_config() { + local pd_port=$1 + printf '{"kv_connector":"OffloadingConnector","kv_role":"kv_both","kv_connector_extra_config":{"spec_name":"TieringOffloadingSpec","cpu_bytes_to_use":%s,"secondary_tiers":[{"type":"p2p","host":"%s","port":%s}]}}' \ + "${CPU_BYTES}" "${P2P_HOST}" "${pd_port}" +} + +# --------------------------------------------------------------------------- +# Per-model run +# --------------------------------------------------------------------------- +run_tests_for_model() { + local model_name=$1 + echo "================================" + echo "Testing model: $model_name" + echo " prefillers=${NUM_PREFILL_INSTANCES} (tp=${PREFILLER_TP_SIZE})" + echo " decoders=${NUM_DECODE_INSTANCES} (tp=${DECODER_TP_SIZE})" + echo " decoder_first=${DECODER_FIRST}" + echo "================================" + + PREFILL_HOSTS=() + PREFILL_PORTS=() + PREFILL_PD_PORTS=() + DECODE_HOSTS=() + DECODE_PORTS=() + DECODE_PD_PORTS=() + + local num_gpus + num_gpus=$(get_num_gpus) + local next_gpu=0 + + # ---- Prefillers ---- + for i in $(seq 0 $((NUM_PREFILL_INSTANCES-1))); do + local gpu_id=$((i * PREFILLER_TP_SIZE % num_gpus)) + local cuda_devs="${gpu_id}" + for (( j=1; j < PREFILLER_TP_SIZE; j++ )); do + cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))" + done + next_gpu=$(((gpu_id + PREFILLER_TP_SIZE) % num_gpus)) + + local http_port=$((PREFILL_HTTP_BASE + i)) + local pd_port=$((PREFILL_PD_BASE + i)) + local kv_cfg + kv_cfg=$(build_kv_config "${pd_port}") + + echo "Prefiller $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}" + + BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \ + PYTHONHASHSEED=42 \ + ${VLLM_BIN} serve ${model_name} \ + --port ${http_port} \ + --enforce-eager \ + --block-size ${PREFILL_BLOCK_SIZE} \ + --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \ + --max-model-len ${MAX_MODEL_LEN} \ + --tensor-parallel-size ${PREFILLER_TP_SIZE} \ + --kv-transfer-config '${kv_cfg}'" + + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" + for arg in "${extra_args[@]}"; do + BASE_CMD="${BASE_CMD} $arg" + done + fi + + eval "${BASE_CMD} &" + + PREFILL_HOSTS+=("${P2P_HOST}") + PREFILL_PORTS+=("${http_port}") + PREFILL_PD_PORTS+=("${pd_port}") + done + + # ---- Decoders ---- + for i in $(seq 0 $((NUM_DECODE_INSTANCES-1))); do + local gpu_id=$(((next_gpu + i * DECODER_TP_SIZE) % num_gpus)) + local cuda_devs="${gpu_id}" + for (( j=1; j < DECODER_TP_SIZE; j++ )); do + cuda_devs="${cuda_devs},$(((gpu_id + j) % num_gpus))" + done + + local http_port=$((DECODE_HTTP_BASE + i)) + local pd_port=$((DECODE_PD_BASE + i)) + local kv_cfg + kv_cfg=$(build_kv_config "${pd_port}") + + echo "Decoder $i: gpu=[${cuda_devs}] http=${http_port} pd=${pd_port}" + + BASE_CMD="CUDA_VISIBLE_DEVICES=${cuda_devs} \ + PYTHONHASHSEED=42 \ + ${VLLM_BIN} serve ${model_name} \ + --port ${http_port} \ + --enforce-eager \ + --block-size ${DECODE_BLOCK_SIZE} \ + --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \ + --max-model-len ${MAX_MODEL_LEN} \ + --tensor-parallel-size ${DECODER_TP_SIZE} \ + --kv-transfer-config '${kv_cfg}'" + + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" + for arg in "${extra_args[@]}"; do + BASE_CMD="${BASE_CMD} $arg" + done + fi + + eval "${BASE_CMD} &" + + DECODE_HOSTS+=("${P2P_HOST}") + DECODE_PORTS+=("${http_port}") + DECODE_PD_PORTS+=("${pd_port}") + done + + # ---- Wait for HTTP readiness ---- + for port in "${PREFILL_PORTS[@]}"; do + echo "Waiting for prefill instance on port $port to start..." + wait_for_server "$port" + done + for port in "${DECODE_PORTS[@]}"; do + echo "Waiting for decode instance on port $port to start..." + wait_for_server "$port" + done + + # ---- Proxy ---- + # The proxy currently advertises a single prefiller PD address to decoders. + # For the 1xM and matched NxM common cases the first prefiller's PD coords + # are the right pick; multi-prefiller PD round-robin is a follow-up. + PROXY_CMD="${PYTHON_BIN} ${SCRIPT_DIR}/p2p_connector_proxy.py \ + --port ${PROXY_PORT} \ + --host ${P2P_HOST} \ + --prefiller-hosts ${PREFILL_HOSTS[*]} \ + --prefiller-ports ${PREFILL_PORTS[*]} \ + --decoder-hosts ${DECODE_HOSTS[*]} \ + --decoder-ports ${DECODE_PORTS[*]} \ + --p2p-connector-host ${P2P_HOST} \ + --p2p-connector-port ${PREFILL_PD_PORTS[0]} \ + --decoder-p2p-connector-host ${P2P_HOST} \ + --decoder-p2p-connector-port ${DECODE_PD_PORTS[0]}" + + if [[ "${DECODER_FIRST}" == "true" ]]; then + PROXY_CMD="${PROXY_CMD} --decoder-first" + fi + + echo "Starting proxy: ${PROXY_CMD}" + eval "${PROXY_CMD} &" + + sleep 5 + + # ---- Run accuracy test (reused from nixl_integration) ---- + echo "Running tests for $model_name" + TEST_MODEL=$model_name "${PYTHON_BIN}" -m pytest -s -x \ + "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/test_accuracy.py" + + cleanup_instances + sleep 3 +} + +# --------------------------------------------------------------------------- +# Drive +# --------------------------------------------------------------------------- +for model in "${MODELS[@]}"; do + run_tests_for_model "$model" +done + +echo "All tests completed!" diff --git a/tests/v1/kv_offload/tiering/p2p/test_data_transport.py b/tests/v1/kv_offload/tiering/p2p/test_data_transport.py new file mode 100644 index 0000000000..d515e102e2 --- /dev/null +++ b/tests/v1/kv_offload/tiering/p2p/test_data_transport.py @@ -0,0 +1,330 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for DataTransport base class and NixlTransport.""" + +from __future__ import annotations + +import ctypes +from unittest.mock import MagicMock, patch + +import numpy as np + +from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult +from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport + +# --------------------------------------------------------------------------- +# DataTransport base class tests +# --------------------------------------------------------------------------- + + +class TestDataTransportBase: + """Tests for the DataTransport abstract base properties.""" + + def _make_view(self, num_blocks: int = 8, block_len: int = 1024) -> memoryview: + """Create a memoryview with the given shape.""" + buf = np.zeros((num_blocks, block_len), dtype=np.uint8) + return memoryview(buf) + + def test_properties(self): + """base_addr, num_blocks, block_len are set from memoryview shape.""" + view = self._make_view(num_blocks=4, block_len=2048) + + # Use NixlTransport (concrete) with NIXL mocked away + with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None): + transport = NixlTransport("test:1", view) + + assert transport.num_blocks == 4 + assert transport.block_len == 2048 + assert transport.base_addr == ctypes.addressof(ctypes.c_char.from_buffer(view)) + + def test_config_fingerprint_empty_when_no_fields(self): + """No config fields → empty fingerprint.""" + view = self._make_view() + with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None): + transport = NixlTransport("test:1", view, config_fields=None) + assert transport.config_fingerprint == "" + + def test_config_fingerprint_deterministic(self): + """Same config fields → same fingerprint.""" + view = self._make_view() + fields = {"model": "llama", "dtype": "float16", "block_size_factor": 1} + with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None): + t1 = NixlTransport("test:1", view, config_fields=fields) + t2 = NixlTransport("test:2", view, config_fields=fields) + assert t1.config_fingerprint == t2.config_fingerprint + assert len(t1.config_fingerprint) == 16 + + def test_config_fingerprint_differs_for_different_fields(self): + """Different config fields → different fingerprint.""" + view = self._make_view() + with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None): + t1 = NixlTransport("test:1", view, config_fields={"model": "a"}) + t2 = NixlTransport("test:2", view, config_fields={"model": "b"}) + assert t1.config_fingerprint != t2.config_fingerprint + + +# --------------------------------------------------------------------------- +# NixlTransport tests (with mocked NIXL agent) +# --------------------------------------------------------------------------- + + +class TestNixlTransportWithMockedAgent: + """Tests for NixlTransport logic with a mocked NIXL agent.""" + + def _make_transport(self) -> NixlTransport: + """Create a NixlTransport with mocked NIXL internals.""" + view = memoryview(np.zeros((8, 1024), dtype=np.uint8)) + + with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None): + transport = NixlTransport("test:1", view) + + # Manually set up a mock agent after construction + agent = MagicMock() + agent.add_remote_agent.return_value = "nixl-peer-name" + agent.get_xfer_descs.return_value = MagicMock() + agent.prep_xfer_dlist.return_value = MagicMock() + agent.make_prepped_xfer.return_value = MagicMock(name="handle") + agent.transfer.return_value = None + agent.check_xfer_state.return_value = "PROC" + agent.get_agent_metadata.return_value = b"test-metadata" + + transport._agent = agent + transport._local_dlist = MagicMock() + return transport + + def test_available_false_without_nixl(self): + """Without NIXL installed, available is False.""" + view = memoryview(np.zeros((4, 512), dtype=np.uint8)) + with patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", None): + transport = NixlTransport("test:1", view) + assert transport.available is False + + def test_available_true_with_agent(self): + transport = self._make_transport() + assert transport.available is True + + def test_get_agent_metadata(self): + transport = self._make_transport() + assert transport.get_agent_metadata() == b"test-metadata" + + def test_write_blocks_returns_none_for_unknown_peer(self): + """write_blocks returns None if peer not registered.""" + transport = self._make_transport() + result = transport.write_blocks("unknown:1", [0, 1], [2, 3]) + assert result is None + + def test_write_blocks_returns_transfer_id(self): + """write_blocks returns an integer transfer_id on success.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0, 1], [2, 3]) + assert tid is not None + assert isinstance(tid, int) + + def test_write_blocks_increments_transfer_id(self): + """Each write_blocks call gets a unique transfer_id.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid1 = transport.write_blocks("peer:1", [0], [1]) + tid2 = transport.write_blocks("peer:1", [2], [3]) + assert tid1 != tid2 + + def test_poll_empty_when_no_inflight(self): + """poll returns empty when nothing is inflight.""" + transport = self._make_transport() + result = transport.poll() + assert result == PollResult(done=(), failed=()) + + def test_poll_returns_done_when_transfer_completes(self): + """Completed transfer appears in poll().done.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0], [1]) + + # Simulate completion + transport._agent.check_xfer_state.return_value = "DONE" + result = transport.poll() + + assert tid in result.done + assert result.failed == () + # Handle released + transport._agent.release_xfer_handle.assert_called() + + def test_poll_returns_failed_for_error_state(self): + """Transfer in error state appears in poll().failed.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0], [1]) + + transport._agent.check_xfer_state.return_value = "ERR" + result = transport.poll() + + assert result.done == () + assert tid in result.failed + + def test_poll_ignores_in_progress(self): + """Transfers in PROC/PEND state stay inflight.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + transport.write_blocks("peer:1", [0], [1]) + + transport._agent.check_xfer_state.return_value = "PROC" + result = transport.poll() + assert result.done == () + assert result.failed == () + + transport._agent.check_xfer_state.return_value = "PEND" + result = transport.poll() + assert result.done == () + assert result.failed == () + + def test_cancel_removes_inflight(self): + """cancel removes transfers and releases handles.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0], [1]) + assert tid in transport._inflight + + result = transport.cancel([tid]) + assert result == [] + assert tid not in transport._inflight + transport._agent.release_xfer_handle.assert_called() + + def test_cancel_ignores_unknown_ids(self): + """cancel with unknown IDs doesn't crash.""" + transport = self._make_transport() + assert transport.cancel([999, 1000]) == [] + assert transport.cancel([999, 1000], mode="wait") == [] + + def test_cancel_wait_release_succeeds(self): + """wait-mode cancel that succeeds pops the entry and returns [].""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0], [1]) + assert tid in transport._inflight + + result = transport.cancel([tid], mode="wait") + assert result == [] + assert tid not in transport._inflight + transport._agent.release_xfer_handle.assert_called_once() + + def test_cancel_wait_release_raises(self): + """wait-mode cancel keeps the entry and returns the tid on raise.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0], [1]) + transport._agent.release_xfer_handle.side_effect = RuntimeError( + "NIXL_ERR_REPOST_ACTIVE" + ) + + result = transport.cancel([tid], mode="wait") + assert result == [tid] + assert tid in transport._inflight + + def test_cancel_wait_then_poll_completes(self): + """A wait-cancel that left a tid pending later completes via poll.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + + tid = transport.write_blocks("peer:1", [0], [1]) + transport._agent.release_xfer_handle.side_effect = RuntimeError("busy") + assert transport.cancel([tid], mode="wait") == [tid] + assert tid in transport._inflight + + transport._agent.release_xfer_handle.side_effect = None + transport._agent.check_xfer_state.return_value = "DONE" + + result = transport.poll() + assert tid in result.done + assert tid not in transport._inflight + + def test_add_and_remove_remote_peer(self): + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + assert "peer:1" in transport._remote_dlists + + transport.remove_remote_peer("peer:1") + assert "peer:1" not in transport._remote_dlists + transport._agent.release_dlist_handle.assert_called() + transport._agent.remove_remote_agent.assert_called() + + def test_close_releases_everything(self): + """close releases all handles and clears state.""" + transport = self._make_transport() + transport.add_remote_peer("peer:1", b"meta", 0x1000, 8, 1024) + transport.write_blocks("peer:1", [0], [1]) + + transport.close() + assert transport._agent is None + assert transport._inflight == {} + assert transport._remote_dlists == {} + + +# --------------------------------------------------------------------------- +# NIXL agent-config selection +# --------------------------------------------------------------------------- + + +class TestNixlAgentConfigSelection: + """Tests that backends/num_threads pick the right nixl_agent_config call. + + Mirrors the conditional in + vllm/distributed/kv_transfer/kv_connector/v1/nixl/base_worker.py:325-329. + """ + + def _make_view(self) -> memoryview: + return memoryview(np.zeros((4, 512), dtype=np.uint8)) + + def test_non_ucx_backends_passes_backends_kwarg(self): + """When any non-UCX backend is requested, pass backends + telemetry.""" + agent_cls = MagicMock() + config_fn = MagicMock(return_value=MagicMock(name="cfg")) + with ( + patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls), + patch( + "vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn + ), + ): + NixlTransport("test:1", self._make_view(), backends=["MOONCAKE"]) + + config_fn.assert_called_once_with(backends=["MOONCAKE"], capture_telemetry=True) + # num_threads must NOT be passed on the non-UCX branch. + assert "num_threads" not in config_fn.call_args.kwargs + + def test_ucx_only_passes_num_threads(self): + """UCX-only configuration passes num_threads + telemetry, no backends.""" + agent_cls = MagicMock() + config_fn = MagicMock(return_value=MagicMock(name="cfg")) + with ( + patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls), + patch( + "vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn + ), + ): + NixlTransport("test:1", self._make_view(), num_threads=8) + + config_fn.assert_called_once_with(num_threads=8, capture_telemetry=True) + assert "backends" not in config_fn.call_args.kwargs + + def test_default_backends_is_ucx_only(self): + """No backends arg → defaults to UCX-only branch.""" + agent_cls = MagicMock() + config_fn = MagicMock(return_value=MagicMock(name="cfg")) + with ( + patch("vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgent", agent_cls), + patch( + "vllm.v1.kv_offload.tiering.p2p.data.nixl._NixlAgentConfig", config_fn + ), + ): + NixlTransport("test:1", self._make_view()) + + # Default num_threads=4, no backends kwarg. + config_fn.assert_called_once_with(num_threads=4, capture_telemetry=True) diff --git a/tests/v1/kv_offload/tiering/p2p/test_manager.py b/tests/v1/kv_offload/tiering/p2p/test_manager.py new file mode 100644 index 0000000000..868b62587f --- /dev/null +++ b/tests/v1/kv_offload/tiering/p2p/test_manager.py @@ -0,0 +1,1370 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for P2PSecondaryTierManager. + +Tests the manager's job routing, session lifecycle, and result collection +using fake transport and session objects. +""" + +from __future__ import annotations + +import time +from types import SimpleNamespace + +import numpy as np + +from vllm.v1.kv_offload.base import LookupResult, ReqContext +from vllm.v1.kv_offload.tiering.base import JobMetadata, JobResult +from vllm.v1.kv_offload.tiering.p2p import manager as manager_module +from vllm.v1.kv_offload.tiering.p2p.manager import ( + _UNBOUND_STORE_TIMEOUT_S, + P2PSecondaryTierManager, +) +from vllm.v1.kv_offload.tiering.p2p.session import ( + LoadResult, + SessionPollResult, + StoreResult, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _prefill_kv_params( + remote_host: str = "10.0.0.1", + remote_port: int = 8000, + kv_request_id: str = "req-1", +) -> dict: + """Decoder-side kv_transfer_params: ``prefill`` sub-dict carries + kv_request_id + remote_host + remote_port.""" + return { + "prefill": { + "kv_request_id": kv_request_id, + "remote_host": remote_host, + "remote_port": remote_port, + }, + } + + +def _decode_kv_params(kv_request_id: str = "req-1") -> dict: + """Prefiller-side kv_transfer_params: ``decode`` sub-dict carries + kv_request_id only.""" + return {"decode": {"kv_request_id": kv_request_id}} + + +def _req_context(kv_params: dict | None = None) -> ReqContext: + return ReqContext(req_id="test", kv_transfer_params=kv_params) + + +def _job_metadata( + job_id: int, + keys: list[bytes] | None = None, + block_ids: list[int] | None = None, + kv_params: dict | None = None, +) -> JobMetadata: + if keys is None: + keys = [b"key1"] + if block_ids is None: + block_ids = list(range(len(keys))) + return JobMetadata( + job_id=job_id, + keys=keys, + block_ids=np.array(block_ids), + is_promotion=False, + req_context=_req_context(kv_params), + ) + + +def _make_manager() -> P2PSecondaryTierManager: + """Create a manager with stubbed __init__.""" + mgr = P2PSecondaryTierManager.__new__(P2PSecondaryTierManager) + mgr._local_id = "127.0.0.1:7777" + mgr._finished_jobs = [] + mgr._failed_req_ids = set() + mgr._sessions = {} + mgr._kv_to_session = {} + mgr._unbound_stores = {} + return mgr + + +# --------------------------------------------------------------------------- +# Tests for _remote_id_from_params +# --------------------------------------------------------------------------- + + +class TestRemoteIdFromParams: + def test_valid_params(self): + result = P2PSecondaryTierManager._remote_id_from_params( + {"remote_host": "10.0.0.1", "remote_port": 8000} + ) + assert result == "10.0.0.1:8000" + + def test_missing_host(self): + result = P2PSecondaryTierManager._remote_id_from_params({"remote_port": 8000}) + assert result is None + + def test_missing_port(self): + result = P2PSecondaryTierManager._remote_id_from_params( + {"remote_host": "10.0.0.1"} + ) + assert result is None + + def test_empty_dict(self): + result = P2PSecondaryTierManager._remote_id_from_params({}) + assert result is None + + +# --------------------------------------------------------------------------- +# Tests for lookup +# --------------------------------------------------------------------------- + + +class TestLookup: + def test_lookup_returns_miss_without_kv_params(self): + mgr = _make_manager() + ctx = _req_context(kv_params=None) + assert mgr.lookup(b"key", ctx) is LookupResult.MISS + + def test_lookup_returns_miss_without_required_fields(self): + mgr = _make_manager() + ctx = _req_context(kv_params={"prefill": {"remote_host": "x"}}) + assert mgr.lookup(b"key", ctx) is LookupResult.MISS + + def test_lookup_returns_hit_for_valid_request(self): + mgr = _make_manager() + ctx = _req_context(kv_params=_prefill_kv_params()) + assert mgr.lookup(b"key", ctx) is LookupResult.HIT + + def test_lookup_returns_miss_for_failed_request(self): + mgr = _make_manager() + mgr._failed_req_ids.add("req-1") + ctx = _req_context(kv_params=_prefill_kv_params(kv_request_id="req-1")) + assert mgr.lookup(b"key", ctx) is LookupResult.MISS + + def test_lookup_returns_hit_for_different_request_id(self): + mgr = _make_manager() + mgr._failed_req_ids.add("req-1") + ctx = _req_context(kv_params=_prefill_kv_params(kv_request_id="req-2")) + assert mgr.lookup(b"key", ctx) is LookupResult.HIT + + def test_lookup_returns_miss_without_prefill_key(self): + """No ``prefill`` sub-dict means the request was not routed for + remote prefill — local prefill should run instead, so lookup() + returns MISS even when a stale ``decode`` block is present.""" + mgr = _make_manager() + ctx = _req_context(kv_params=_decode_kv_params()) + assert mgr.lookup(b"key", ctx) is LookupResult.MISS + + +# --------------------------------------------------------------------------- +# Tests for submit_store +# --------------------------------------------------------------------------- + + +class TestSubmitStore: + def test_no_decode_succeeds_immediately(self): + """Without a ``decode`` block, job succeeds immediately.""" + mgr = _make_manager() + job = _job_metadata(job_id=1, kv_params={}) + mgr.submit_store(job) + assert mgr._finished_jobs == [JobResult(job_id=1, success=True)] + + def test_missing_kv_request_id_fails(self): + """Missing kv_request_id inside ``decode`` fails the job.""" + mgr = _make_manager() + params: dict = {"decode": {}} + job = _job_metadata(job_id=1, kv_params=params) + mgr.submit_store(job) + assert mgr._finished_jobs == [JobResult(job_id=1, success=False)] + + def test_no_binding_yet_parks_in_unbound_stores(self): + """submit_store without a bound session buffers the batch keyed + by kv_request_id; no session is pre-created (peer_id is unknown + to the producer at store time).""" + mgr = _make_manager() + job = _job_metadata( + job_id=1, + keys=[b"k1", b"k2"], + block_ids=[3, 4], + kv_params=_decode_kv_params(kv_request_id="req-1"), + ) + mgr.submit_store(job) + + assert mgr._sessions == {} + assert mgr._finished_jobs == [] + batches = mgr._unbound_stores["req-1"] + assert len(batches) == 1 + assert ( + batches[0].job_id, + list(batches[0].keys), + list(batches[0].block_ids), + ) == ( + 1, + [b"k1", b"k2"], + [3, 4], + ) + + def test_routes_to_bound_session(self): + """If a session has already received FetchMsg for this + kv_request_id (so _kv_to_session is populated), submit_store + forwards directly to that session rather than re-buffering.""" + mgr = _make_manager() + bound = _FakeSession(peer_id="10.0.0.1:8000", connected=True) + mgr._kv_to_session["req-1"] = bound # type: ignore[assignment] + # Note: _sessions is intentionally untouched — this test isolates + # the kv_request_id → session fast path. + job = _job_metadata( + job_id=7, + keys=[b"k1", b"k2"], + block_ids=[3, 4], + kv_params=_decode_kv_params(kv_request_id="req-1"), + ) + mgr.submit_store(job) + + assert len(bound.stores_added) == 1 + kv_req_id, keys, _, job_id = bound.stores_added[0] + assert (kv_req_id, keys, job_id) == ("req-1", [b"k1", b"k2"], 7) + assert mgr._unbound_stores == {} + assert mgr._finished_jobs == [] + + def test_extra_top_level_keys_are_ignored(self): + """Producer-side kv_transfer_params should not pre-create a + session even when a stale caller still passes a top-level + ``remote_host``/``remote_port`` next to ``decode``.""" + mgr = _make_manager() + params = _decode_kv_params() + params["remote_host"] = "stale" + params["remote_port"] = 12345 + job = _job_metadata(job_id=1, kv_params=params) + mgr.submit_store(job) + # No session pre-created, no peer-keyed state. + assert mgr._sessions == {} + assert "req-1" in mgr._unbound_stores + + +# --------------------------------------------------------------------------- +# Tests for submit_load +# --------------------------------------------------------------------------- + + +class TestSubmitLoad: + def test_missing_params_fails(self): + """Missing required kv_params fields fails the job.""" + mgr = _make_manager() + job = _job_metadata(job_id=1, kv_params={}) + mgr.submit_load(job) + assert mgr._finished_jobs == [JobResult(job_id=1, success=False)] + + def test_empty_keys_succeeds_immediately(self): + """Empty key list succeeds immediately.""" + mgr = _make_manager() + job = _job_metadata( + job_id=1, keys=[], block_ids=[], kv_params=_prefill_kv_params() + ) + mgr.submit_load(job) + assert mgr._finished_jobs == [JobResult(job_id=1, success=True)] + + def test_no_session_fails(self): + """No session for peer fails and marks request failed.""" + mgr = _make_manager() + job = _job_metadata(job_id=1, kv_params=_prefill_kv_params()) + mgr.submit_load(job) + assert mgr._finished_jobs == [JobResult(job_id=1, success=False)] + assert "req-1" in mgr._failed_req_ids + + def test_happy_path_with_active_session(self): + """When the peer's session exists, submit_load forwards to + session.request_blocks and does NOT add a finished result yet.""" + mgr = _make_manager() + peer_id = "10.0.0.1:8000" + existing = _FakeSession(peer_id=peer_id, connected=True) + mgr._sessions[peer_id] = existing # type: ignore[assignment] + job = _job_metadata( + job_id=42, + keys=[b"k1", b"k2"], + block_ids=[5, 6], + kv_params=_prefill_kv_params(kv_request_id="req-42"), + ) + mgr.submit_load(job) + + assert existing.requests == [(42, "req-42")] + assert mgr._finished_jobs == [] + assert "req-42" not in mgr._failed_req_ids + + +# --------------------------------------------------------------------------- +# Tests for on_request_finished +# --------------------------------------------------------------------------- + + +class TestOnRequestFinished: + def _make_with_failed(self) -> P2PSecondaryTierManager: + mgr = _make_manager() + mgr._failed_req_ids = {"req-1"} + return mgr + + def test_prunes_failed_req_ids(self): + mgr = self._make_with_failed() + ctx = _req_context(kv_params=_prefill_kv_params(kv_request_id="req-1")) + mgr.on_request_finished(ctx) + assert "req-1" not in mgr._failed_req_ids + + def test_no_kv_params_does_nothing(self): + mgr = self._make_with_failed() + ctx = _req_context(kv_params=None) + mgr.on_request_finished(ctx) + assert "req-1" in mgr._failed_req_ids + + def test_no_kv_request_id_does_nothing(self): + mgr = self._make_with_failed() + ctx = _req_context(kv_params={"remote_host": "x", "remote_port": 1}) + mgr.on_request_finished(ctx) + assert "req-1" in mgr._failed_req_ids + + def test_decoder_side_calls_session_finish_request(self): + """Decoder-side finish (``prefill`` set) still routes via peer_id + because the consumer addresses the producer it loaded from. The + session's finish_request cancels the client-role load.""" + mgr = _make_manager() + peer_id = "10.0.0.1:8000" + session = _FakeSession(peer_id=peer_id) + mgr._sessions[peer_id] = session + ctx = _req_context(kv_params=_prefill_kv_params(kv_request_id="req-1")) + mgr.on_request_finished(ctx) + assert session.finishes == ["req-1"] + + def test_prefiller_bound_id_routes_via_kv_to_session(self): + """Prefiller-side finish for an id whose session is already bound + (FetchMsg received) routes via _kv_to_session and pops the entry.""" + mgr = _make_manager() + bound = _FakeSession(peer_id="some-peer:1", connected=True) + mgr._kv_to_session["req-1"] = bound # type: ignore[assignment] + ctx = _req_context(kv_params=_decode_kv_params(kv_request_id="req-1")) + mgr.on_request_finished(ctx) + assert bound.finishes == ["req-1"] + assert "req-1" not in mgr._kv_to_session + + def test_prefiller_unbound_id_leaves_batches_parked(self): + """Prefiller-side finish for an id with parked unbound batches + and no session binding is a no-op on `_unbound_stores`. The + parked batches survive until a peer fetches them or the + `_reap_unbound_stores` timeout fires — `on_request_finished` + must not evict them.""" + from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch + + mgr = _make_manager() + mgr._unbound_stores["req-1"] = [ + _UnboundStoreBatch(job_id=10, keys=[b"k"], block_ids=[0]), + _UnboundStoreBatch(job_id=11, keys=[b"k2"], block_ids=[1]), + ] + ctx = _req_context(kv_params=_decode_kv_params(kv_request_id="req-1")) + mgr.on_request_finished(ctx) + assert "req-1" in mgr._unbound_stores + assert [b.job_id for b in mgr._unbound_stores["req-1"]] == [10, 11] + outcomes = {(r.job_id, r.success) for r in mgr._finished_jobs} + assert (10, False) not in outcomes + assert (11, False) not in outcomes + + +# --------------------------------------------------------------------------- +# Tests for get_finished_jobs +# --------------------------------------------------------------------------- + + +class _FakeServerHalf: + def __init__(self) -> None: + self._inflight: dict[int, object] = {} + + +class _FakeClientHalf: + def __init__(self) -> None: + self._inbound: dict[int, object] = {} + + +class _FakeSession: + """Fake bidirectional session that returns canned poll() results.""" + + def __init__( + self, + peer_id: str = "fake:1", + alive: bool = True, + connected: bool = True, + loads: list[LoadResult] | None = None, + stores: list[StoreResult] | None = None, + new_fetch_ids: list[str] | None = None, + close_loads: list[tuple[int, str]] | None = None, + close_stores: list[int] | None = None, + ) -> None: + self.peer_id = peer_id + self.alive = alive + self.connected = connected + self.ready = True + self._loads = loads or [] + self._stores = stores or [] + self._new_fetch_ids = new_fetch_ids or [] + self._close_loads = close_loads or [] + self._close_stores = close_stores or [] + self.requests: list[tuple[int, str]] = [] + self.stores_added: list[tuple[str, list, object, int]] = [] + self.attached: list[object] = [] + self.finishes: list[str] = [] + # Mirror P2PSession._server._inflight (transfer_id → handle) and + # P2PSession._client._inbound for the shutdown-drain and drain_jobs + # paths. Tests populate _server._inflight when needed. + self._server = _FakeServerHalf() + self._client = _FakeClientHalf() + + def poll(self): + result = SessionPollResult( + loads=self._loads, + stores=self._stores, + new_fetch_ids=self._new_fetch_ids, + ) + self._loads = [] + self._stores = [] + self._new_fetch_ids = [] + return result + + def request_blocks(self, job_id, kv_request_id, keys, block_ids): + self.requests.append((job_id, kv_request_id)) + + def add_stored_blocks(self, kv_request_id, keys, block_ids, job_id): + self.stores_added.append((kv_request_id, list(keys), block_ids, job_id)) + + def attach_connection(self, conn): + self.attached.append(conn) + self.connected = True + + def finish_request(self, kv_request_id): + self.finishes.append(kv_request_id) + + def close(self): + return self._close_loads, self._close_stores + + +class TestGetFinished: + def _make(self) -> P2PSecondaryTierManager: + mgr = _make_manager() + mgr._finished_jobs = [ + JobResult(job_id=1, success=True), + JobResult(job_id=2, success=False), + ] + + class FakeControl: + def poll(self): + return [] + + mgr._control = FakeControl() # type: ignore[assignment] + mgr._data = None # type: ignore[assignment] + return mgr + + def test_drains_finished_jobs(self): + """get_finished_jobs returns and clears accumulated results.""" + mgr = self._make() + results = list(mgr.get_finished_jobs()) + assert len(results) == 2 + assert JobResult(job_id=1, success=True) in results + assert JobResult(job_id=2, success=False) in results + # Second call returns empty. + assert list(mgr.get_finished_jobs()) == [] + + def test_reaps_dead_sessions(self): + """Dead connected sessions are removed and their pending jobs failed.""" + + class FakeData: + def remove_remote_peer(self, pid): + pass + + mgr = self._make() + mgr._data = FakeData() # type: ignore[assignment] + dead = _FakeSession( + peer_id="dead:1234", + alive=False, + connected=True, + close_loads=[(20, "req-load")], + close_stores=[10, 11], + ) + mgr._sessions["dead:1234"] = dead # type: ignore[assignment] + + results = list(mgr.get_finished_jobs()) + # 2 baseline + 1 failed load + 2 failed stores + assert len(results) == 5 + assert JobResult(job_id=10, success=False) in results + assert JobResult(job_id=11, success=False) in results + assert JobResult(job_id=20, success=False) in results + assert "dead:1234" not in mgr._sessions + assert "req-load" in mgr._failed_req_ids + + def test_unbound_store_kept_within_timeout(self): + """Recently-parked unbound stores stay across a poll.""" + mgr = self._make() + from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch + + mgr._unbound_stores["req-fresh"] = [ + _UnboundStoreBatch(job_id=99, keys=[b"k"], block_ids=[0]) + ] + list(mgr.get_finished_jobs()) + assert "req-fresh" in mgr._unbound_stores + + def test_unbound_store_reaped_after_timeout(self): + """Unbound stores past _UNBOUND_STORE_TIMEOUT_S surface as failed + and their kv_request_id lands in _failed_req_ids so a late + FetchMsg/lookup doesn't try to satisfy them.""" + from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch + + mgr = self._make() + stale = _UnboundStoreBatch(job_id=10, keys=[b"k"], block_ids=[0]) + # Backdate the submission so the head batch is past the deadline. + stale.submitted_at = time.monotonic() - _UNBOUND_STORE_TIMEOUT_S - 1.0 + mgr._unbound_stores["req-stale"] = [ + stale, + _UnboundStoreBatch(job_id=11, keys=[b"k2"], block_ids=[1]), + ] + + results = list(mgr.get_finished_jobs()) + + assert "req-stale" not in mgr._unbound_stores + # 2 baseline + 2 buffered stores + assert JobResult(job_id=10, success=False) in results + assert JobResult(job_id=11, success=False) in results + assert "req-stale" in mgr._failed_req_ids + + def test_submit_store_parks_unbound_batch(self): + """submit_store on an unbound id appends a batch with a fresh + submitted_at stamp so the unbound-store sweep can age it out.""" + mgr = _make_manager() + job = _job_metadata( + job_id=1, kv_params=_decode_kv_params(kv_request_id="req-1") + ) + before = time.monotonic() + mgr.submit_store(job) + after = time.monotonic() + batches = mgr._unbound_stores["req-1"] + assert len(batches) == 1 + assert before <= batches[0].submitted_at <= after + + +# --------------------------------------------------------------------------- +# has_pending_work +# --------------------------------------------------------------------------- + + +class TestHasPendingWork: + """has_pending_work() must always return True so the engine keeps + ticking the offload pipeline — that's the only thread driving + _control.poll() (incoming peer connects) and session.poll() + (incoming fetch messages on existing sessions).""" + + def test_returns_true_unconditionally_to_keep_engine_ticking(self): + """Even with no sessions and no jobs, has_pending_work() returns + True so the engine keeps calling get_finished_jobs(), which is + what drives _control.poll() for inbound peer connects.""" + mgr = _make_manager() + assert mgr.has_pending_work() is True + + def test_returns_true_even_when_sessions_present(self): + """The result is the same regardless of session state — there is + no 'idle' branch.""" + mgr = _make_manager() + mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # type: ignore[assignment] + assert mgr.has_pending_work() is True + + +# --------------------------------------------------------------------------- +# Shutdown drain +# --------------------------------------------------------------------------- + + +class _ShutdownFakeData: + """Fake DataTransport that records cancel/poll/close calls and + drives the wait-cancel loop with a scriptable `still` queue.""" + + def __init__(self, still_queue: list[list[int]] | None = None) -> None: + # Each list in still_queue is the set of ids the next + # cancel(mode="wait") should report as still inflight. The last + # entry repeats once exhausted. + self._still_queue = list(still_queue) if still_queue else [[]] + self.cancel_calls: list[tuple[list[int], str]] = [] + self.poll_calls: int = 0 + self.close_calls: int = 0 + + def cancel(self, transfer_ids, mode: str = "immediate") -> list[int]: + ids = list(transfer_ids) + self.cancel_calls.append((ids, mode)) + if mode == "wait": + if len(self._still_queue) > 1: + return list(self._still_queue.pop(0)) + return list(self._still_queue[0]) + return [] + + def poll(self): + self.poll_calls += 1 + + class _Empty: + done: list[int] = [] + failed: list[int] = [] + + return _Empty() + + def close(self) -> None: + self.close_calls += 1 + + +class _ShutdownFakeControl: + def __init__(self) -> None: + self.close_calls: int = 0 + + def close(self) -> None: + self.close_calls += 1 + + +class TestShutdownDrain: + """shutdown() drains inflight transfers via cancel(mode='wait') + before calling _data.close(), with a 3s deadline fallback to + cancel(mode='immediate').""" + + def _prep( + self, + still_queue: list[list[int]] | None = None, + inflight_ids: list[int] | None = None, + ) -> tuple[P2PSecondaryTierManager, _ShutdownFakeData, _ShutdownFakeControl]: + mgr = _make_manager() + data = _ShutdownFakeData(still_queue=still_queue) + control = _ShutdownFakeControl() + mgr._data = data # type: ignore[assignment] + mgr._control = control # type: ignore[assignment] + if inflight_ids: + session = _FakeSession(peer_id="peer:1", connected=True) + session._server._inflight = {tid: object() for tid in inflight_ids} + mgr._sessions["peer:1"] = session # type: ignore[assignment] + return mgr, data, control + + def test_shutdown_drains_inflight_via_wait_cancel(self): + # First cancel(wait) returns the input still inflight; second returns []. + mgr, data, control = self._prep( + still_queue=[[42, 43], []], + inflight_ids=[42, 43], + ) + mgr.shutdown() + + wait_calls = [c for c in data.cancel_calls if c[1] == "wait"] + immediate_calls = [c for c in data.cancel_calls if c[1] == "immediate"] + assert len(wait_calls) >= 1 + assert wait_calls[0][0] == [42, 43] + assert immediate_calls == [] + # poll() was driven between cancel attempts. + assert data.poll_calls >= 1 + # _data and _control were closed exactly once each, after the drain. + assert data.close_calls == 1 + assert control.close_calls == 1 + + def test_shutdown_force_cancels_after_timeout(self, monkeypatch): + # Drain never completes — wait-cancel keeps returning the inflight set. + # Use a synthetic clock so the test does not depend on real wallclock + # being able to advance in <50ms on a loaded CI node: + # call 1 (deadline calc): 100.0 -> deadline = 100.05 + # call 2 (loop predicate): 100.0 -> enters loop, one wait-cancel + # call 3 (loop predicate): 100.06 -> exits, force-cancel runs + monkeypatch.setattr(manager_module, "_SHUTDOWN_DRAIN_TIMEOUT_S", 0.05) + monkeypatch.setattr(manager_module, "_DRAIN_SLEEP_S", 0.0) + times = iter([100.0, 100.0, 100.06]) + # Patch via a fake module on `manager_module.time` so we do not mutate + # the global `time` module — other code in the process (e.g. the + # buildkite test collector's pytest_runtest_logreport hook) calls + # time.monotonic() before monkeypatch teardown. + fake_time = SimpleNamespace(monotonic=lambda: next(times), sleep=time.sleep) + monkeypatch.setattr(manager_module, "time", fake_time) + + mgr, data, control = self._prep( + still_queue=[[42]], + inflight_ids=[42], + ) + mgr.shutdown() + + wait_calls = [c for c in data.cancel_calls if c[1] == "wait"] + immediate_calls = [c for c in data.cancel_calls if c[1] == "immediate"] + assert len(wait_calls) == 1 + assert wait_calls[0][0] == [42] + assert immediate_calls == [([42], "immediate")] + assert data.close_calls == 1 + assert control.close_calls == 1 + + def test_shutdown_no_inflight_skips_drain(self): + mgr, data, control = self._prep() + mgr.shutdown() + + assert data.cancel_calls == [] + assert data.poll_calls == 0 + assert data.close_calls == 1 + assert control.close_calls == 1 + + +# --------------------------------------------------------------------------- +# Bidirectional regression test — both managers act as client AND server +# toward each other on the same peer_id. This is the case the unification +# is meant to fix. +# --------------------------------------------------------------------------- + + +class _LoopbackControl: + """In-memory control transport that pairs two managers head-to-head. + + Each manager hands its outbound message buffer to the other's inbound + queue on poll(). connect() returns a connection whose send() writes + into the peer's inbound side; recv() reads what the peer's connect-or- + poll path delivered for us. + """ + + def __init__(self, local_id: str) -> None: + self._local_id = local_id + self._peer: _LoopbackControl | None = None + self._inbound_outbound: dict[str, _LoopbackConnection] = {} + # Pending inbound for a peer that has not yet been registered. + self._pending: list[tuple[str, dict]] = [] + + def pair(self, peer: _LoopbackControl) -> None: + self._peer = peer + peer._peer = self + + def connect(self, peer_id: str): + if peer_id in self._inbound_outbound: + raise AssertionError(f"already connected to {peer_id}") + conn = _LoopbackConnection(self, peer_id) + self._inbound_outbound[peer_id] = conn + return conn + + def poll(self): + # Drain whatever the peer has sent toward us. + new = [] + if self._peer is not None: + for pid, msg in self._peer._drain_outbound_to(self._local_id): + conn = self._inbound_outbound.get(pid) + if conn is None: + conn = _LoopbackConnection(self, pid) + self._inbound_outbound[pid] = conn + new.append(conn) + conn._inbox.append(msg) + return new + + def _drain_outbound_to(self, peer_local_id: str): + # Peer is calling our poll → return all messages we've sent toward + # peer_local_id (which is the peer's own local_id). + out: list[tuple[str, dict]] = [] + for pid, conn in list(self._inbound_outbound.items()): + # Each outgoing message goes to peer_local_id and is tagged + # with the sender's local_id (i.e., self._local_id). + for msg in conn._outbox: + out.append((self._local_id, msg)) + conn._outbox.clear() + return out + + def close(self): + for conn in self._inbound_outbound.values(): + conn.close() + self._inbound_outbound.clear() + + +class _LoopbackConnection: + def __init__(self, transport: _LoopbackControl, peer_id: str) -> None: + self._transport = transport + self.peer_id = peer_id + self._inbox: list[dict] = [] + self._outbox: list[dict] = [] + self._closed = False + + @property + def alive(self) -> bool: + return not self._closed + + def send(self, msg: dict) -> None: + if self._closed: + raise RuntimeError("send on closed conn") + self._outbox.append(msg) + + def recv(self) -> list[dict]: + msgs = self._inbox + self._inbox = [] + return msgs + + def mark_dead(self) -> None: + self._closed = True + + def close(self) -> None: + self._closed = True + + +class _FakeData: + """Minimal NIXL fake that lets matched transfers complete on the next poll.""" + + def __init__(self, local_id: str) -> None: + self._local_id = local_id + self.block_len = 4096 + self.base_addr = 0x1000 + self.num_blocks = 16 + self.config_fingerprint = "" + self._remote_peers: dict[str, dict] = {} + self._inflight_done: list[int] = [] + self._next_id = 0 + + def get_agent_metadata(self) -> bytes: + return f"meta-{self._local_id}".encode() + + def add_remote_peer( + self, peer_id, agent_metadata, base_addr, num_blocks, block_len + ) -> None: + self._remote_peers[peer_id] = { + "agent_metadata": agent_metadata, + "base_addr": base_addr, + "num_blocks": num_blocks, + "block_len": block_len, + } + + def remove_remote_peer(self, peer_id: str) -> None: + self._remote_peers.pop(peer_id, None) + + def write_blocks(self, peer_id, local_idxs, remote_idxs): + if peer_id not in self._remote_peers: + return None + tid = self._next_id + self._next_id += 1 + self._inflight_done.append(tid) + return tid + + def poll(self): + from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult + + done = self._inflight_done[:] + self._inflight_done.clear() + return PollResult(done=done, failed=[]) + + def cancel(self, transfer_ids) -> None: + pass + + def close(self) -> None: + pass + + +def _build_paired_managers() -> tuple[P2PSecondaryTierManager, P2PSecondaryTierManager]: + """Two managers each acting as both client and server toward the other. + + Wires _LoopbackControl pair + per-side _FakeData so transfers complete + on the next poll. The test drives polling by calling get_finished_jobs(), + which invokes _poll_once synchronously on the calling thread. + """ + mgr_a = _make_manager() + mgr_b = _make_manager() + mgr_a._local_id = "A:1" + mgr_b._local_id = "B:2" + + ctrl_a = _LoopbackControl(mgr_a._local_id) + ctrl_b = _LoopbackControl(mgr_b._local_id) + ctrl_a.pair(ctrl_b) + + mgr_a._control = ctrl_a # type: ignore[assignment] + mgr_b._control = ctrl_b # type: ignore[assignment] + mgr_a._data = _FakeData(mgr_a._local_id) # type: ignore[assignment] + mgr_b._data = _FakeData(mgr_b._local_id) # type: ignore[assignment] + + return mgr_a, mgr_b + + +class TestBidirectionalManager: + """Two managers each load FROM and serve TO the other over a single peer.""" + + def test_both_loads_succeed(self): + mgr_a, mgr_b = _build_paired_managers() + + a_loads_kv = "req-AtoB-load" # A loads, B serves + b_loads_kv = "req-BtoA-load" # B loads, A serves + + a_decoder_params = { + "prefill": { + "kv_request_id": a_loads_kv, + "remote_host": "B", + "remote_port": 2, + }, + } + b_decoder_params = { + "prefill": { + "kv_request_id": b_loads_kv, + "remote_host": "A", + "remote_port": 1, + }, + } + a_prefiller_params = {"decode": {"kv_request_id": b_loads_kv}} + b_prefiller_params = {"decode": {"kv_request_id": a_loads_kv}} + + # 1. Both sides open client-role sessions toward the peer. + mgr_a.on_new_request(_req_context(a_decoder_params)) + mgr_b.on_new_request(_req_context(b_decoder_params)) + + # 2. Both sides store the blocks the peer will fetch. + mgr_a.submit_store( + _job_metadata( + job_id=100, + keys=[b"a-block"], + block_ids=[0], + kv_params=a_prefiller_params, + ) + ) + mgr_b.submit_store( + _job_metadata( + job_id=200, + keys=[b"b-block"], + block_ids=[0], + kv_params=b_prefiller_params, + ) + ) + + # 3. Both sides submit loads. + mgr_a.submit_load( + _job_metadata( + job_id=101, + keys=[b"b-block"], + block_ids=[0], + kv_params=a_decoder_params, + ) + ) + mgr_b.submit_load( + _job_metadata( + job_id=201, + keys=[b"a-block"], + block_ids=[0], + kv_params=b_decoder_params, + ) + ) + + # 4. Drive several poll iterations on each side. Each + # get_finished_jobs() call invokes _poll_once synchronously. + all_a: list[JobResult] = [] + all_b: list[JobResult] = [] + for _ in range(8): + all_a.extend(list(mgr_a.get_finished_jobs())) + all_b.extend(list(mgr_b.get_finished_jobs())) + + # Both load jobs and both store jobs must complete successfully. + a_ok = {r.job_id for r in all_a if r.success} + b_ok = {r.job_id for r in all_b if r.success} + # A: load job 101 + store job 100 + assert 101 in a_ok, f"A loads succeeded: {a_ok}" + assert 100 in a_ok, f"A stores succeeded: {a_ok}" + # B: load job 201 + store job 200 + assert 201 in b_ok, f"B loads succeeded: {b_ok}" + assert 200 in b_ok, f"B stores succeeded: {b_ok}" + + +# --------------------------------------------------------------------------- +# _accept_new_peers — duplicate connection rejection +# --------------------------------------------------------------------------- + + +class _RecordingConn: + """Inbound connection stub that records close()/peer_id only.""" + + def __init__(self, peer_id: str) -> None: + self.peer_id = peer_id + self.close_calls: int = 0 + + def close(self) -> None: + self.close_calls += 1 + + +class TestAcceptNewPeers: + """A second inbound from an already-connected peer is rejected and the + new conn is closed; the existing session is left untouched.""" + + def test_duplicate_connection_is_closed_and_existing_session_untouched(self): + mgr = _make_manager() + peer_id = "10.0.0.1:8000" + existing = _FakeSession(peer_id=peer_id, connected=True) + mgr._sessions[peer_id] = existing # type: ignore[assignment] + + new_conn = _RecordingConn(peer_id) + mgr._accept_new_peers([new_conn]) + + # Manager swallowed the ValueError and closed the duplicate conn. + assert new_conn.close_calls == 1 + # Existing session was NOT re-attached. + assert existing.attached == [] + # Session map unchanged. + assert mgr._sessions[peer_id] is existing + + def test_creates_session_for_new_peer(self): + """An inbound conn from a peer with no existing session creates + a fresh connected session and registers it under conn.peer_id. + The prefiller has no pre-created pending session anymore — the + first signal of a peer's existence is its inbound connection.""" + + class FakeData: + block_len = 4096 + base_addr = 0x1000 + num_blocks = 16 + config_fingerprint = "" + + def get_agent_metadata(self): + return b"meta" + + def add_remote_peer(self, *args, **kwargs): + pass + + mgr = _make_manager() + mgr._data = FakeData() # type: ignore[assignment] + peer_id = "10.0.0.1:8000" + + # Real ControlConnection-shaped fake: send/close/peer_id only. + sent: list[dict] = [] + + class _Conn: + def __init__(self, pid: str) -> None: + self.peer_id = pid + self.alive = True + + def send(self, msg: dict) -> None: + sent.append(msg) + + def close(self) -> None: + self.alive = False + + mgr._accept_new_peers([_Conn(peer_id)]) # type: ignore[arg-type] + + assert peer_id in mgr._sessions + assert mgr._sessions[peer_id].connected is True + # Session sent its ConnectMsg on the new connection. + assert any(m for m in sent) + + +# --------------------------------------------------------------------------- +# _poll_once orchestration +# --------------------------------------------------------------------------- + + +class TestPollOnce: + """_poll_once must drain control, accept new peers, poll every session, + surface results, and reap dead sessions — in that order.""" + + def test_orchestrates_accept_poll_and_reap(self): + mgr = _make_manager() + + # Alive session whose poll() returns one load + one store. + peer_alive = "10.0.0.2:9000" + alive = _FakeSession( + peer_id=peer_alive, + alive=True, + connected=True, + loads=[LoadResult(job_id=11, kv_request_id="req-11", success=True)], + stores=[StoreResult(job_id=22, success=True)], + ) + mgr._sessions[peer_alive] = alive # type: ignore[assignment] + + # Dead session whose pending close() jobs surface as failures. + peer_dead = "10.0.0.3:9999" + dead = _FakeSession( + peer_id=peer_dead, + alive=False, + connected=True, + close_loads=[(33, "req-33")], + close_stores=[44], + ) + mgr._sessions[peer_dead] = dead # type: ignore[assignment] + + class _Ctrl: + def poll(self_inner): + return [] + + class _Data: + def remove_remote_peer(self_inner, pid): + pass + + mgr._control = _Ctrl() # type: ignore[assignment] + mgr._data = _Data() # type: ignore[assignment] + + mgr._poll_once() + + # Every session was polled — alive's results landed. + # Dead session was reaped — its close() failures landed. + finished = mgr._finished_jobs + ok = {(r.job_id, r.success) for r in finished} + assert (11, True) in ok # alive load result + assert (22, True) in ok # alive store result + assert (33, False) in ok # dead session's pending load + assert (44, False) in ok # dead session's pending store + assert "req-33" in mgr._failed_req_ids + assert peer_dead not in mgr._sessions + assert peer_alive in mgr._sessions + + def test_new_fetch_id_binds_and_replays_unbound_batches(self): + """When session.poll() reports a kv_request_id whose FetchMsg + arrived this tick, the manager binds it to that session and + replays every parked submit_store batch via add_stored_blocks.""" + from vllm.v1.kv_offload.tiering.p2p.manager import _UnboundStoreBatch + + mgr = _make_manager() + peer = "10.0.0.1:8000" + sess = _FakeSession( + peer_id=peer, + alive=True, + connected=True, + new_fetch_ids=["req-1"], + ) + mgr._sessions[peer] = sess # type: ignore[assignment] + mgr._unbound_stores["req-1"] = [ + _UnboundStoreBatch(job_id=5, keys=[b"k1"], block_ids=[0]), + _UnboundStoreBatch(job_id=6, keys=[b"k2"], block_ids=[1]), + ] + + class _Ctrl: + def poll(self_inner): + return [] + + mgr._control = _Ctrl() # type: ignore[assignment] + mgr._poll_once() + + assert mgr._kv_to_session["req-1"] is sess + assert "req-1" not in mgr._unbound_stores + replayed = [ + (kv_req_id, list(keys), job_id) + for kv_req_id, keys, _, job_id in sess.stores_added + ] + assert replayed == [("req-1", [b"k1"], 5), ("req-1", [b"k2"], 6)] + + def test_new_fetch_id_with_no_unbound_still_binds(self): + """A FetchMsg for a kv_request_id with no parked batches still + records the binding so subsequent submit_stores route fast.""" + mgr = _make_manager() + peer = "10.0.0.1:8000" + sess = _FakeSession( + peer_id=peer, + alive=True, + connected=True, + new_fetch_ids=["req-fast"], + ) + mgr._sessions[peer] = sess # type: ignore[assignment] + + class _Ctrl: + def poll(self_inner): + return [] + + mgr._control = _Ctrl() # type: ignore[assignment] + mgr._poll_once() + + assert mgr._kv_to_session["req-fast"] is sess + assert sess.stores_added == [] + + def test_failed_load_records_kv_request_id(self): + """A LoadResult(success=False) from session.poll() must add its + kv_request_id to _failed_req_ids so future lookups return MISS.""" + mgr = _make_manager() + peer = "10.0.0.1:8000" + sess = _FakeSession( + peer_id=peer, + alive=True, + connected=True, + loads=[LoadResult(job_id=5, kv_request_id="req-5", success=False)], + ) + mgr._sessions[peer] = sess # type: ignore[assignment] + + class _Ctrl: + def poll(self_inner): + return [] + + mgr._control = _Ctrl() # type: ignore[assignment] + + mgr._poll_once() + + assert mgr._finished_jobs == [JobResult(job_id=5, success=False)] + assert "req-5" in mgr._failed_req_ids + + +# --------------------------------------------------------------------------- +# drain_jobs +# --------------------------------------------------------------------------- + + +class _DrainCtrl: + """Trivial control fake whose poll() returns an empty list.""" + + def poll(self): + return [] + + +class TestDrainJobs: + def test_returns_immediately_when_quiescent(self): + """No sessions and no inflight: drain_jobs returns without sleeping.""" + mgr = _make_manager() + mgr._control = _DrainCtrl() # type: ignore[assignment] + + sleeps: list[float] = [] + # If drain_jobs sleeps when nothing is pending, that's a regression. + import vllm.v1.kv_offload.tiering.p2p.manager as m + + original_sleep = m.time.sleep + m.time.sleep = lambda s: sleeps.append(s) # type: ignore[assignment] + try: + mgr.drain_jobs() + finally: + m.time.sleep = original_sleep # type: ignore[assignment] + + assert sleeps == [] + + def test_returns_when_session_has_no_inflight_or_inbound(self): + """A session with empty _inbound and _inflight does not block drain.""" + mgr = _make_manager() + mgr._control = _DrainCtrl() # type: ignore[assignment] + mgr._sessions["peer:1"] = _FakeSession(peer_id="peer:1") # type: ignore[assignment] + # Should return on the first iteration. + mgr.drain_jobs() + + def test_logs_warning_after_5s_then_completes(self, monkeypatch): + """A session that stays inflight past 5s triggers the warning, and + once it clears the loop returns.""" + mgr = _make_manager() + mgr._control = _DrainCtrl() # type: ignore[assignment] + sess = _FakeSession(peer_id="peer:1") + sess._server._inflight = {1: object()} # non-empty + mgr._sessions["peer:1"] = sess # type: ignore[assignment] + + # Synthetic monotonic clock: 100.0 for the start stamp, then 106.0 + # for the first elapsed-check (past the 5s warning threshold), then + # steady at 106.0 for any later checks. + clock = iter([100.0, 106.0]) + + def fake_monotonic() -> float: + try: + return next(clock) + except StopIteration: + return 106.0 + + monkeypatch.setattr(manager_module, "_DRAIN_SLEEP_S", 0.0) + + # Spy on the warning logger directly — vllm's logger does not + # propagate to root, so caplog can't see it. + warnings: list[str] = [] + + def record_warning(msg, *args, **_kwargs): + warnings.append(msg % args if args else msg) + + monkeypatch.setattr(manager_module.logger, "warning", record_warning) + + # Clear inflight after the first sleep so drain can exit on the + # next iteration's `pending` check. + n_sleeps = 0 + + def clearing_sleep(_s): + nonlocal n_sleeps + n_sleeps += 1 + sess._server._inflight = {} + + # Patch via a fake module on `manager_module.time` so we do not mutate + # the global `time` module — other code in the process (e.g. the + # buildkite test collector's pytest_runtest_logreport hook) calls + # time.monotonic() before monkeypatch teardown. + fake_time = SimpleNamespace(monotonic=fake_monotonic, sleep=clearing_sleep) + monkeypatch.setattr(manager_module, "time", fake_time) + + mgr.drain_jobs() + + assert any("still draining after 5s" in w for w in warnings), warnings + + +# --------------------------------------------------------------------------- +# on_schedule_end +# --------------------------------------------------------------------------- + + +class TestOnScheduleEnd: + def test_is_noop(self): + """on_schedule_end is a documented no-op; just confirm it doesn't + raise and doesn't mutate state.""" + mgr = _make_manager() + before_sessions = dict(mgr._sessions) + before_jobs = list(mgr._finished_jobs) + assert mgr.on_schedule_end() is None + assert mgr._sessions == before_sessions + assert mgr._finished_jobs == before_jobs + + +# --------------------------------------------------------------------------- +# Connection death mid-transfer (real P2PSession via paired managers) +# --------------------------------------------------------------------------- + + +class TestConnectionDeathMidTransfer: + """When a peer's control connection dies while a load is in flight, + the load surfaces as failed and its kv_request_id lands in + _failed_req_ids so future lookups route to local prefill. The + prefiller-side store no longer travels through the session at store + time (it's parked in _unbound_stores keyed by kv_request_id), so its + cleanup on connection death is via on_request_finished or the + unbound-store timeout — covered separately below.""" + + def test_dead_connection_with_pending_work_surfaces_failures(self): + mgr_a, mgr_b = _build_paired_managers() + + a_decoder_params = { + "prefill": { + "kv_request_id": "req-load", + "remote_host": "B", + "remote_port": 2, + }, + } + a_prefiller_params = {"decode": {"kv_request_id": "req-store"}} + + # Open the outbound session A->B and submit one load + one store. + mgr_a.on_new_request(_req_context(a_decoder_params)) + mgr_a.submit_store( + _job_metadata( + job_id=900, + keys=[b"a-block"], + block_ids=[0], + kv_params=a_prefiller_params, + ) + ) + mgr_a.submit_load( + _job_metadata( + job_id=901, + keys=[b"b-block"], + block_ids=[0], + kv_params=a_decoder_params, + ) + ) + + # Drain anything the loopback can deliver synchronously, but stop + # before the remote side has had time to complete the transfers. + list(mgr_a.get_finished_jobs()) + + # Sanity: store 900 is parked in unbound_stores, not in any + # session — the producer no longer learns the peer at store time. + assert "req-store" in mgr_a._unbound_stores + + # Kill the control connection out from under the session. + peer_id = "B:2" + sess = mgr_a._sessions[peer_id] + assert sess._conn is not None + sess._conn.mark_dead() + + # Reap surfaces the load (which lived inside the session) as + # failed. The store 900 is not session-scoped — it survives the + # session reap and waits for the unbound-store timeout. + results: list[JobResult] = [] + for _ in range(3): + results.extend(list(mgr_a.get_finished_jobs())) + + outcomes = {(r.job_id, r.success) for r in results} + assert (901, False) in outcomes, f"load should fail: {outcomes}" + assert (900, False) not in outcomes, f"store should still be parked: {outcomes}" + assert "req-load" in mgr_a._failed_req_ids + # Session removed. + assert peer_id not in mgr_a._sessions + # Store batch is still parked. + assert "req-store" in mgr_a._unbound_stores + + # The engine signals the producer's request is done. That is a + # no-op for the parked batch — `on_request_finished` does not + # evict unbound stores; only `_reap_unbound_stores` does, after + # the unbound-store timeout. Job 900 stays unfinished here. + mgr_a.on_request_finished(_req_context(a_prefiller_params)) + finishes = {(r.job_id, r.success) for r in mgr_a._finished_jobs} + assert (900, False) not in finishes + assert (900, True) not in finishes + assert "req-store" in mgr_a._unbound_stores diff --git a/tests/v1/kv_offload/tiering/p2p/test_sessions.py b/tests/v1/kv_offload/tiering/p2p/test_sessions.py new file mode 100644 index 0000000000..05c0a3abdd --- /dev/null +++ b/tests/v1/kv_offload/tiering/p2p/test_sessions.py @@ -0,0 +1,1645 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the unified bidirectional P2PSession. + +A P2PSession owns a single ControlConnection and dispatches every message +type — both client-role (FetchMsg / TransferDoneMsg / AbortAck) and +server-role (FetchMsg / TransferDoneMsg / AbortAck from the peer's +perspective). These tests exercise both flows independently and the +bidirectional case where one session simultaneously serves a fetch and +completes its own load. +""" + +from __future__ import annotations + +import time + +import pytest + +from vllm.v1.kv_offload.tiering.p2p.session import ( + LoadResult, + P2PSession, + StoreResult, +) +from vllm.v1.kv_offload.tiering.p2p.session.client import ( + _ABORT_ACK_TIMEOUT_S, + _LOAD_TIMEOUT_S, +) +from vllm.v1.kv_offload.tiering.p2p.session.protocol import ( + TYPE_KEY, + AbortAckMsg, + AbortFetchMsg, + ConnectAckMsg, + ConnectMsg, + DisconnectMsg, + FetchMsg, + TransferDoneMsg, +) +from vllm.v1.kv_offload.tiering.p2p.session.server import ( + _CANCEL_DRAIN_TIMEOUT_S, + _InflightXfer, +) +from vllm.v1.kv_offload.tiering.p2p.session.session import ( + _MAX_CONSECUTIVE_DISPATCH_ERRORS, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class FakeDataTransport: + """Minimal fake DataTransport for testing sessions.""" + + def __init__( + self, + base_addr: int = 0x1000, + num_blocks: int = 16, + block_len: int = 4096, + config_fingerprint: str = "", + ) -> None: + self._base_addr = base_addr + self._num_blocks = num_blocks + self._block_len = block_len + self._config_fingerprint = config_fingerprint + self._remote_peers: dict[str, dict] = {} + self._transfers: dict[int, tuple] = {} + self._next_id = 0 + self._poll_done: list[int] = [] + self._poll_failed: list[int] = [] + self._cancel_still_inflight: set[int] = set() + self._cancel_calls: list[tuple[list[int], str]] = [] + + @property + def base_addr(self) -> int: + return self._base_addr + + @property + def num_blocks(self) -> int: + return self._num_blocks + + @property + def block_len(self) -> int: + return self._block_len + + @property + def config_fingerprint(self) -> str: + return self._config_fingerprint + + def get_agent_metadata(self) -> bytes: + return b"fake-metadata" + + def add_remote_peer( + self, peer_id, agent_metadata, base_addr, num_blocks, block_len + ) -> None: + self._remote_peers[peer_id] = { + ConnectMsg.AGENT_METADATA: agent_metadata, + ConnectMsg.BASE_ADDR: base_addr, + "num_blocks": num_blocks, + "block_len": block_len, + } + + def remove_remote_peer(self, peer_id: str) -> None: + self._remote_peers.pop(peer_id, None) + + def write_blocks(self, peer_id, local_idxs, remote_idxs) -> int | None: + if peer_id not in self._remote_peers: + return None + tid = self._next_id + self._next_id += 1 + self._transfers[tid] = (peer_id, local_idxs, remote_idxs) + return tid + + def poll(self): + from vllm.v1.kv_offload.tiering.p2p.data.base import PollResult + + result = PollResult(done=list(self._poll_done), failed=list(self._poll_failed)) + self._poll_done.clear() + self._poll_failed.clear() + return result + + def cancel(self, transfer_ids, mode: str = "immediate") -> list[int]: + ids = list(transfer_ids) + self._cancel_calls.append((ids, mode)) + if mode == "wait": + still: list[int] = [] + for tid in ids: + if tid in self._cancel_still_inflight: + still.append(tid) + else: + self._transfers.pop(tid, None) + return still + for tid in ids: + self._transfers.pop(tid, None) + self._cancel_still_inflight.discard(tid) + return [] + + def close(self) -> None: + pass + + +class FakeConnection: + """Fake ControlConnection that captures sent messages.""" + + def __init__(self, peer_id: str = "peer:8000") -> None: + self.peer_id = peer_id + self._inbox: list[dict] = [] + self._sent: list[dict] = [] + self._closed = False + + @property + def alive(self) -> bool: + return not self._closed + + def send(self, msg: dict) -> None: + self._sent.append(msg) + + def recv(self) -> list[dict]: + msgs = self._inbox + self._inbox = [] + return msgs + + def enqueue(self, msg: dict) -> None: + self._inbox.append(msg) + + def mark_dead(self) -> None: + self._closed = True + + def close(self) -> None: + self._closed = True + + +def _peer_connect_msg( + peer_id: str = "peer:8000", + block_len: int = 4096, + fingerprint: str | None = None, +) -> dict: + """Build a ConnectMsg as if the peer sent it.""" + msg = { + TYPE_KEY: ConnectMsg.TYPE, + ConnectMsg.PEER_ID: peer_id, + ConnectMsg.AGENT_METADATA: b"peer-metadata", + ConnectMsg.BASE_ADDR: 0x2000, + ConnectMsg.NUM_BLOCKS: 16, + ConnectMsg.BLOCK_LEN: block_len, + } + if fingerprint is not None: + msg[ConnectMsg.CONFIG_FINGERPRINT] = fingerprint + return msg + + +def _make_session( + conn: FakeConnection | None = None, + transport: FakeDataTransport | None = None, + peer_id: str = "peer:8000", + local_id: str = "local:9000", +) -> tuple[P2PSession, FakeConnection, FakeDataTransport]: + if conn is None: + conn = FakeConnection(peer_id=peer_id) + if transport is None: + transport = FakeDataTransport() + session = P2PSession( + peer_id=peer_id, + local_id=local_id, + transport=transport, # type: ignore[arg-type] + local_block_len=transport.block_len, + conn=conn, # type: ignore[arg-type] + ) + return session, conn, transport + + +def _activate( + session: P2PSession, conn: FakeConnection, peer_id: str = "peer:8000" +) -> None: + """Drive the handshake: peer sends ConnectMsg + ConnectAckMsg.""" + conn.enqueue(_peer_connect_msg(peer_id=peer_id)) + conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: peer_id}) + session.poll() + + +# --------------------------------------------------------------------------- +# Connect / handshake +# --------------------------------------------------------------------------- + + +class TestConnectHandshake: + def test_connect_msg_sent_on_creation(self): + """Session sends its own ConnectMsg on connection.""" + session, conn, _ = _make_session() + assert len(conn._sent) == 1 + msg = conn._sent[0] + assert msg[TYPE_KEY] == ConnectMsg.TYPE + assert msg[ConnectMsg.PEER_ID] == "local:9000" + assert msg[ConnectMsg.NUM_BLOCKS] == 16 + assert msg[ConnectMsg.BLOCK_LEN] == 4096 + assert ConnectMsg.AGENT_METADATA in msg + + def test_peer_connect_triggers_add_remote_and_ack(self): + """Receiving ConnectMsg registers peer and replies with ConnectAck.""" + session, conn, transport = _make_session() + conn.enqueue(_peer_connect_msg()) + session.poll() + assert "peer:8000" in transport._remote_peers + ack = next(m for m in conn._sent if m[TYPE_KEY] == ConnectAckMsg.TYPE) + assert ack[ConnectAckMsg.PEER_ID] == "local:9000" + + def test_connect_ack_makes_session_ready(self): + """Session.ready becomes True after ConnectAckMsg.""" + session, conn, _ = _make_session() + assert not session.ready + conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "peer:8000"}) + session.poll() + assert session.ready + + def test_messages_queued_before_ack_flush_on_ack(self): + """Outgoing messages sent before ConnectAck are flushed after.""" + session, conn, _ = _make_session() + session.request_blocks( + job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0] + ) + # Before ack: only our ConnectMsg was sent. + assert len(conn._sent) == 1 + assert conn._sent[0][TYPE_KEY] == ConnectMsg.TYPE + # Ack arrives. + conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "peer:8000"}) + session.poll() + # Queued fetch is now sent. + assert any(m[TYPE_KEY] == FetchMsg.TYPE for m in conn._sent) + + def test_block_len_mismatch_marks_dead(self): + """Mismatched block_len rejects peer and marks connection dead.""" + session, conn, transport = _make_session() + conn.enqueue(_peer_connect_msg(block_len=8192)) # mismatch + session.poll() + assert "peer:8000" not in transport._remote_peers + assert not session.alive + + def test_config_fingerprint_mismatch_marks_dead(self): + """Mismatched config fingerprint rejects peer.""" + transport = FakeDataTransport(config_fingerprint="abc123") + session, conn, _ = _make_session(transport=transport) + conn.enqueue(_peer_connect_msg(fingerprint="different")) + session.poll() + assert "peer:8000" not in transport._remote_peers + assert not session.alive + + def test_config_fingerprint_match_succeeds(self): + """Matching fingerprints register the peer.""" + transport = FakeDataTransport(config_fingerprint="same_fp") + session, conn, _ = _make_session(transport=transport) + conn.enqueue(_peer_connect_msg(fingerprint="same_fp")) + session.poll() + assert "peer:8000" in transport._remote_peers + assert session.alive + + def test_missing_fingerprint_allowed(self): + """Missing fingerprint on either side is allowed.""" + transport = FakeDataTransport(config_fingerprint="abc123") + session, conn, _ = _make_session(transport=transport) + conn.enqueue(_peer_connect_msg()) # no fingerprint + session.poll() + assert "peer:8000" in transport._remote_peers + + +# --------------------------------------------------------------------------- +# Client-role flows +# --------------------------------------------------------------------------- + + +class TestClientFlows: + def test_request_blocks_sends_fetch(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=1, kv_request_id="req-1", keys=[b"k1", b"k2"], block_ids=[0, 1] + ) + lookup = conn._sent[-1] + assert lookup[TYPE_KEY] == FetchMsg.TYPE + assert lookup[FetchMsg.KV_REQUEST_ID] == "req-1" + assert lookup[FetchMsg.BLOCK_HASHES] == [b"k1", b"k2"] + assert lookup[FetchMsg.BLOCK_INDEXES] == [0, 1] + + def test_transfer_done_success(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0] + ) + conn.enqueue( + { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: "req-1", + TransferDoneMsg.SUCCESS: True, + } + ) + loads = session.poll().loads + assert loads == [LoadResult(job_id=1, kv_request_id="req-1", success=True)] + + def test_transfer_done_failure(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0] + ) + conn.enqueue( + { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: "req-1", + TransferDoneMsg.SUCCESS: False, + } + ) + loads = session.poll().loads + assert loads == [LoadResult(job_id=1, kv_request_id="req-1", success=False)] + + def test_finish_request_sends_abort(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0] + ) + session.finish_request("req-1") + abort = conn._sent[-1] + assert abort[TYPE_KEY] == AbortFetchMsg.TYPE + assert abort[AbortFetchMsg.KV_REQUEST_ID] == "req-1" + + def test_load_timeout_sends_abort(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=1, kv_request_id="req-1", keys=[b"k"], block_ids=[0] + ) + session._client._inbound["req-1"].submitted_at = time.monotonic() - 60.0 + session.poll() + abort = conn._sent[-1] + assert abort[TYPE_KEY] == AbortFetchMsg.TYPE + + def test_load_abort_ack_timeout_surfaces_failure(self): + """After load timeout sends AbortFetch, if no AbortAck arrives within + _ABORT_ACK_TIMEOUT_S the request is surfaced as failed and removed + from _inbound — the engine cannot wait forever on a peer that won't + ack. + """ + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=7, kv_request_id="req-7", keys=[b"k"], block_ids=[0] + ) + # 1) Trip the load timeout to send AbortFetch and stamp aborted_at. + session._client._inbound["req-7"].submitted_at = ( + time.monotonic() - _LOAD_TIMEOUT_S - 1.0 + ) + loads = session.poll().loads + assert loads == [] + assert any( + m.get(TYPE_KEY) == AbortFetchMsg.TYPE + and m[AbortFetchMsg.KV_REQUEST_ID] == "req-7" + for m in conn._sent + ) + assert session._client._inbound["req-7"].aborted_at is not None + + # 2) Now backdate aborted_at past the abort-ack timeout. No ack ever + # arrived from the peer. + session._client._inbound["req-7"].aborted_at = ( + time.monotonic() - _ABORT_ACK_TIMEOUT_S - 1.0 + ) + loads = session.poll().loads + assert loads == [LoadResult(job_id=7, kv_request_id="req-7", success=False)] + assert "req-7" not in session._client._inbound + + def test_load_abort_ack_clears_request(self): + """After load timeout sends AbortFetch, an arriving AbortAckMsg from + the peer surfaces the failure cleanly and removes the request from + _inbound — covers the on_abort_ack arrival path.""" + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks( + job_id=8, kv_request_id="req-8", keys=[b"k"], block_ids=[0] + ) + session._client._inbound["req-8"].submitted_at = ( + time.monotonic() - _LOAD_TIMEOUT_S - 1.0 + ) + # First poll: AbortFetch goes out. + session.poll() + assert session._client._inbound["req-8"].aborted_at is not None + + # Peer acks the abort. + conn.enqueue( + { + TYPE_KEY: AbortAckMsg.TYPE, + AbortAckMsg.KV_REQUEST_ID: "req-8", + } + ) + loads = session.poll().loads + assert loads == [LoadResult(job_id=8, kv_request_id="req-8", success=False)] + assert "req-8" not in session._client._inbound + + +# --------------------------------------------------------------------------- +# Server-role flows +# --------------------------------------------------------------------------- + + +class TestServerFlows: + def test_store_then_fetch_matches(self): + """Blocks stored before fetch demand are matched on demand arrival.""" + session, conn, transport = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-1", [b"k1", b"k2"], [0, 1], job_id=1) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [10, 11], + } + ) + session.poll() + assert len(transport._transfers) == 1 + _, (peer, local, remote) = next(iter(transport._transfers.items())) + assert local == [0, 1] + assert remote == [10, 11] + + def test_fetch_then_store_matches(self): + """Fetch demand registered before store; store fulfills it.""" + session, conn, transport = _make_session() + _activate(session, conn) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [5], + } + ) + session.poll() + assert len(transport._transfers) == 0 + session.add_stored_blocks("req-1", [b"k1"], [3], job_id=1) + assert len(transport._transfers) == 1 + + def test_transfer_completion_emits_store_result_and_done(self): + """Completed transfer reports StoreResult and sends TransferDoneMsg.""" + session, conn, transport = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [5], + } + ) + session.poll() + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + stores = session.poll().stores + assert StoreResult(job_id=1, success=True) in stores + assert any(m[TYPE_KEY] == TransferDoneMsg.TYPE for m in conn._sent) + + def test_abort_fetch_replies_with_ack(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: "req-1", + } + ) + session.poll() + ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE) + assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1" + assert "req-1" not in session._server._pending_aborts + + def test_abort_fetch_defers_ack_when_cancel_pending(self): + """If cancel(mode='wait') reports still-inflight tids, the ack is + deferred and the abort is parked in _pending_aborts.""" + session, conn, transport = _make_session() + _activate(session, conn) + # Seed an inflight transfer for req-1 that the transport pretends + # cannot be canceled yet. + tid = 42 + session._server._inflight_add( + tid, + _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + ) + transport._cancel_still_inflight.add(tid) + + conn.enqueue( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: "req-1", + } + ) + session.poll() + + assert not any(m[TYPE_KEY] == AbortAckMsg.TYPE for m in conn._sent) + assert "req-1" in session._server._pending_aborts + # First attempt happens inside _on_abort_fetch; the per-tick + # drain runs again at the end of poll() — both are wait-mode. + assert all(mode == "wait" for _, mode in transport._cancel_calls) + assert tid in session._server._inflight # still tracked + + def test_abort_fetch_acks_after_drain(self): + """Once the transport reports the tid as DONE the parked abort + completes and AbortAckMsg is sent.""" + session, conn, transport = _make_session() + _activate(session, conn) + tid = 42 + session._server._inflight_add( + tid, + _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + ) + transport._cancel_still_inflight.add(tid) + + conn.enqueue( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: "req-1", + } + ) + session.poll() + assert "req-1" in session._server._pending_aborts + + # Backend finishes draining: transport.poll() will return tid as + # DONE, and the next cancel(mode='wait') call sees it's gone. + transport._cancel_still_inflight.discard(tid) + transport._poll_done.append(tid) + + session.poll() + + ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE) + assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1" + assert "req-1" not in session._server._pending_aborts + assert tid not in session._server._inflight + + def test_abort_fetch_force_cancels_after_timeout(self): + """If wait-mode never drains, the deadline forces immediate + cancel and an ack is still sent.""" + session, conn, transport = _make_session() + _activate(session, conn) + tid = 42 + session._server._inflight_add( + tid, + _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + ) + transport._cancel_still_inflight.add(tid) + + conn.enqueue( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: "req-1", + } + ) + session.poll() + assert "req-1" in session._server._pending_aborts + # Backdate past the drain deadline. + session._server._pending_aborts["req-1"] = ( + time.monotonic() - _CANCEL_DRAIN_TIMEOUT_S - 1.0 + ) + # Even if the transport still claims it can't cancel, the + # session must force-pop and ack. + transport._cancel_calls.clear() + + session.poll() + + ack = next(m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE) + assert ack[AbortAckMsg.KV_REQUEST_ID] == "req-1" + assert "req-1" not in session._server._pending_aborts + assert tid not in session._server._inflight + assert ([tid], "immediate") in transport._cancel_calls + + def test_abort_fetch_idempotent_while_draining(self): + """Receiving AbortFetchMsg twice for the same kv_request_id + keeps a single pending entry and produces a single ack.""" + session, conn, transport = _make_session() + _activate(session, conn) + tid = 42 + session._server._inflight_add( + tid, + _InflightXfer(kv_request_id="req-1", block_count=1, job_ids={1}), + ) + transport._cancel_still_inflight.add(tid) + + conn.enqueue( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: "req-1", + } + ) + session.poll() + first_started_at = session._server._pending_aborts["req-1"] + + # Second AbortFetchMsg for the same kv_request_id while still + # draining must not reset the deadline. + conn.enqueue( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: "req-1", + } + ) + session.poll() + assert session._server._pending_aborts["req-1"] == first_started_at + + # Now let the drain succeed and confirm exactly one ack ever. + transport._cancel_still_inflight.discard(tid) + transport._poll_done.append(tid) + session.poll() + + acks = [m for m in conn._sent if m[TYPE_KEY] == AbortAckMsg.TYPE] + assert len(acks) == 1 + assert acks[0][AbortAckMsg.KV_REQUEST_ID] == "req-1" + + def test_store_timeout(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + # Backdate. + session._server._store_jobs[1] = time.monotonic() - 60.0 + stores = session.poll().stores + assert StoreResult(job_id=1, success=False) in stores + + def test_store_timeout_then_late_completion_no_duplicate(self): + """A job timed out by _timeout_pending_store_jobs must not also + emit a contradictory StoreResult(success=True) when the transport + later reports the same transfer as done.""" + session, conn, transport = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [5], + } + ) + session.poll() + tid = next(iter(transport._transfers)) + + # Backdate the store job so the next poll times it out. + session._server._store_jobs[1] = time.monotonic() - 60.0 + stores = session.poll().stores + assert StoreResult(job_id=1, success=False) in stores + assert StoreResult(job_id=1, success=True) not in stores + + # Transport later reports the same transfer as done — must not + # emit a second (contradictory) StoreResult for job_id=1. + transport._poll_done.append(tid) + stores = session.poll().stores + assert all(s.job_id != 1 for s in stores), ( + f"unexpected duplicate StoreResult after timeout: {stores}" + ) + + def test_store_timeout_then_late_failure_no_duplicate(self): + """Symmetric guard: a timed-out job must not also emit a second + StoreResult(success=False) when the transport later reports the + same transfer as failed.""" + session, conn, transport = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [5], + } + ) + session.poll() + tid = next(iter(transport._transfers)) + + session._server._store_jobs[1] = time.monotonic() - 60.0 + stores = session.poll().stores + assert [s for s in stores if s.job_id == 1] == [ + StoreResult(job_id=1, success=False) + ] + + transport._poll_failed.append(tid) + stores = session.poll().stores + assert all(s.job_id != 1 for s in stores), ( + f"unexpected duplicate StoreResult after timeout: {stores}" + ) + + +# --------------------------------------------------------------------------- +# finish_request server-role early-fail flow +# --------------------------------------------------------------------------- + + +class TestFinishRequestServerSide: + def _last_transfer_done(self, conn: FakeConnection) -> dict | None: + for msg in reversed(conn._sent): + if msg[TYPE_KEY] == TransferDoneMsg.TYPE: + return msg + return None + + def test_no_inflight_unmatched_demand_sends_failure(self): + """finish_request with unmatched demand and no inflight -> + immediate TransferDoneMsg(success=False); _outbound cleared.""" + session, conn, _ = _make_session() + _activate(session, conn) + # Decoder demanded a block we never stored. + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [5], + } + ) + session.poll() + assert "req-1" in session._server._outbound + + session.finish_request("req-1") + + msg = self._last_transfer_done(conn) + assert msg is not None + assert msg[TransferDoneMsg.KV_REQUEST_ID] == "req-1" + assert msg[TransferDoneMsg.SUCCESS] is False + assert "req-1" not in session._server._outbound + + def test_with_inflight_defers_then_fires_on_last_transfer(self): + """finish_request with inflight defers; last transfer fires the + early-fail message and clears _outbound.""" + session, conn, transport = _make_session() + _activate(session, conn) + # Demand 2 blocks; we store 1 (kicks one inflight transfer). + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [10, 11], + } + ) + session.poll() + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + assert len(transport._transfers) == 1 + + # finish_request while inflight: no early-fail yet. + before = len(conn._sent) + session.finish_request("req-1") + assert len(conn._sent) == before + assert "req-1" in session._server._outbound + assert session._server._outbound["req-1"].finishing + + # Last inflight settles -> early-fail fires. + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + session.poll() + + msg = self._last_transfer_done(conn) + assert msg is not None + assert msg[TransferDoneMsg.KV_REQUEST_ID] == "req-1" + assert msg[TransferDoneMsg.SUCCESS] is False + assert "req-1" not in session._server._outbound + + def test_full_demand_satisfied_still_sends_success(self): + """finish_request must not override a fully-satisfied transfer: + when remaining hits 0, success=True still fires.""" + session, conn, transport = _make_session() + _activate(session, conn) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [10], + } + ) + session.poll() + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + # Mark finishing (e.g., on_request_finished racing with the last + # store) — but all demand is satisfied. + session.finish_request("req-1") + + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + session.poll() + + msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE) + assert msg[TransferDoneMsg.SUCCESS] is True + + def test_prefiller_first_finish_before_fetch(self): + """Prefiller-first: finish_request runs before the decoder's + fetch arrives. State is held until fetch, then + finalized — success=True if all demand was matched against + available blocks, else success=False.""" + # Case A: all demand satisfied by available blocks. + session, conn, transport = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + # finish_request first — no demand received yet -> defer. + session.finish_request("req-1") + assert "req-1" in session._server._outbound + # Fetch arrives now: demand fully satisfied by available. + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [10], + } + ) + session.poll() + # The transfer was inflight; finalize it. + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + session.poll() + msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE) + assert msg[TransferDoneMsg.SUCCESS] is True + + # Case B: demand exceeds available -> early-fail fires from fetch. + session, conn, transport = _make_session() + _activate(session, conn) + session.add_stored_blocks("req-2", [b"k1"], [0], job_id=2) + session.finish_request("req-2") + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-2", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [10, 11], + } + ) + session.poll() + # Inflight for k1 still in flight; nothing yet for the early-fail + # — the same code path will fire from _collect_store_results. + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + session.poll() + msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE) + assert msg[TransferDoneMsg.SUCCESS] is False + assert "req-2" not in session._server._outbound + + def test_unknown_request_is_noop(self): + session, conn, _ = _make_session() + _activate(session, conn) + before = len(conn._sent) + session.finish_request("never-existed") + assert len(conn._sent) == before + + def test_finish_request_no_inflight_emits_store_failure(self): + """finish_request with a stored-but-unmatched job and no inflight -> + TransferDoneMsg(success=False) AND deferred + StoreResult(success=False) for the submit_store'd job, instead of + the 30s _STORE_TIMEOUT_S path.""" + session, conn, _ = _make_session() + _activate(session, conn) + # Decoder demanded b"demand"; we never stored it. + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"demand"], + FetchMsg.BLOCK_INDEXES: [5], + } + ) + session.poll() + # We did submit_store a different block — goes to available, never + # matches demand. Without the shortcut, job 42 sits in _store_jobs + # for _STORE_TIMEOUT_S. + session.add_stored_blocks("req-1", [b"unrelated"], [0], job_id=42) + assert session._server._outbound["req-1"].pending_job_ids == {42} + + session.finish_request("req-1") + + # Peer notified immediately with success=False (remaining > 0). + msg = self._last_transfer_done(conn) + assert msg is not None + assert msg[TransferDoneMsg.SUCCESS] is False + assert "req-1" not in session._server._outbound + + # Local store job surfaces on the next poll, success=False. + stores = session.poll().stores + assert StoreResult(job_id=42, success=False) in stores + assert 42 not in session._server._store_jobs + + def test_finish_request_remaining_zero_emits_success_via_inflight(self): + """Deferred-via-inflight path: finish_request with inflight, last + transfer drains remaining to 0 -> TransferDoneMsg(success=True) + AND StoreResult(success=True).""" + session, conn, transport = _make_session() + _activate(session, conn) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [10], + } + ) + session.poll() + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=7) + # finish_request races with the inflight transfer. + session.finish_request("req-1") + assert "req-1" in session._server._outbound # deferred + + # Last inflight completes -> _finalize_outbound(success=True) fires. + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + stores = session.poll().stores + + msg = self._last_transfer_done(conn) + assert msg is not None + assert msg[TransferDoneMsg.SUCCESS] is True + assert StoreResult(job_id=7, success=True) in stores + assert "req-1" not in session._server._outbound + + def test_write_blocks_failure_finalizes_with_failure(self): + """write_blocks returning None must not leave the request hanging. + + The matched blocks are gone from req.demanded but no inflight + will satisfy them, so remaining > 0 forever. Setting finishing + and calling _finalize_outbound(success=False) immediately (no + other inflight) tells the peer + emits StoreResult(success=False) + without waiting on _STORE_TIMEOUT_S or _LOAD_TIMEOUT_S. + """ + session, conn, transport = _make_session() + _activate(session, conn) + # Decoder demands b"k1". + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [10], + } + ) + session.poll() + # Force write_blocks to fail on the next call. + transport.write_blocks = lambda *a, **kw: None # type: ignore[assignment] + + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=42) + + # Outbound was finalized immediately (no other inflight). + assert "req-1" not in session._server._outbound + # Peer notified with success=False. + msg = next(m for m in conn._sent if m[TYPE_KEY] == TransferDoneMsg.TYPE) + assert msg[TransferDoneMsg.KV_REQUEST_ID] == "req-1" + assert msg[TransferDoneMsg.SUCCESS] is False + # Local store job surfaces on the next poll. + stores = session.poll().stores + assert StoreResult(job_id=42, success=False) in stores + assert 42 not in session._server._store_jobs + + def test_partial_match_completes_in_two_rounds(self): + """Peer demand for [k1, k2, k3]; first round only k1 is available, + second round adds k2 and k3. Each round transfers what's matched + and the request finalizes with success once remaining hits zero. + """ + session, conn, transport = _make_session() + _activate(session, conn) + + # Peer fetches three blocks. + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2", b"k3"], + FetchMsg.BLOCK_INDEXES: [10, 11, 12], + } + ) + session.poll() + # Demand registered, no matches yet. + assert session._server._inflight == {} + outbound = session._server._outbound["req-1"] + assert outbound.remaining == 3 + assert set(outbound.demanded.keys()) == {b"k1", b"k2", b"k3"} + + # Round 1: only k1 is stored locally. + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=100) + # One inflight transfer for k1. + assert len(session._server._inflight) == 1 + tid_1 = next(iter(session._server._inflight)) + assert transport._transfers[tid_1][1] == [0] + assert transport._transfers[tid_1][2] == [10] + # k2 and k3 still demanded. + assert set(outbound.demanded.keys()) == {b"k2", b"k3"} + + # Transfer 1 completes. + transport._poll_done.append(tid_1) + stores = session.poll().stores + assert StoreResult(job_id=100, success=True) in stores + assert session._server._inflight == {} + assert outbound.remaining == 2 + # Not yet finalized — still 2 blocks demanded. + assert "req-1" in session._server._outbound + + # Round 2: k2 and k3 arrive together. + session.add_stored_blocks("req-1", [b"k2", b"k3"], [1, 2], job_id=200) + assert len(session._server._inflight) == 1 + tid_2 = next(iter(session._server._inflight)) + assert tid_2 != tid_1 + assert sorted(transport._transfers[tid_2][1]) == [1, 2] + + # Transfer 2 completes — request now fully satisfied. + transport._poll_done.append(tid_2) + stores = session.poll().stores + assert StoreResult(job_id=200, success=True) in stores + # _finalize_outbound fired — request gone, peer notified with success. + assert "req-1" not in session._server._outbound + done = next(m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE) + assert done[TransferDoneMsg.KV_REQUEST_ID] == "req-1" + assert done[TransferDoneMsg.SUCCESS] is True + + def test_write_blocks_failure_finalizes_after_last_inflight_completes(self): + """write_blocks returns None on a SECOND match while a first transfer + is still inflight. The request should NOT finalize until the inflight + completes, then the elif branch in collect_results + (``finishing and not _has_inflight_for(...)``) finalizes it as failure. + """ + session, conn, transport = _make_session() + _activate(session, conn) + + # Peer demands two blocks. + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [10, 11], + } + ) + session.poll() + + # Round 1: k1 transfers cleanly. + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=100) + assert len(session._server._inflight) == 1 + tid_1 = next(iter(session._server._inflight)) + outbound = session._server._outbound["req-1"] + assert outbound.remaining == 2 # decrement happens on completion + assert outbound.finishing is False + + # Round 2: write_blocks fails for k2 while transfer_1 is still inflight. + transport.write_blocks = lambda *a, **kw: None # type: ignore[assignment] + session.add_stored_blocks("req-1", [b"k2"], [1], job_id=200) + # No new transfer was registered. + assert list(session._server._inflight.keys()) == [tid_1] + # Marked finishing, but NOT finalized yet (transfer_1 still inflight). + assert outbound.finishing is True + assert "req-1" in session._server._outbound + done_msgs = [m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE] + assert done_msgs == [] + + # Transfer 1 completes — now ``_has_inflight_for("req-1")`` is False + # and the elif branch in collect_results fires _finalize(success=False). + # The k1 success result is direct; the k2 failure result is queued + # in _pending_store_results and surfaces on the NEXT poll. + transport._poll_done.append(tid_1) + stores_first = session.poll().stores + assert StoreResult(job_id=100, success=True) in stores_first + # Outbound state cleaned up; peer notified with success=False. + assert "req-1" not in session._server._outbound + done = next(m for m in conn._sent if m.get(TYPE_KEY) == TransferDoneMsg.TYPE) + assert done[TransferDoneMsg.KV_REQUEST_ID] == "req-1" + assert done[TransferDoneMsg.SUCCESS] is False + + # Next poll drains the queued failure. + stores_second = session.poll().stores + assert StoreResult(job_id=200, success=False) in stores_second + + +# --------------------------------------------------------------------------- +# Bidirectional — the case the unification is meant to fix +# --------------------------------------------------------------------------- + + +class TestBidirectional: + def test_session_handles_both_roles_concurrently(self): + """Single session simultaneously serves a fetch and completes a load. + + This is the regression test for the unification: with the old + split design the inbound FetchMsg would be dispatched to a + client-only session and dropped (or a server-only session would + miss the TransferDoneMsg). One unified session handles both. + """ + session, conn, transport = _make_session() + _activate(session, conn) + + # Server role: the peer fetches a block from us. + session.add_stored_blocks("req-srv", [b"served"], [0], job_id=100) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-srv", + FetchMsg.BLOCK_HASHES: [b"served"], + FetchMsg.BLOCK_INDEXES: [7], + } + ) + + # Client role: we ask the peer for a different block. + session.request_blocks( + job_id=200, kv_request_id="req-cli", keys=[b"loaded"], block_ids=[3] + ) + + # Both flows progress in the same poll. + session.poll() + + # Server side: write_blocks was submitted. + assert len(transport._transfers) == 1 + # Client side: the lookup was sent. + assert any( + m[TYPE_KEY] == FetchMsg.TYPE and m[FetchMsg.KV_REQUEST_ID] == "req-cli" + for m in conn._sent + ) + + # Peer now signals: server-side transfer completes AND a + # TransferDoneMsg arrives for our client-side request, all in + # one batch on the same connection. + tid = next(iter(transport._transfers)) + transport._poll_done.append(tid) + conn.enqueue( + { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: "req-cli", + TransferDoneMsg.SUCCESS: True, + } + ) + result_ = session.poll() + loads = result_.loads + stores = result_.stores + + assert LoadResult(job_id=200, kv_request_id="req-cli", success=True) in loads + assert StoreResult(job_id=100, success=True) in stores + + +# --------------------------------------------------------------------------- +# Pending sessions (no connection yet) +# --------------------------------------------------------------------------- + + +class TestPendingSession: + def test_pending_session_buffers_stored_blocks(self): + """Pending session accepts add_stored_blocks but cannot send.""" + transport = FakeDataTransport() + session = P2PSession( + peer_id="peer:8000", + local_id="local:9000", + transport=transport, # type: ignore[arg-type] + local_block_len=4096, + conn=None, + ) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + result_ = session.poll() + loads = result_.loads + stores = result_.stores + assert loads == [] + assert stores == [] + assert not session.connected + assert session.alive + + def test_attach_connection_sends_connect(self): + """attach_connection triggers our ConnectMsg send.""" + transport = FakeDataTransport() + session = P2PSession( + peer_id="peer:8000", + local_id="local:9000", + transport=transport, # type: ignore[arg-type] + local_block_len=4096, + conn=None, + ) + conn = FakeConnection(peer_id="peer:8000") + session.attach_connection(conn) # type: ignore[arg-type] + assert conn._sent + assert conn._sent[0][TYPE_KEY] == ConnectMsg.TYPE + + def test_attach_connection_twice_raises(self): + """attach_connection on an already-connected session raises.""" + session, conn, _ = _make_session() + with pytest.raises(ValueError, match="already connected"): + session.attach_connection(FakeConnection()) # type: ignore[arg-type] + + def test_pending_close_returns_pending_stores(self): + """Closing a pending session reports buffered stores as failed.""" + transport = FakeDataTransport() + session = P2PSession( + peer_id="peer:8000", + local_id="local:9000", + transport=transport, # type: ignore[arg-type] + local_block_len=4096, + conn=None, + ) + session.add_stored_blocks("req-1", [b"k1"], [0], job_id=1) + session.add_stored_blocks("req-2", [b"k2"], [1], job_id=2) + failed_loads, failed_stores = session.close() + assert failed_loads == [] + assert set(failed_stores) == {1, 2} + + +# --------------------------------------------------------------------------- +# Disconnect +# --------------------------------------------------------------------------- + + +class TestDisconnect: + def test_disconnect_marks_session_dead(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue({TYPE_KEY: DisconnectMsg.TYPE}) + session.poll() + assert not session.alive + + def test_close_returns_pending_loads_and_stores(self): + session, conn, _ = _make_session() + _activate(session, conn) + session.request_blocks(1, "req-1", [b"k"], [0]) + session.request_blocks(2, "req-2", [b"k"], [0]) + session.add_stored_blocks("req-srv", [b"k"], [0], job_id=10) + failed_loads, failed_stores = session.close() + assert set(failed_loads) == {(1, "req-1"), (2, "req-2")} + assert set(failed_stores) == {10} + + +# --------------------------------------------------------------------------- +# Adversarial / malformed messages +# --------------------------------------------------------------------------- + + +class TestAdversarial: + def test_unknown_message_type_logged(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue({TYPE_KEY: "evil_command"}) + result_ = session.poll() + loads = result_.loads + stores = result_.stores + assert loads == [] + assert stores == [] + + def test_empty_message(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue({}) + result_ = session.poll() + loads = result_.loads + stores = result_.stores + assert loads == [] + assert stores == [] + + def test_non_dict_message(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn._inbox.append(42) # type: ignore[arg-type] + result_ = session.poll() + loads = result_.loads + stores = result_.stores + assert loads == [] + assert stores == [] + + def test_fetch_mismatched_lengths(self): + session, conn, transport = _make_session() + _activate(session, conn) + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-bad", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [1], + } + ) + session.poll() + assert len(transport._transfers) == 0 + # Protocol violation: session disconnects immediately so the peer + # can't keep wedging us with malformed traffic. + assert not session.alive + assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent) + + def test_transfer_done_missing_kv_request_id(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue({TYPE_KEY: TransferDoneMsg.TYPE, TransferDoneMsg.SUCCESS: True}) + loads = session.poll().loads + assert loads == [] + assert not session.alive + assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent) + + def test_duplicate_connect_ack(self): + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue({TYPE_KEY: ConnectAckMsg.TYPE, ConnectAckMsg.PEER_ID: "peer:8000"}) + session.poll() + assert session.ready + + +class TestDispatchErrorHandling: + """Errors raised by message handlers split into two classes: + + - Protocol-contract violations from the peer (ValueError) → disconnect + on the first occurrence; retrying won't help and may corrupt state. + - Anything else is treated as an internal bug: log loudly, count, and + only disconnect once errors arrive in a tight burst. A successful + dispatch in between resets the counter. + """ + + def test_value_error_disconnects_on_first_occurrence(self): + """A FetchMsg that fails validate() raises ValueError and must + terminate the session immediately, with a DisconnectMsg sent.""" + session, conn, _ = _make_session() + _activate(session, conn) + # length mismatch → FetchMsg.validate raises ValueError + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-bad", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [1], + } + ) + session.poll() + assert not session.alive + assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent) + + def test_transfer_done_missing_field_disconnects(self): + """Same contract for a malformed TransferDoneMsg from the peer.""" + session, conn, _ = _make_session() + _activate(session, conn) + conn.enqueue({TYPE_KEY: TransferDoneMsg.TYPE, TransferDoneMsg.SUCCESS: True}) + session.poll() + assert not session.alive + + def test_internal_error_does_not_disconnect_once(self): + """A non-ValueError raised by a handler is treated as an internal + bug: counter increments, session stays alive on a single hit.""" + session, conn, _ = _make_session() + _activate(session, conn) + + def _boom(*args, **kwargs): + raise RuntimeError("simulated internal bug") + + session._server.on_fetch = _boom # type: ignore[assignment] + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [0], + } + ) + session.poll() + assert session.alive + assert session._dispatch_error_count == 1 + + def test_internal_error_threshold_disconnects(self): + """Once consecutive non-protocol errors hit the threshold, the + session tears down via _protocol_error.""" + session, conn, _ = _make_session() + _activate(session, conn) + + def _boom(*args, **kwargs): + raise RuntimeError("simulated internal bug") + + session._server.on_fetch = _boom # type: ignore[assignment] + for _ in range(_MAX_CONSECUTIVE_DISPATCH_ERRORS): + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [0], + } + ) + session.poll() + assert not session.alive + assert any(m[TYPE_KEY] == DisconnectMsg.TYPE for m in conn._sent) + + def test_internal_error_counter_resets_on_success(self): + """A successful dispatch between errors prevents the threshold + from being reached.""" + session, conn, _ = _make_session() + _activate(session, conn) + + original_on_fetch = session._server.on_fetch + + def _boom(*args, **kwargs): + raise RuntimeError("simulated internal bug") + + # Alternate (boom, success) (_MAX-1) times: counter rises to 1 + # then resets to 0 each cycle, never reaching the threshold. + for _ in range(_MAX_CONSECUTIVE_DISPATCH_ERRORS - 1): + session._server.on_fetch = _boom # type: ignore[assignment] + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1"], + FetchMsg.BLOCK_INDEXES: [0], + } + ) + session.poll() + session._server.on_fetch = original_on_fetch # type: ignore[assignment] + # A benign no-op message (unknown type) dispatches cleanly + # and resets the consecutive-error counter. + conn.enqueue({TYPE_KEY: "unknown_for_test"}) + session.poll() + + assert session.alive + assert session._dispatch_error_count == 0 + + +class TestInflightPerReqInvariant: + """`_inflight_per_req` is the O(1) replacement for the previous + O(N) scan in `_has_inflight_for`. These tests check that every + mutation site keeps the counter in sync with `_inflight` and that + the lookup is correct under high fan-out. + """ + + def test_invariant_holds_through_lifecycle(self): + """Run a full submit→complete sequence for two concurrent + kv_request_ids and assert the counter matches `_inflight` at + every observable step, including the empty-after-finish case.""" + session, conn, transport = _make_session() + _activate(session, conn) + + def _invariant_holds() -> bool: + counted = sum(session._server._inflight_per_req.values()) + return counted == len(session._server._inflight) and all( + v > 0 for v in session._server._inflight_per_req.values() + ) + + assert _invariant_holds() + + # Two requests, two blocks each, all dispatched in one batch. + session.add_stored_blocks("req-A", [b"a1", b"a2"], [0, 1], job_id=10) + session.add_stored_blocks("req-B", [b"b1", b"b2"], [2, 3], job_id=11) + for kv_id, hashes, indexes in ( + ("req-A", [b"a1", b"a2"], [100, 101]), + ("req-B", [b"b1", b"b2"], [102, 103]), + ): + conn.enqueue( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: kv_id, + FetchMsg.BLOCK_HASHES: hashes, + FetchMsg.BLOCK_INDEXES: indexes, + } + ) + session.poll() + + assert _invariant_holds() + assert session._server._has_inflight_for("req-A") + assert session._server._has_inflight_for("req-B") + assert not session._server._has_inflight_for("req-C") + + # Complete req-A's transfer first; req-B should still be inflight. + a_tids = [ + tid + for tid, x in session._server._inflight.items() + if x.kv_request_id == "req-A" + ] + for tid in a_tids: + transport._poll_done.append(tid) + session.poll() + + assert _invariant_holds() + assert not session._server._has_inflight_for("req-A") + assert "req-A" not in session._server._inflight_per_req # entry was removed + assert session._server._has_inflight_for("req-B") + + # Complete req-B; counter must drain to empty. + b_tids = [ + tid + for tid, x in session._server._inflight.items() + if x.kv_request_id == "req-B" + ] + for tid in b_tids: + transport._poll_done.append(tid) + session.poll() + + assert _invariant_holds() + assert session._server._inflight == {} + assert session._server._inflight_per_req == {} + + def test_has_inflight_for_correct_with_many_requests(self): + """Populate many inflight xfers across many ids; lookup must + match the actual presence in `_inflight` for both hits and + misses. The whole point of the counter is that this lookup is + constant-time, but we assert correctness, not timing.""" + session, _, _ = _make_session() + for kv_id_idx in range(100): + kv_id = f"req-{kv_id_idx}" + for j in range(10): + tid = kv_id_idx * 10 + j + session._server._inflight_add( + tid, + _InflightXfer(kv_request_id=kv_id, block_count=1, job_ids={tid}), + ) + assert sum(session._server._inflight_per_req.values()) == len( + session._server._inflight + ) + assert session._server._has_inflight_for("req-0") + assert session._server._has_inflight_for("req-99") + assert not session._server._has_inflight_for("req-missing") + + # Drain all entries for req-50 and confirm the entry disappears. + tids_50 = [ + tid + for tid, x in session._server._inflight.items() + if x.kv_request_id == "req-50" + ] + for tid in tids_50: + session._server._inflight_pop(tid) + assert "req-50" not in session._server._inflight_per_req + assert not session._server._has_inflight_for("req-50") + # Other ids unaffected. + assert session._server._has_inflight_for("req-49") + + +# --------------------------------------------------------------------------- +# Protocol validation tests (unchanged from the prior file) +# --------------------------------------------------------------------------- + + +class TestConnectMsgValidation: + def _valid_msg(self) -> dict: + return { + TYPE_KEY: ConnectMsg.TYPE, + ConnectMsg.PEER_ID: "peer:1", + ConnectMsg.AGENT_METADATA: b"meta", + ConnectMsg.BASE_ADDR: 0x1000, + ConnectMsg.NUM_BLOCKS: 8, + ConnectMsg.BLOCK_LEN: 4096, + } + + def test_valid_message_passes(self): + ConnectMsg.validate(self._valid_msg()) + + def test_missing_peer_id(self): + msg = self._valid_msg() + del msg[ConnectMsg.PEER_ID] + with pytest.raises(ValueError, match="peer_id"): + ConnectMsg.validate(msg) + + def test_missing_agent_metadata(self): + msg = self._valid_msg() + del msg[ConnectMsg.AGENT_METADATA] + with pytest.raises(ValueError, match="agent_metadata"): + ConnectMsg.validate(msg) + + def test_agent_metadata_wrong_type(self): + msg = self._valid_msg() + msg[ConnectMsg.AGENT_METADATA] = "not bytes" + with pytest.raises(ValueError, match="agent_metadata"): + ConnectMsg.validate(msg) + + def test_base_addr_negative(self): + msg = self._valid_msg() + msg[ConnectMsg.BASE_ADDR] = -1 + with pytest.raises(ValueError, match="base_addr"): + ConnectMsg.validate(msg) + + def test_num_blocks_zero(self): + msg = self._valid_msg() + msg[ConnectMsg.NUM_BLOCKS] = 0 + with pytest.raises(ValueError, match="num_blocks"): + ConnectMsg.validate(msg) + + def test_block_len_zero(self): + msg = self._valid_msg() + msg[ConnectMsg.BLOCK_LEN] = 0 + with pytest.raises(ValueError, match="block_len"): + ConnectMsg.validate(msg) + + +class TestFetchMsgValidation: + def _valid_msg(self) -> dict: + return { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: "req-1", + FetchMsg.BLOCK_HASHES: [b"k1", b"k2"], + FetchMsg.BLOCK_INDEXES: [0, 1], + } + + def test_valid_message_passes(self): + FetchMsg.validate(self._valid_msg()) + + def test_length_mismatch(self): + msg = self._valid_msg() + msg[FetchMsg.BLOCK_INDEXES] = [0] + with pytest.raises(ValueError, match="length mismatch"): + FetchMsg.validate(msg) + + def test_negative_index(self): + msg = self._valid_msg() + msg[FetchMsg.BLOCK_INDEXES] = [0, -1] + with pytest.raises(ValueError, match="invalid index"): + FetchMsg.validate(msg) + + +class TestTransferDoneMsgValidation: + def test_valid_message_passes(self): + msg = { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: "req-1", + TransferDoneMsg.SUCCESS: True, + } + TransferDoneMsg.validate(msg) + + def test_success_wrong_type(self): + msg = { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: "req-1", + TransferDoneMsg.SUCCESS: 1, + } + with pytest.raises(ValueError, match="success"): + TransferDoneMsg.validate(msg) diff --git a/tests/v1/kv_offload/tiering/p2p/test_zmq_transport.py b/tests/v1/kv_offload/tiering/p2p/test_zmq_transport.py new file mode 100644 index 0000000000..101d4291cf --- /dev/null +++ b/tests/v1/kv_offload/tiering/p2p/test_zmq_transport.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for vllm.v1.kv_offload.tiering.p2p.control.zmq.""" + +from __future__ import annotations + +import socket +import time + +import pytest +import zmq + +from vllm.v1.kv_offload.tiering.p2p.control.zmq import ( + ZmqConnection, + ZmqTransport, + _Sockets, +) + + +def _free_port() -> int: + """Find a free TCP port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _make_transport(host: str = "127.0.0.1", attempts: int = 8): + """Construct a ZmqTransport on a fresh port, retrying on bind collisions. + + Why: _free_port() releases the probe socket before ZmqTransport binds the + same port — a parallel test run can steal it in between. Retrying on + ZMQError/OSError closes that race without a production change. + """ + last_err: Exception | None = None + for _ in range(attempts): + port = _free_port() + try: + return ZmqTransport(f"{host}:{port}", host, port), port + except (zmq.ZMQError, OSError) as e: + last_err = e + assert last_err is not None + raise last_err + + +def _wait_for_inbound(transport: ZmqTransport, deadline: float = 2.0): + """Poll until at least one new inbound connection is accepted, or fail.""" + end = time.monotonic() + deadline + while time.monotonic() < end: + new = transport.poll() + if new: + return new + time.sleep(0.005) + raise AssertionError(f"no inbound connection within {deadline}s") + + +def _wait_for_messages( + transport: ZmqTransport, + conn: ZmqConnection, + n: int, + deadline: float = 2.0, +) -> list[dict]: + """Poll until `conn` has received at least `n` messages, then return them.""" + end = time.monotonic() + deadline + msgs: list[dict] = [] + while time.monotonic() < end: + transport.poll() + msgs.extend(conn.recv()) + if len(msgs) >= n: + return msgs + time.sleep(0.005) + raise AssertionError(f"got {len(msgs)}/{n} messages within {deadline}s") + + +def _make_mock_connection(peer_id: str = "test:1234") -> ZmqConnection: + """Create a ZmqConnection with mock sockets for unit testing.""" + from unittest.mock import MagicMock + + sockets = _Sockets(dealer=MagicMock(), monitor=MagicMock()) + return ZmqConnection(peer_id, sockets) + + +class TestZmqConnection: + """Tests for ZmqConnection in isolation (no real sockets).""" + + def test_enqueue_and_recv(self): + """Messages enqueued are returned by recv() in order.""" + conn = _make_mock_connection() + + conn.enqueue({"type": "a"}) + conn.enqueue({"type": "b"}) + + msgs = conn.recv() + assert list(msgs) == [{"type": "a"}, {"type": "b"}] + # Second recv is empty + assert not conn.recv() + + def test_recv_returns_empty_initially(self): + conn = _make_mock_connection() + assert not conn.recv() + + def test_alive_initially_true(self): + conn = _make_mock_connection() + assert conn.alive is True + + def test_mark_dead(self): + conn = _make_mock_connection() + conn.mark_dead() + assert conn.alive is False + + def test_send_raises_when_closed(self): + conn = _make_mock_connection() + conn.mark_dead() + + with pytest.raises(RuntimeError, match="closed connection"): + conn.send({"type": "test"}) + + +class TestZmqTransportConnectivity: + """Integration tests for ZmqTransport with real ZMQ sockets.""" + + def test_connect_and_send_message(self): + """Two transports can connect and exchange messages.""" + transport_a, port_a = _make_transport() + transport_b, port_b = _make_transport() + + try: + peer_a_id = f"127.0.0.1:{port_a}" + conn_b_to_a = transport_b.connect(peer_a_id) + conn_b_to_a.send({"type": "hello", "data": 42}) + + new_conns = _wait_for_inbound(transport_a) + assert len(new_conns) == 1 + + conn_a_from_b = new_conns[0] + assert conn_a_from_b.peer_id == f"127.0.0.1:{port_b}" + + msgs = _wait_for_messages(transport_a, conn_a_from_b, 1) + assert msgs == [{"type": "hello", "data": 42}] + finally: + transport_a.close() + transport_b.close() + + def test_bidirectional_messaging(self): + """Both sides can send and receive after connection.""" + transport_a, port_a = _make_transport() + transport_b, _ = _make_transport() + + try: + conn_b = transport_b.connect(f"127.0.0.1:{port_a}") + conn_b.send({"type": "connect", "from": "b"}) + + new_conns = _wait_for_inbound(transport_a) + assert len(new_conns) == 1 + conn_a = new_conns[0] + + conn_a.send({"type": "reply", "from": "a"}) + + msgs = _wait_for_messages(transport_b, conn_b, 1) + assert msgs == [{"type": "reply", "from": "a"}] + finally: + transport_a.close() + transport_b.close() + + def test_poll_returns_empty_when_no_connections(self): + transport, _ = _make_transport() + try: + assert not transport.poll() + finally: + transport.close() + + def test_multiple_messages(self): + """Multiple messages are buffered and returned together.""" + transport_a, port_a = _make_transport() + transport_b, _ = _make_transport() + + try: + conn_b = transport_b.connect(f"127.0.0.1:{port_a}") + conn_b.send({"seq": 1}) + conn_b.send({"seq": 2}) + conn_b.send({"seq": 3}) + + new_conns = _wait_for_inbound(transport_a) + assert len(new_conns) == 1 + conn_a = new_conns[0] + + msgs = _wait_for_messages(transport_a, conn_a, 3) + assert [m["seq"] for m in msgs] == [1, 2, 3] + finally: + transport_a.close() + transport_b.close() + + def test_duplicate_connect_asserts(self): + """Connecting to the same peer twice raises AssertionError.""" + # port_a is never bound — we just need a syntactically-valid peer id. + port_a = _free_port() + transport_b, _ = _make_transport() + try: + transport_b.connect(f"127.0.0.1:{port_a}") + with pytest.raises(AssertionError, match="already exists"): + transport_b.connect(f"127.0.0.1:{port_a}") + finally: + transport_b.close() + + def test_dead_connection_removed_on_poll(self): + """Dead connections are cleaned up during poll.""" + transport_a, port_a = _make_transport() + transport_b, _ = _make_transport() + + try: + conn_b = transport_b.connect(f"127.0.0.1:{port_a}") + conn_b.send({"type": "hello"}) + + new_conns = _wait_for_inbound(transport_a) + assert len(new_conns) == 1 + + # Mark the inbound connection dead manually. + new_conns[0].mark_dead() + + # Pruning is synchronous within poll(). + transport_a.poll() + assert len(transport_a._connections) == 0 + finally: + transport_a.close() + transport_b.close() + + def test_close_is_idempotent(self): + """Calling close() twice doesn't raise.""" + transport, _ = _make_transport() + transport.close() + transport.close() # should not raise diff --git a/vllm/v1/kv_offload/tiering/factory.py b/vllm/v1/kv_offload/tiering/factory.py index ed69de9b27..180c87d094 100644 --- a/vllm/v1/kv_offload/tiering/factory.py +++ b/vllm/v1/kv_offload/tiering/factory.py @@ -66,6 +66,12 @@ SecondaryTierFactory.register_tier( "FileSystemTierManager", ) +SecondaryTierFactory.register_tier( + "p2p", + "vllm.v1.kv_offload.tiering.p2p.manager", + "P2PSecondaryTierManager", +) + SecondaryTierFactory.register_tier( "obj", "vllm.v1.kv_offload.tiering.obj.manager", diff --git a/vllm/v1/kv_offload/tiering/p2p/__init__.py b/vllm/v1/kv_offload/tiering/p2p/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vllm/v1/kv_offload/tiering/p2p/control/__init__.py b/vllm/v1/kv_offload/tiering/p2p/control/__init__.py new file mode 100644 index 0000000000..e8dbbe391b --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/control/__init__.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.v1.kv_offload.tiering.p2p.control.base import ( + ControlConnection, + ControlTransport, +) +from vllm.v1.kv_offload.tiering.p2p.control.zmq import ( + ZmqConnection, + ZmqTransport, +) + +__all__ = [ + "ControlConnection", + "ControlTransport", + "ZmqConnection", + "ZmqTransport", +] diff --git a/vllm/v1/kv_offload/tiering/p2p/control/base.py b/vllm/v1/kv_offload/tiering/p2p/control/base.py new file mode 100644 index 0000000000..6b4d4cfcb5 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/control/base.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Abstract base classes for the P2P control-plane transport. + +The control plane handles peer discovery, connection lifecycle, and +message routing. It is message-content agnostic — it moves opaque +dicts between peers without interpreting them. + +Architecture +------------ + + ControlTransport (one per node) + ├── listen for inbound connections + ├── connect() to outbound peers + └── poll() → new ControlConnections + + ControlConnection (one per peer) + ├── send(msg) — enqueue a message to the peer + ├── recv() — drain buffered inbound messages + ├── mark_dead() — signal that the peer is gone + └── close() — tear down the connection + +Threading model: all I/O is driven by the caller invoking poll(). +No background threads. poll() must be called periodically to: + - receive messages (buffered per-connection) + - accept new inbound peers + - detect disconnections + +Implementor contracts +--------------------- + +- ControlConnection.send() must not block. Messages are serialized + and queued for the next I/O pass. +- ControlConnection.recv() returns all messages received since the + last call (may be empty). Messages are dicts (already deserialized). +- ControlConnection.alive returns False after mark_dead() or close(). +- ControlTransport.poll() returns newly accepted connections only — + not previously returned ones. Each connection appears exactly once. +- ControlTransport.connect() creates an outbound connection to a peer + identified by peer_id (format: "host:port"). Raises on failure. +- Messages may arrive from unknown peers (new inbound connections). + The transport creates a ControlConnection and returns it from poll() + with the first message(s) already in its recv() buffer. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Sequence + + +class ControlConnection(ABC): + """Bidirectional message channel to a single remote peer. + + Lifecycle: + 1. Created by ControlTransport (connect() or poll()) + 2. Used for send/recv by sessions + 3. Marked dead on disconnect (mark_dead()) + 4. Cleaned up with close() + + Once mark_dead() is called, alive becomes False and the owning + session should stop using this connection. close() releases + underlying resources (sockets, monitors). + """ + + def __init__(self, peer_id: str) -> None: + self.peer_id = peer_id + + @property + @abstractmethod + def alive(self) -> bool: + """True if the connection is usable. False after mark_dead/close.""" + ... + + @abstractmethod + def send(self, msg: dict) -> None: + """Enqueue a message for delivery to the peer. + + Must not block. Serialization happens internally. + Raises on closed connection. + """ + ... + + @abstractmethod + def recv(self) -> Sequence[dict]: + """Drain and return all messages received since the last call. + + Returns an empty sequence if no messages are pending. + The returned sequence is read-only — callers must not mutate it. + Messages are dicts deserialized from the wire format. + """ + ... + + @abstractmethod + def mark_dead(self) -> None: + """Mark this connection as dead (peer disconnected). + + After this call, alive returns False. The session should + stop using this connection and the transport will clean it up. + """ + ... + + @abstractmethod + def close(self) -> None: + """Release all resources (sockets, monitors). + + Idempotent. After close(), alive returns False. + """ + ... + + +class ControlTransport(ABC): + """Manages peer connections and drives all control-plane I/O. + + Owns the listening socket and all active connections. + The caller must invoke poll() periodically to process I/O. + + Lifecycle: + 1. Constructed with a local identity and listen address + 2. connect() to reach remote peers + 3. poll() to accept inbound peers and process messages + 4. close() to shut down + """ + + @abstractmethod + def connect(self, peer_id: str) -> ControlConnection: + """Create an outbound connection to a remote peer. + + Args: + peer_id: Remote peer identity (format: "host:port"). + + Returns: + A new ControlConnection ready for send/recv. + + The connection's send queue is live immediately — messages + sent before the remote peer's poll() will be buffered. + """ + ... + + @abstractmethod + def poll(self) -> Sequence[ControlConnection]: + """Process all pending I/O and return newly accepted connections. + + This is the main I/O driver. Each call: + - Receives messages from all connected peers (buffered in + each connection's recv() queue) + - Accepts new inbound peers and creates connections for them + (first message already in recv() buffer) + - Detects disconnections and marks connections dead + + Returns: + Newly accepted inbound connections (not previously returned). + The returned sequence is read-only — callers must not mutate + it. The caller is responsible for creating sessions for + these connections. + """ + ... + + @abstractmethod + def close(self) -> None: + """Shut down the transport and all connections. + + Closes the listening socket and all active connections. + Idempotent. + """ + ... diff --git a/vllm/v1/kv_offload/tiering/p2p/control/zmq.py b/vllm/v1/kv_offload/tiering/p2p/control/zmq.py new file mode 100644 index 0000000000..4e9069a471 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/control/zmq.py @@ -0,0 +1,307 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +ZMQ-based transport layer for P2P KV cache sharing. + +Provides ZmqConnection (per-peer messaging) and ZmqTransport (connection +management). Message-content agnostic. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + +import msgspec +import zmq +import zmq.utils.monitor + +from vllm.logger import init_logger +from vllm.v1.kv_offload.tiering.p2p.control.base import ( + ControlConnection, + ControlTransport, +) + +logger = init_logger(__name__) + +_HEARTBEAT_IVL_MS = 2000 +_HEARTBEAT_TIMEOUT_MS = 10000 +_HEARTBEAT_TTL_MS = 10000 + +# Shared sentinels returned when there is nothing to report. +_EMPTY_INBOX: tuple[dict, ...] = () +_EMPTY_NEW_CONNECTIONS: tuple[ControlConnection, ...] = () + + +def _tcp_addr(host: str, port: int | str) -> str: + return f"tcp://{host}:{port}" + + +def _apply_heartbeat(sock: zmq.Socket) -> None: + sock.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS) + sock.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS) + sock.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS) + + +@dataclass +class _Sockets: + dealer: zmq.Socket + monitor: zmq.Socket + + +class ZmqConnection(ControlConnection): + """Bidirectional message channel to a single remote peer.""" + + def __init__(self, peer_id: str, sockets: _Sockets) -> None: + super().__init__(peer_id) + self._sockets = sockets + self._closed = False + self._inbox: list[dict] = [] + + def send(self, msg: dict) -> None: + """Send a msgpack-encoded message to this peer.""" + if self._closed: + raise RuntimeError( + f"ZmqConnection: send on closed connection to {self.peer_id}" + ) + data = msgspec.msgpack.encode(msg) + self._sockets.dealer.send(data) + + def recv(self) -> Sequence[dict]: + """Drain and return all buffered incoming messages.""" + if not self._inbox: + return _EMPTY_INBOX + msgs = self._inbox + self._inbox = [] + return msgs + + @property + def alive(self) -> bool: + return not self._closed + + def close(self) -> None: + if self._closed: + return + self._closed = True + logger.info("ZmqConnection: closing connection to %s", self.peer_id) + self._sockets.monitor.close() + self._sockets.dealer.close() + + def enqueue(self, msg: dict) -> None: + """Buffer an incoming message.""" + self._inbox.append(msg) + + def mark_dead(self) -> None: + """Mark connection as disconnected.""" + self._closed = True + + @property + def monitor_socket(self) -> zmq.Socket: + """Monitor socket for disconnect detection (used by ZmqTransport).""" + return self._sockets.monitor + + +class ZmqTransport(ControlTransport): + """ZMQ implementation of ControlTransport. + + Manages a ROUTER socket for accepting connections and DEALER sockets + for outbound connections. Message-content agnostic. + """ + + def __init__(self, local_id: str, host: str, port: int) -> None: + self._local_id = local_id + self._closed = False + + self._connections: dict[str, ZmqConnection] = {} + self._pending_inbound: list[tuple[str, dict]] = [] + + self._zmq_ctx = zmq.Context() + self._router: zmq.Socket = self._zmq_ctx.socket(zmq.ROUTER) + _apply_heartbeat(self._router) + bind_addr = _tcp_addr(host, port) + self._router.bind(bind_addr) + logger.info("ZmqTransport %s: ROUTER bound on %s", self._local_id, bind_addr) + + # ------------------------------------------------------------------ + # ZmqConnection lifecycle + # ------------------------------------------------------------------ + + def connect(self, peer_id: str) -> ZmqConnection: + """Open an outbound connection to a remote peer.""" + assert peer_id not in self._connections, ( + f"ZmqConnection to {peer_id} already exists" + ) + logger.info( + "ZmqTransport %s: opening OUTBOUND connection to %s", + self._local_id, + peer_id, + ) + return self._open_connection(peer_id, direction="outbound") + + def poll(self) -> Sequence[ControlConnection]: + """Process all pending I/O. Returns newly accepted connections. + + - Receives messages (buffered in each connection's inbox) + - Creates connections for new inbound peers (connect msg in inbox) + - Checks monitors for disconnections + - Removes and closes dead connections + """ + self._recv_router() + self._check_monitors() + + # Create connections for new inbound peers + new_connections: list[ControlConnection] | None = None + for sender_id, msg in self._pending_inbound: + conn = self._connections.get(sender_id) + if conn is None: + logger.info( + "ZmqTransport %s: accepting INBOUND connection from %s", + self._local_id, + sender_id, + ) + conn = self._open_connection(sender_id, direction="inbound") + if new_connections is None: + new_connections = [] + new_connections.append(conn) + conn.enqueue(msg) + self._pending_inbound.clear() + + # Remove dead connections + for pid in [p for p, c in self._connections.items() if not c.alive]: + self._connections.pop(pid).close() + + return ( + new_connections if new_connections is not None else _EMPTY_NEW_CONNECTIONS + ) + + def close(self) -> None: + if self._closed: + return + self._closed = True + + for conn in self._connections.values(): + conn.close() + self._connections.clear() + + self._router.setsockopt(zmq.LINGER, 0) + self._router.close() + self._zmq_ctx.destroy(linger=0) + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _open_connection( + self, peer_id: str, direction: str = "outbound" + ) -> ZmqConnection: + """Create a DEALER socket + monitor and register the connection.""" + host, port_str = peer_id.rsplit(":", 1) + dealer_addr = _tcp_addr(host, port_str) + + logger.debug( + "ZmqTransport %s: creating DEALER for %s peer %s -> %s", + self._local_id, + direction, + peer_id, + dealer_addr, + ) + + dealer = self._zmq_ctx.socket(zmq.DEALER) + _apply_heartbeat(dealer) + dealer.identity = self._local_id.encode() + + safe_id = peer_id.replace(":", "-").replace("/", "-") + monitor_addr = f"inproc://p2p-monitor-{safe_id}" + dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED) + + monitor_sock = self._zmq_ctx.socket(zmq.PAIR) + monitor_sock.connect(monitor_addr) + + dealer.connect(dealer_addr) + + sockets = _Sockets(dealer=dealer, monitor=monitor_sock) + conn = ZmqConnection(peer_id, sockets) + self._connections[peer_id] = conn + logger.info( + "ZmqTransport %s: %s connection established to %s (active connections: %d)", + self._local_id, + direction, + peer_id, + len(self._connections), + ) + return conn + + def _recv_router(self) -> None: + """Non-blocking: receive all pending messages from ROUTER.""" + while True: + try: + frames = self._router.recv_multipart(zmq.NOBLOCK) + except zmq.Again: + break + except zmq.ZMQError as exc: + logger.warning("ZmqTransport %s: recv error: %s", self._local_id, exc) + break + + if len(frames) != 2: + logger.warning( + "ZmqTransport %s: dropping message with %d frames (expected 2)", + self._local_id, + len(frames), + ) + continue + + identity, data = frames + sender_id = identity.decode() + + logger.debug( + "ZmqTransport %s: ROUTER recv from %s (%d bytes)", + self._local_id, + sender_id, + len(data), + ) + + try: + msg = msgspec.msgpack.decode(data) + except Exception as exc: + logger.warning( + "ZmqTransport %s: failed to decode message from %s: %s", + self._local_id, + sender_id, + exc, + ) + continue + + conn = self._connections.get(sender_id) + + if conn is not None: + conn.enqueue(msg) + else: + self._pending_inbound.append((sender_id, msg)) + + def _check_monitors(self) -> None: + """Non-blocking: check all monitor sockets for disconnection.""" + for conn in self._connections.values(): + if not conn.alive: + continue + try: + event = zmq.utils.monitor.recv_monitor_message( + conn.monitor_socket, zmq.NOBLOCK + ) + except zmq.Again: + continue + except zmq.ZMQError as exc: + logger.warning( + "ZmqTransport %s: monitor error for peer %s: %s", + self._local_id, + conn.peer_id, + exc, + ) + continue + + if event["event"] == zmq.EVENT_DISCONNECTED: + logger.debug( + "ZmqTransport %s: peer %s disconnected", + self._local_id, + conn.peer_id, + ) + conn.mark_dead() diff --git a/vllm/v1/kv_offload/tiering/p2p/data/__init__.py b/vllm/v1/kv_offload/tiering/p2p/data/__init__.py new file mode 100644 index 0000000000..5c7715e30c --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/data/__init__.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.v1.kv_offload.tiering.p2p.data.base import DataTransport, PollResult +from vllm.v1.kv_offload.tiering.p2p.data.nixl import NixlTransport + +__all__ = [ + "DataTransport", + "NixlTransport", + "PollResult", +] diff --git a/vllm/v1/kv_offload/tiering/p2p/data/base.py b/vllm/v1/kv_offload/tiering/p2p/data/base.py new file mode 100644 index 0000000000..a18b1ca0b3 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/data/base.py @@ -0,0 +1,259 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Abstract base class for the P2P data-plane transport. + +The data plane handles RDMA (or similar) block transfers between peers. +It is independent of the control plane — the control plane establishes +who can talk to whom, then the data plane moves blocks at wire speed. + +Architecture +------------ + + DataTransport (one per node) + ├── owns the local KV block memory region + ├── registers remote peers (add_remote_peer / remove_remote_peer) + ├── submits block writes (write_blocks → transfer_id) + ├── polls for completion (poll → done/failed IDs) + └── cancels inflight transfers (cancel) + +Memory model +------------ + +The local node exposes a contiguous block region: + + base_addr ──► ┌─────────────┐ block 0 + ├─────────────┤ block 1 + ├─────────────┤ ... + └─────────────┘ block (num_blocks - 1) + + Each block is block_len bytes. + +Remote peers expose the same layout. write_blocks() copies local +blocks to a remote peer's block region by index: + + write_blocks("peer:1", local_idxs=[0, 3], remote_idxs=[5, 7]) + → writes local block 0 → remote block 5 + local block 3 → remote block 7 + +Transfer lifecycle +------------------ + +1. Register remote peer: add_remote_peer(peer_id, metadata, ...) + - Provides the remote memory layout so transfers can target it +2. Submit: write_blocks(peer_id, local_idxs, remote_idxs) → int + - Returns a transfer_id (opaque int) for tracking + - Returns None if peer not registered or submission fails +3. Poll: poll() → PollResult(done=[...], failed=[...]) + - Returns transfer_ids that completed or failed since last poll + - Completed transfers are automatically cleaned up +4. Cancel: cancel(transfer_ids, mode="immediate" | "wait") + - Best-effort cancellation of inflight transfers + - mode="wait" returns ids still in PROC/PEND so the caller can poll + them to completion + +Implementor contracts +--------------------- + +- write_blocks() must not block. The transfer runs asynchronously. +- poll() must be called periodically. It drives completion checking. +- transfer_ids are unique across the lifetime of the transport. +- add_remote_peer() must be called before write_blocks() to that peer. +- get_agent_metadata() returns opaque bytes that the remote peer + needs to call add_remote_peer() (e.g., RDMA connection info). +- config_fingerprint is a content hash of the model configuration. + Peers with different fingerprints are incompatible and must not + exchange blocks (validated during the control-plane handshake). +- close() releases all resources (memory registrations, handles). + After close(), no other methods may be called. + +Threading model: no background threads. All I/O driven by poll(). +""" + +from __future__ import annotations + +import ctypes +import hashlib +import json +from abc import ABC, abstractmethod +from collections.abc import Iterable, Sequence +from typing import Literal, NamedTuple + +CancelMode = Literal["immediate", "wait"] + + +class PollResult(NamedTuple): + """Result of polling inflight transfers. + + Attributes: + done: Transfer IDs that completed successfully. + failed: Transfer IDs that failed (error, timeout, etc.). + """ + + done: Sequence[int] + failed: Sequence[int] + + +class DataTransport(ABC): + """Abstract data-plane transport for RDMA-style block transfers. + + Owns the local KV block memory region and manages transfers to/from + registered remote peers. + + Construction: + view: A 2D memoryview (num_blocks × block_len bytes) over the + local KV cache block storage. + config_fields: Dict of model config values used to compute the + compatibility fingerprint. None → empty fingerprint + (compatible with any peer). + """ + + def __init__(self, view: memoryview, config_fields: dict | None = None) -> None: + assert view.shape is not None + self._view = view + self._base_addr = ctypes.addressof(ctypes.c_char.from_buffer(view)) + self._num_blocks = view.shape[0] + self._block_len = view.shape[1] + self._config_fingerprint = self._compute_fingerprint(config_fields) + + @property + def base_addr(self) -> int: + """Base address of the local block memory region.""" + return self._base_addr + + @property + def num_blocks(self) -> int: + """Number of blocks in the local region.""" + return self._num_blocks + + @property + def block_len(self) -> int: + """Size of each block in bytes.""" + return self._block_len + + @property + def config_fingerprint(self) -> str: + """Content-hash of the model configuration (hex string). + + Peers must have matching fingerprints to exchange blocks. + Empty string means no fingerprint (always compatible). + """ + return self._config_fingerprint + + @staticmethod + def _compute_fingerprint(config_fields: dict | None) -> str: + if not config_fields: + return "" + canonical = json.dumps(config_fields, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode()).hexdigest()[:16] + + @abstractmethod + def get_agent_metadata(self) -> bytes: + """Return opaque metadata needed by remote peers to connect. + + The returned bytes are sent during the control-plane handshake + and passed to the remote peer's add_remote_peer(). + """ + ... + + @abstractmethod + def add_remote_peer( + self, + peer_id: str, + agent_metadata: bytes, + base_addr: int, + num_blocks: int, + block_len: int, + ) -> None: + """Register a remote peer for block transfers. + + Must be called before write_blocks() to this peer. + + Args: + peer_id: Unique identifier for the remote peer. + agent_metadata: Opaque bytes from the peer's get_agent_metadata(). + base_addr: Base address of the peer's block memory region. + num_blocks: Number of blocks in the peer's region. + block_len: Size of each block (must match local block_len). + """ + ... + + @abstractmethod + def remove_remote_peer(self, peer_id: str) -> None: + """Unregister a remote peer and release associated resources. + + Inflight transfers to this peer should be cancelled first. + """ + ... + + @abstractmethod + def write_blocks( + self, + peer_id: str, + local_idxs: list[int], + remote_idxs: list[int], + ) -> int | None: + """Submit a WRITE transfer: local blocks → remote peer's blocks. + + Args: + peer_id: Target peer (must be registered via add_remote_peer). + local_idxs: Indexes of local blocks to read from. + remote_idxs: Indexes of remote blocks to write to. + Must be same length as local_idxs. + + Returns: + A unique transfer_id (int) to track this transfer, or + None if the peer is not registered or submission failed. + """ + ... + + @abstractmethod + def poll(self) -> PollResult: + """Poll all inflight transfers for completion. + + Returns: + PollResult with lists of completed and failed transfer_ids. + Completed/failed transfers are removed from the inflight set. + + Must be called periodically to drive progress checking. + """ + ... + + @abstractmethod + def cancel( + self, + transfer_ids: Iterable[int], + mode: CancelMode = "immediate", + ) -> list[int]: + """Cancel inflight transfers by their IDs. + + Best-effort: transfers that already completed are ignored. + + Args: + transfer_ids: IDs to cancel. Unknown IDs are ignored. + mode: + "immediate" (default): pop and release each handle and + return []. Matches the legacy fire-and-forget + behavior — the caller does not wait for the + underlying transfer to drain. + "wait": attempt to release each handle. If the release + cannot complete because the transfer is still + PROC/PEND, the entry stays in the inflight set and + its id is included in the returned list. The + caller is expected to keep calling poll() until + every returned id surfaces in done/failed. + + Returns: + For mode="wait", the subset of *transfer_ids* still + tracked as inflight after the cancel attempt. For + mode="immediate", always []. + """ + ... + + @abstractmethod + def close(self) -> None: + """Release all resources (registrations, handles, memory). + + Cancels any remaining inflight transfers. Idempotent. + After close(), no other methods may be called. + """ + ... diff --git a/vllm/v1/kv_offload/tiering/p2p/data/nixl.py b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py new file mode 100644 index 0000000000..5f283c4815 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/data/nixl.py @@ -0,0 +1,297 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL. +""" + +from __future__ import annotations + +import itertools +from collections.abc import Iterable +from typing import Any + +from vllm.distributed.nixl_utils import NixlWrapper as _NixlAgent +from vllm.distributed.nixl_utils import nixl_agent_config as _NixlAgentConfig +from vllm.logger import init_logger +from vllm.v1.kv_offload.tiering.p2p.data.base import ( + CancelMode, + DataTransport, + PollResult, +) + +logger = init_logger(__name__) + +# Shared sentinel returned by poll() in the steady state (no inflight, or +# no transfer changed state since the last poll). Tuples make it immutable; +# callers only iterate / membership-test / equality-check. +_EMPTY_POLL_RESULT: PollResult = PollResult(done=(), failed=()) + + +class NixlTransport(DataTransport): + """Manages a NIXL agent, memory registration, and block transfers. + + Wraps the NIXL C library behind a Python interface so the rest of the + P2P tier code never touches NIXL types directly. Tracks inflight + handles internally and returns completed/failed tags on poll. + """ + + def __init__( + self, + local_id: str, + view: memoryview, + config_fields: dict | None = None, + backends: list[str] | None = None, + num_threads: int = 4, + ) -> None: + super().__init__(view, config_fields=config_fields) + self._local_id = local_id + self._backends = list(backends) if backends else ["UCX"] + self._num_threads = num_threads + self._agent: Any = None + self._reg: Any = None + self._local_dlist: Any = None + self._remote_dlists: dict[str, object] = {} + self._peer_nixl_names: dict[str, str] = {} + self._inflight: dict[int, object] = {} # transfer_id → handle + self._next_id = itertools.count() + + self._init(view) + + @property + def available(self) -> bool: + return self._agent is not None + + def _init(self, view: memoryview) -> None: + if _NixlAgent is None: + return + + non_ucx_backends = [b for b in self._backends if b != "UCX"] + if non_ucx_backends: + cfg = _NixlAgentConfig(backends=self._backends, capture_telemetry=True) + logger.info( + "NixlTransport %s: NIXL backends=%s", + self._local_id, + self._backends, + ) + else: + cfg = _NixlAgentConfig( + num_threads=self._num_threads, capture_telemetry=True + ) + logger.info( + "NixlTransport %s: NIXL backends=[UCX] num_threads=%d", + self._local_id, + self._num_threads, + ) + self._agent = _NixlAgent(self._local_id, cfg) + + total_size = self._num_blocks * self._block_len + reg_descs = [(self._base_addr, total_size, 0, "")] + self._reg = self._agent.register_memory(reg_descs, mem_type="DRAM") + + block_tuples = [ + (self._base_addr + i * self._block_len, self._block_len, 0) + for i in range(self._num_blocks) + ] + xfer_dlist = self._agent.get_xfer_descs(block_tuples, mem_type="DRAM") + self._local_dlist = self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", xfer_dlist) + logger.info( + "NixlTransport %s: registered %d blocks", self._local_id, self._num_blocks + ) + + def get_agent_metadata(self) -> bytes: + assert self._agent is not None + return self._agent.get_agent_metadata() + + # ------------------------------------------------------------------ + # Peer management + # ------------------------------------------------------------------ + + def add_remote_peer( + self, + peer_id: str, + agent_metadata: bytes, + base_addr: int, + num_blocks: int, + block_len: int, + ) -> None: + nixl_name = self._agent.add_remote_agent(agent_metadata) + block_descs = [ + (base_addr + i * block_len, block_len, 0) for i in range(num_blocks) + ] + xfer_dlist = self._agent.get_xfer_descs(block_descs, mem_type="DRAM") + remote_dlist = self._agent.prep_xfer_dlist(nixl_name, xfer_dlist) + self._peer_nixl_names[peer_id] = nixl_name + self._remote_dlists[peer_id] = remote_dlist + + def remove_remote_peer(self, peer_id: str) -> None: + nixl_name = self._peer_nixl_names.pop(peer_id, None) + dlist = self._remote_dlists.pop(peer_id, None) + if self._agent is not None: + if dlist is not None: + self._agent.release_dlist_handle(dlist) + if nixl_name: + self._agent.remove_remote_agent(nixl_name) + + # ------------------------------------------------------------------ + # Transfer submission and polling + # ------------------------------------------------------------------ + + def write_blocks( + self, + peer_id: str, + local_idxs: list[int], + remote_idxs: list[int], + ) -> int | None: + """Submit a WRITE transfer to *peer_id*. + + Returns a transfer ID, or None if the peer is not registered. + The ID is returned via poll() when the transfer completes or fails. + """ + remote_dlist = self._remote_dlists.get(peer_id) + if remote_dlist is None: + logger.warning( + "NixlTransport %s: write_blocks NO REMOTE DLIST for peer=%s " + "(known peers=%s)", + self._local_id, + peer_id, + list(self._remote_dlists.keys()), + ) + return None + logger.debug( + "NixlTransport %s: write_blocks NIXL.transfer peer=%s blocks=%d", + self._local_id, + peer_id, + len(local_idxs), + ) + handle = self._agent.make_prepped_xfer( + "WRITE", + self._local_dlist, + local_idxs, + remote_dlist, + remote_idxs, + ) + self._agent.transfer(handle) + transfer_id = next(self._next_id) + self._inflight[transfer_id] = handle + return transfer_id + + def poll(self) -> PollResult: + """Poll all inflight transfers. + + Returns PollResult(done=..., failed=...) with transfer IDs. + Completed handles are released automatically. + """ + if not self._inflight: + return _EMPTY_POLL_RESULT + + done_ids: list[int] | None = None + failed_ids: list[int] | None = None + + for transfer_id, handle in self._inflight.items(): + try: + state = self._agent.check_xfer_state(handle) + except Exception as exc: + logger.warning( + "NixlTransport %s: check_xfer_state failed for transfer_id=%d: %s", + self._local_id, + transfer_id, + exc, + ) + continue + if state == "DONE": + if done_ids is None: + done_ids = [] + done_ids.append(transfer_id) + elif state not in ("PROC", "PEND"): + if failed_ids is None: + failed_ids = [] + failed_ids.append(transfer_id) + + if done_ids is None and failed_ids is None: + return _EMPTY_POLL_RESULT + + handles_to_release = [] + for tid in done_ids or (): + handles_to_release.append(self._inflight.pop(tid)) + for tid in failed_ids or (): + handles_to_release.append(self._inflight.pop(tid)) + self._release_handles(handles_to_release) + + return PollResult( + done=done_ids if done_ids is not None else _EMPTY_POLL_RESULT.done, + failed=failed_ids if failed_ids is not None else _EMPTY_POLL_RESULT.failed, + ) + + def cancel( + self, + transfer_ids: Iterable[int], + mode: CancelMode = "immediate", + ) -> list[int]: + """Cancel inflight transfers by their IDs. + + See ``DataTransport.cancel`` for the contract. In "wait" mode, + transfers whose ``release_xfer_handle`` raises (NIXL could not + complete the abort because the backend is still draining) stay + in ``self._inflight`` so a later ``poll()`` will observe them. + """ + if mode == "immediate": + handles = [ + self._inflight.pop(tid) for tid in transfer_ids if tid in self._inflight + ] + self._release_handles(handles) + return [] + + still_inflight: list[int] = [] + for tid in transfer_ids: + handle = self._inflight.get(tid) + if handle is None: + continue + try: + self._agent.release_xfer_handle(handle) + except Exception as exc: + logger.debug( + "NixlTransport %s: cancel pending for transfer_id=%d: %s", + self._local_id, + tid, + exc, + ) + still_inflight.append(tid) + continue + del self._inflight[tid] + return still_inflight + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def close(self) -> None: + if self._agent is None: + return + self._release_handles(list(self._inflight.values())) + self._inflight.clear() + for peer_id in list(self._remote_dlists): + self.remove_remote_peer(peer_id) + if self._local_dlist is not None: + self._agent.release_dlist_handle(self._local_dlist) + self._local_dlist = None + if self._reg is not None: + self._agent.deregister_memory(self._reg) + self._reg = None + self._agent = None + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _release_handles(self, handles: list[object]) -> None: + if self._agent is None: + return + for handle in handles: + try: + self._agent.release_xfer_handle(handle) + except Exception as exc: + logger.warning( + "NixlTransport %s: release_xfer_handle failed: %s", + self._local_id, + exc, + ) diff --git a/vllm/v1/kv_offload/tiering/p2p/manager.py b/vllm/v1/kv_offload/tiering/p2p/manager.py new file mode 100644 index 0000000000..c6a64268ad --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/manager.py @@ -0,0 +1,664 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +P2PSecondaryTierManager: Secondary tier for P2P KV cache sharing. + +Owns transports and a single bidirectional P2PSession per remote peer. +""" + +from __future__ import annotations + +import time +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from typing_extensions import override + +from vllm.logger import init_logger +from vllm.v1.kv_offload.base import ( + LookupResult, + OffloadKey, + ReqContext, + RequestOffloadingContext, +) +from vllm.v1.kv_offload.file_mapper import FileMapper +from vllm.v1.kv_offload.tiering.base import ( + JobMetadata, + JobResult, + SecondaryTierManager, +) +from vllm.v1.kv_offload.tiering.p2p.control import ControlTransport, ZmqTransport +from vllm.v1.kv_offload.tiering.p2p.data import DataTransport, NixlTransport +from vllm.v1.kv_offload.tiering.p2p.session import P2PSession + +if TYPE_CHECKING: + from vllm.v1.kv_offload.base import OffloadingSpec + from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection + +logger = init_logger(__name__) + +# Reap unbound store batches that have been parked without a FetchMsg +# binding them to a session for longer than this. Protects against the +# prefiller buffering blocks for a decoder that never asks (decoder died, +# network partition, lost kv_request_id). Must be longer than the per-store +# deadline so the store-timeout path fires first for individual jobs. +_UNBOUND_STORE_TIMEOUT_S = 60.0 + +# Time we wait during shutdown for inflight transfers to drain via +# cancel(mode="wait") before falling back to mode="immediate". Bounded +# so a wedged peer can't hang shutdown. +_SHUTDOWN_DRAIN_TIMEOUT_S = 3.0 + +# Sleep between iterations of the bounded drain loops in drain_jobs() and +# _drain_inflight_for_shutdown(). Short enough to keep latency low, long +# enough to avoid busy-spinning the scheduler thread. +_DRAIN_SLEEP_S = 0.001 + + +def _prefill_params(kv_params: dict | None) -> dict | None: + """Return the ``prefill`` sub-dict, or None if absent. + + Set on decoder requests; carries kv_request_id, remote_host, remote_port. + """ + if not kv_params: + return None + return kv_params.get("prefill") + + +def _decode_params(kv_params: dict | None) -> dict | None: + """Return the ``decode`` sub-dict, or None if absent. + + Set on prefiller requests; carries kv_request_id. + """ + if not kv_params: + return None + return kv_params.get("decode") + + +@dataclass +class _UnboundStoreBatch: + """A submit_store batch parked at the manager before any peer has fetched. + + Indexed by kv_request_id only — the prefiller no longer learns the peer + identity at store time. When a FetchMsg(kv_request_id) arrives on some + session, the manager binds the kv_request_id to that session and replays + every parked batch into ServerRole via session.add_stored_blocks. + """ + + job_id: int + keys: list[OffloadKey] + block_ids: Sequence[int] + submitted_at: float = field(default_factory=time.monotonic) + + +class P2PSecondaryTierManager(SecondaryTierManager): + """Secondary tier for P2P KV cache sharing. + + A single P2PSession per remote peer handles both client-role (loading + blocks from the peer) and server-role (serving blocks to the peer) + over the same control connection. + + Single-threaded: every public method runs on the scheduler thread, and + the engine drives polling via ``get_finished_jobs()`` once per step. + ``has_pending_work()`` keeps the engine ticking so the control transport + and existing sessions are polled even when no requests are scheduled. + """ + + def __init__( + self, + offloading_spec: OffloadingSpec, + primary_kv_view: memoryview, + tier_type: str = "p2p", + host: str = "0.0.0.0", + port: int = 7777, + backends: list[str] | None = None, + num_threads: int = 4, + **kwargs, + ) -> None: + """Initialize the P2P secondary tier manager. + + All keyword arguments after ``primary_kv_view`` come from the + ``secondary_tiers`` entry in ``kv_connector_extra_config``. See + ``docs/features/kv_offloading_usage.md`` for the user-facing + configuration reference. + + Args: + offloading_spec: Owning ``OffloadingSpec`` (provides + ``vllm_config`` and the offloaded block layout). + primary_kv_view: Memoryview over the CPU primary tier; the + NIXL agent registers this region for RDMA transfers. + tier_type: Tier identifier (defaults to ``"p2p"``). + host: Address the ZMQ control socket binds to. + port: Port for the ZMQ control socket. Must be reachable + from peers. + backends: NIXL transport backends (e.g. ``["UCX"]``, + ``["MOONCAKE"]``, ``["LIBFABRIC"]``). Defaults to + ``["UCX"]``. When any non-UCX backend is requested, the + NIXL agent is initialized with ``backends=...``; + otherwise it falls back to a UCX-only agent with + ``num_threads`` threads. + num_threads: NIXL agent worker threads for the UCX-only + branch. Ignored when ``backends`` contains a non-UCX + entry. + **kwargs: Reserved for future tier-specific options. + """ + super().__init__(offloading_spec, primary_kv_view, tier_type) + port = int(port) + self._local_id = f"{host}:{port}" + + config_fields = FileMapper.from_offloading_spec( + root_dir="", + offloading_spec=offloading_spec, + gpu_blocks_per_file=offloading_spec.block_size_factor, + parallel_agnostic=True, + ).get_run_config() + self._data: DataTransport = NixlTransport( + self._local_id, + primary_kv_view, + config_fields=config_fields, + backends=backends, + num_threads=int(num_threads), + ) + self._control: ControlTransport = ZmqTransport(self._local_id, host, port) + + self._sessions: dict[str, P2PSession] = {} + # kv_request_id → session, set when the bound session has received + # FetchMsg for that id. submit_store after binding routes directly + # to the session; before binding, batches are parked in + # _unbound_stores below. Stays in sync with _sessions: entries + # pointing to a reaped session are purged in _reap_dead_sessions. + self._kv_to_session: dict[str, P2PSession] = {} + # kv_request_id → list of batches submit_store'd before any peer + # asked for that id. Drained into a session by _on_session_fetch + # when the corresponding FetchMsg arrives, or surfaced as failures + # by _reap_unbound_stores after _UNBOUND_STORE_TIMEOUT_S. + self._unbound_stores: dict[str, list[_UnboundStoreBatch]] = {} + + self._finished_jobs: list[JobResult] = [] + # kv_request_ids that hit a transport/session failure; On load lookup() + # rejects them so the request falls back to local prefill. + self._failed_req_ids: set[str] = set() + + # ------------------------------------------------------------------ + # SecondaryTierManager interface + # ------------------------------------------------------------------ + + @override + def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: + prefill = _prefill_params(req_context.kv_transfer_params) + if ( + not prefill + or not prefill.get("remote_host") + or not prefill.get("remote_port") + or not prefill.get("kv_request_id") + ): + return LookupResult.MISS + + kv_request_id = prefill["kv_request_id"] + if kv_request_id in self._failed_req_ids: + return LookupResult.MISS + return LookupResult.HIT + + @override + def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: + """Open the outbound session toward the producer if needed. + + On the decoder side (``prefill`` set), open a session toward the + producer at remote_host:remote_port so submit_load can issue + FetchMsg as soon as it fires. On the prefiller side, sessions + are created when the consumer's inbound connection arrives in + _accept_new_peers — submit_store no longer pre-creates anything. + """ + prefill = _prefill_params(req_context.kv_transfer_params) + if prefill: + peer_id = self._remote_id_from_params(prefill) + if peer_id: + self._get_or_create_session(peer_id) + return RequestOffloadingContext() + + @override + def on_request_finished(self, req_context: ReqContext) -> None: + """Cancels pending loads and prunes session-scoped state. + + Decoder side (``prefill`` set): looks up the session by peer_id + because the producer's address is what addresses the client-role + load to cancel. Prefiller side (``decode`` set): looks up via + kv_request_id because peer_id is no longer carried on store-time + kv_transfer_params; if a session has bound the id, finish it. If + no session has bound the id yet, this is a no-op: parked batches + in `_unbound_stores` are left in place and cleaned up only by + `_reap_unbound_stores` after `_UNBOUND_STORE_TIMEOUT_S`. + """ + kv_params = req_context.kv_transfer_params + if not kv_params: + return + prefill = _prefill_params(kv_params) + decode = _decode_params(kv_params) + kv_request_id = (prefill or decode or {}).get("kv_request_id") + if not kv_request_id: + return + self._failed_req_ids.discard(kv_request_id) + + if prefill: + peer_id = self._remote_id_from_params(prefill) + if peer_id: + session = self._sessions.get(peer_id) + if session is not None: + session.finish_request(kv_request_id) + return + + # Prefiller-side finish: identify the session via kv_request_id. + session = self._kv_to_session.pop(kv_request_id, None) + if session is not None: + session.finish_request(kv_request_id) + return + + @override + def submit_store(self, job_metadata: JobMetadata) -> None: + job_id = job_metadata.job_id + keys = list(job_metadata.keys) + block_ids = job_metadata.block_ids + + assert len(keys) == len(block_ids) + + kv_params = job_metadata.req_context.kv_transfer_params + decode = _decode_params(kv_params) + logger.debug( + "P2P %s: submit_store ENTRY job_id=%d blocks=%d decode=%s kv_request_id=%s", + self._local_id, + job_id, + len(block_ids), + decode is not None, + (decode or {}).get("kv_request_id"), + ) + # Absent ``decode`` block => not a remote-decode request: succeed + # locally without parking. An empty/malformed dict is still a + # remote-decode signal and must fail the missing-id check below. + if decode is None: + self._finished_jobs.append(JobResult(job_id=job_id, success=True)) + return + + kv_request_id = decode.get("kv_request_id") + if not kv_request_id: + logger.warning( + "P2P %s: submit_store missing kv_request_id", + self._local_id, + ) + self._finished_jobs.append(JobResult(job_id=job_id, success=False)) + return + + # Fast path: a session has already received FetchMsg for this id, + # so we can route the batch straight into its ServerRole. + session = self._kv_to_session.get(kv_request_id) + if session is not None: + session.add_stored_blocks(kv_request_id, keys, block_ids, job_id) + return + + # No session bound yet — park the batch keyed by kv_request_id. + # _on_session_fetch drains it on the first FetchMsg; if no peer + # ever asks, _reap_unbound_stores surfaces the job as failed. + self._unbound_stores.setdefault(kv_request_id, []).append( + _UnboundStoreBatch( + job_id=job_id, + keys=keys, + block_ids=block_ids, + ) + ) + logger.debug( + "P2P %s: parked submit_store kv_request_id=%s job_id=%d blocks=%d", + self._local_id, + kv_request_id, + job_id, + len(block_ids), + ) + + @override + def submit_load(self, job_metadata: JobMetadata) -> None: + job_id = job_metadata.job_id + keys = list(job_metadata.keys) + block_ids = job_metadata.block_ids + + prefill = _prefill_params(job_metadata.req_context.kv_transfer_params) + logger.debug( + "P2P %s: submit_load ENTRY job_id=%d blocks=%d kv_request_id=%s peer=%s", + self._local_id, + job_id, + len(block_ids), + (prefill or {}).get("kv_request_id"), + self._remote_id_from_params(prefill or {}), + ) + if ( + not prefill + or not prefill.get("remote_host") + or not prefill.get("remote_port") + or not prefill.get("kv_request_id") + ): + logger.debug( + "P2P %s: submit_load job_id=%d FAILED missing prefill params", + self._local_id, + job_id, + ) + self._finished_jobs.append(JobResult(job_id=job_id, success=False)) + return + + kv_request_id = prefill["kv_request_id"] + peer_id = self._remote_id_from_params(prefill) + assert peer_id is not None # guaranteed by prefill checks above + + if not keys: + logger.debug( + "P2P %s: submit_load job_id=%d short-circuit success (no keys)", + self._local_id, + job_id, + ) + self._finished_jobs.append(JobResult(job_id=job_id, success=True)) + return + + session = self._sessions.get(peer_id) + if session is None: + logger.warning( + "P2P %s: submit_load job_id=%d NO SESSION for peer=%s", + self._local_id, + job_id, + peer_id, + ) + self._finished_jobs.append(JobResult(job_id=job_id, success=False)) + self._failed_req_ids.add(kv_request_id) + return + logger.debug( + "P2P %s: submit_load job_id=%d -> request_blocks peer=%s " + "kv_request_id=%s blocks=%d session_ready=%s", + self._local_id, + job_id, + peer_id, + kv_request_id, + len(block_ids), + session.ready, + ) + session.request_blocks(job_id, kv_request_id, keys, block_ids) + + @override + def get_finished_jobs(self) -> Iterable[JobResult]: + # Drive one polling sweep on the scheduler thread, then hand off + # whatever has accumulated. The engine calls this once per step + # (and keeps stepping while has_pending_work() is True). + self._poll_once() + result = self._finished_jobs + self._finished_jobs = [] + return result + + @override + def has_pending_work(self) -> bool: + # The engine tick is the only driver of _control.poll() and + # session.poll(); without it we miss new peer connects and + # inbound fetch messages on existing sessions. Keep the engine + # ticking for the lifetime of this manager. + return True + + @override + def drain_jobs(self) -> None: + """Block until every submitted load/store job has completed or failed. + + Loops calling ``_poll_once()`` until no session has outstanding + inbound loads or in-flight outbound stores. Mid-flight transfers + are NOT cancelled — the caller (``TieringOffloadingManager.reset_cache``) + needs the primary memoryview to be quiescent, not aborted. Results + accumulate in ``_finished_jobs`` and are surfaced by the next + ``get_finished_jobs()`` call. + """ + start = time.monotonic() + warned = False + while True: + self._poll_once() + pending = any( + s._client._inbound or s._server._inflight + for s in self._sessions.values() + ) + if not pending: + return + if not warned and time.monotonic() - start > 5.0: + logger.warning( + "P2PSecondaryTierManager.drain_jobs: still draining " + "after 5s; a stuck transfer will block the engine.", + ) + warned = True + time.sleep(_DRAIN_SLEEP_S) + + @override + def on_schedule_end(self) -> None: + return + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + @staticmethod + def _remote_id_from_params(role_params: dict) -> str | None: + """Build peer_id from a role-scoped sub-dict (``prefill``/``p2p``).""" + host = role_params.get("remote_host") + port = role_params.get("remote_port") + if host and port: + return f"{host}:{port}" + return None + + def _get_or_create_session(self, peer_id: str) -> P2PSession: + """Return the existing session for peer_id, or open one outbound. + + Decoder-side helper for on_new_request: when ``prefill`` is set, + the consumer must reach the producer at peer_id. If we already + have a session toward that peer (from a prior load or a + peer-initiated inbound), reuse it; otherwise open an outbound + ControlConnection and build a connected session. + """ + session = self._sessions.get(peer_id) + if session is not None: + return session + conn = self._control.connect(peer_id) + session = P2PSession( + peer_id=peer_id, + local_id=self._local_id, + transport=self._data, + local_block_len=self._data.block_len, + conn=conn, + ) + self._sessions[peer_id] = session + return session + + def _accept_new_peers(self, new_connections: Sequence[ControlConnection]) -> None: + for conn in new_connections: + logger.info( + "P2P %s: accepting incoming connection from %s", + self._local_id, + conn.peer_id, + ) + try: + existing = self._sessions.get(conn.peer_id) + if existing is not None: + raise ValueError(f"duplicate connection from {conn.peer_id}") + self._sessions[conn.peer_id] = P2PSession( + peer_id=conn.peer_id, + local_id=self._local_id, + transport=self._data, + local_block_len=self._data.block_len, + conn=conn, + ) + logger.info( + "P2P %s: created connected session for %s", + self._local_id, + conn.peer_id, + ) + except (ValueError, KeyError, TypeError, AssertionError) as exc: + logger.error("P2P %s: rejecting peer: %s", self._local_id, exc) + conn.close() + + def _reap_dead_sessions(self) -> None: + # Reap connected sessions whose connection died — peer is gone. + # Stranded prefiller-side stores are no longer tracked through a + # session (they live in _unbound_stores keyed by kv_request_id); + # _reap_unbound_stores handles their timeout independently. + dead: list[str] | None = None + for pid, s in self._sessions.items(): + if s.connected and not s.alive: + if dead is None: + dead = [] + dead.append(pid) + if dead is None: + return + for pid in dead: + session = self._sessions.pop(pid) + # Purge any kv_request_id → session entries pointing at this + # session so subsequent submit_stores fall back to the unbound + # path (which will time out into failure if no peer rebinds). + stale_kv_ids = [ + kid for kid, s in self._kv_to_session.items() if s is session + ] + for kid in stale_kv_ids: + del self._kv_to_session[kid] + failed_loads, failed_stores = session.close() + for job_id, kv_request_id in failed_loads: + self._finished_jobs.append(JobResult(job_id=job_id, success=False)) + self._failed_req_ids.add(kv_request_id) + for job_id in failed_stores: + self._finished_jobs.append(JobResult(job_id=job_id, success=False)) + self._data.remove_remote_peer(pid) + logger.warning("P2P %s: peer %s down", self._local_id, pid) + + def _reap_unbound_stores(self) -> None: + """Time out submit_store batches that no peer has ever fetched. + + Walks `_unbound_stores` for entries whose oldest batch is older + than `_UNBOUND_STORE_TIMEOUT_S`. Drops the kv_request_id, surfaces + every batched job as failed, and adds the id to `_failed_req_ids` + so a late inbound FetchMsg short-circuits to a clean rejection. + """ + if not self._unbound_stores: + return + deadline = time.monotonic() - _UNBOUND_STORE_TIMEOUT_S + expired: list[str] | None = None + for kid, batches in self._unbound_stores.items(): + # Batches are appended in arrival order, so the head is oldest. + if batches and batches[0].submitted_at <= deadline: + if expired is None: + expired = [] + expired.append(kid) + if expired is None: + return + for kid in expired: + batches = self._unbound_stores.pop(kid) + self._failed_req_ids.add(kid) + for batch in batches: + self._finished_jobs.append( + JobResult(job_id=batch.job_id, success=False) + ) + logger.warning( + "P2P %s: unbound store kv_request_id=%s timed out after %.0fs " + "without a fetch — failing %d job(s)", + self._local_id, + kid, + _UNBOUND_STORE_TIMEOUT_S, + len(batches), + ) + + # ------------------------------------------------------------------ + # Polling + # ------------------------------------------------------------------ + + def _poll_once(self) -> None: + """One sweep of the polling work. + + Drains the control transport, polls every session, accumulates + their results into ``_finished_jobs``, and reaps any dead sessions. + Runs on the scheduler thread. + """ + new_connections = self._control.poll() + if new_connections: + logger.info( + "P2P %s: _poll_once got %d new connection(s): %s", + self._local_id, + len(new_connections), + [c.peer_id for c in new_connections], + ) + + self._accept_new_peers(new_connections) + + for session in self._sessions.values(): + result = session.poll() + for lr in result.loads: + self._finished_jobs.append( + JobResult(job_id=lr.job_id, success=lr.success) + ) + if not lr.success: + self._failed_req_ids.add(lr.kv_request_id) + for sr in result.stores: + self._finished_jobs.append( + JobResult(job_id=sr.job_id, success=sr.success) + ) + # Bind kv_request_id → session for any FetchMsg this tick and + # replay any submit_store batches parked while no peer was + # asking. ServerRole.on_fetch already recorded the demand + # inline in dispatch, so the replayed add_stored_blocks calls + # match that demand and submit transfers immediately. + for kv_request_id in result.new_fetch_ids: + self._kv_to_session[kv_request_id] = session + for batch in self._unbound_stores.pop(kv_request_id, ()): + session.add_stored_blocks( + kv_request_id, batch.keys, batch.block_ids, batch.job_id + ) + + self._reap_dead_sessions() + self._reap_unbound_stores() + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + @override + def shutdown(self) -> None: + self._drain_inflight_for_shutdown() + for session in self._sessions.values(): + session.close() + self._sessions.clear() + self._kv_to_session.clear() + # Surface buffered store jobs as failed so the engine doesn't + # leak them; the manager is going away after this call. + for batches in self._unbound_stores.values(): + for batch in batches: + self._finished_jobs.append( + JobResult(job_id=batch.job_id, success=False) + ) + self._unbound_stores.clear() + self._control.close() + self._data.close() + + def _drain_inflight_for_shutdown(self) -> None: + """Best-effort drain of inflight transfers before closing _data. + + Mirrors session._drain_abort but as a single bounded loop. Collects + inflight transfer_ids from each session, repeatedly calls + _data.cancel(..., mode="wait") and _data.poll() so handles can + surface as done/failed, and falls back to mode="immediate" once + _SHUTDOWN_DRAIN_TIMEOUT_S elapses so a wedged peer can't hang us. + """ + ids = [tid for s in self._sessions.values() for tid in s._server._inflight] + if not ids: + return + deadline = time.monotonic() + _SHUTDOWN_DRAIN_TIMEOUT_S + still: list[int] = ids + while still and time.monotonic() < deadline: + still = list(self._data.cancel(still, mode="wait")) + if not still: + break + # poll() advances NIXL handle state so the next wait-cancel + # has a chance to release the handles. + self._data.poll() + time.sleep(_DRAIN_SLEEP_S) + if still: + logger.warning( + "P2P %s: shutdown drain timed out after %.1fs with %d " + "transfers still inflight — force-cancelling", + self._local_id, + _SHUTDOWN_DRAIN_TIMEOUT_S, + len(still), + ) + self._data.cancel(still, mode="immediate") diff --git a/vllm/v1/kv_offload/tiering/p2p/session/__init__.py b/vllm/v1/kv_offload/tiering/p2p/session/__init__.py new file mode 100644 index 0000000000..82148e8340 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/session/__init__.py @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.v1.kv_offload.tiering.p2p.session.session import ( + LoadResult, + P2PSession, + SessionPollResult, + StoreResult, +) + +__all__ = [ + "LoadResult", + "P2PSession", + "SessionPollResult", + "StoreResult", +] diff --git a/vllm/v1/kv_offload/tiering/p2p/session/client.py b/vllm/v1/kv_offload/tiering/p2p/session/client.py new file mode 100644 index 0000000000..fce5008deb --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/session/client.py @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Client-role state machine for a single peer session. + +Handles outgoing fetch requests, abort-on-timeout, abort-ack timeout, +and produces ``LoadResult`` for completed loads. The session coordinator +parses wire messages and dispatches typed arguments here; this module +never touches ``ControlConnection`` directly — it emits via the ``send`` +callback injected by the coordinator (which gates on ConnectAck). +""" + +from __future__ import annotations + +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, NamedTuple + +from vllm.logger import init_logger +from vllm.v1.kv_offload.tiering.p2p.session.protocol import ( + TYPE_KEY, + AbortFetchMsg, + FetchMsg, +) + +if TYPE_CHECKING: + from vllm.v1.kv_offload.tiering.base import JobId + +logger = init_logger(__name__) + +_LOAD_TIMEOUT_S = 30.0 +_ABORT_ACK_TIMEOUT_S = 10.0 + + +@dataclass +class _InboundRequestState: + """Client-role state for a single load request.""" + + job_id: int # opaque ID assigned by the manager to this load request + kv_request_id: str + submitted_at: float + aborted_at: float | None = None + + +class LoadResult(NamedTuple): + """Result from a session poll, client side.""" + + job_id: int + kv_request_id: str + success: bool + + +class ClientRole: + """Client-side load state machine for one peer session. + + The coordinator owns the connection and the send-gating; this role + is given a ``send`` callback and a ``peer_id`` for log messages and + is otherwise self-contained. + """ + + def __init__(self, peer_id: str, send: Callable[[dict], None]) -> None: + self._peer_id = peer_id + self._send = send + self._inbound: dict[str, _InboundRequestState] = {} + self._completed_loads: list[LoadResult] = [] + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def request_blocks( + self, + job_id: JobId, + kv_request_id: str, + keys: Sequence[bytes], + block_ids: Sequence[int], + send_ready: bool, + ) -> None: + """Register a load request and send the FetchMsg.""" + logger.debug( + "P2PSession %s: request_blocks job_id=%d kv_request_id=%s " + "blocks=%d ready=%s", + self._peer_id, + job_id, + kv_request_id, + len(block_ids), + send_ready, + ) + self._inbound[kv_request_id] = _InboundRequestState( + job_id=job_id, + kv_request_id=kv_request_id, + submitted_at=time.monotonic(), + ) + self._send( + { + TYPE_KEY: FetchMsg.TYPE, + FetchMsg.KV_REQUEST_ID: kv_request_id, + FetchMsg.BLOCK_HASHES: list(keys), + FetchMsg.BLOCK_INDEXES: [int(idx) for idx in block_ids], + } + ) + + def cancel(self, kv_request_id: str) -> None: + """Cancel a pending load. Sends AbortFetchMsg if still active.""" + req = self._inbound.pop(kv_request_id, None) + if req is not None and req.aborted_at is None: + self._send( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: kv_request_id, + } + ) + + def on_transfer_done(self, kv_request_id: str, success: bool) -> None: + """Handle a TransferDoneMsg from the peer.""" + req = self._inbound.pop(kv_request_id, None) + if req is not None: + self._completed_loads.append( + LoadResult( + job_id=req.job_id, + kv_request_id=kv_request_id, + success=success, + ) + ) + else: + # No matching _inbound entry: either a duplicate + # transfer_done from the peer (protocol violation) or a + # benign race with a local cancel/abort/timeout that + # already popped the entry. We don't track terminated ids, + # so we can't tell — log so it's findable. + logger.warning( + "P2PSession %s: transfer_done for unknown kv_request_id=%s " + "(duplicate from peer, or raced with local cancel/timeout)", + self._peer_id, + kv_request_id, + ) + + def on_abort_ack(self, kv_request_id: str) -> None: + """Handle an AbortAckMsg from the peer.""" + req = self._inbound.pop(kv_request_id, None) + if req is not None: + self._completed_loads.append( + LoadResult( + job_id=req.job_id, + kv_request_id=kv_request_id, + success=False, + ) + ) + else: + # See on_transfer_done: same ambiguity (duplicate ack + # vs. raced with local cancel/timeout that already popped). + logger.warning( + "P2PSession %s: abort_ack for unknown kv_request_id=%s " + "(duplicate from peer, or raced with local cancel/timeout)", + self._peer_id, + kv_request_id, + ) + + def collect_results(self) -> list[LoadResult]: + """Walk timeouts and drain completed loads. + + Active requests past ``_LOAD_TIMEOUT_S`` get an AbortFetchMsg + sent and enter the aborting phase. Aborting requests past + ``_ABORT_ACK_TIMEOUT_S`` are surfaced as failed loads. + """ + now = time.monotonic() + to_remove: list[str] = [] + for req_id, req in self._inbound.items(): + if req.aborted_at is None: + if now - req.submitted_at >= _LOAD_TIMEOUT_S: + req.aborted_at = now + logger.warning( + "P2PSession %s: %s timed out, sending abort", + self._peer_id, + req_id, + ) + self._send( + { + TYPE_KEY: AbortFetchMsg.TYPE, + AbortFetchMsg.KV_REQUEST_ID: req_id, + } + ) + else: + if now - req.aborted_at >= _ABORT_ACK_TIMEOUT_S: + to_remove.append(req_id) + self._completed_loads.append( + LoadResult( + job_id=req.job_id, + kv_request_id=req_id, + success=False, + ) + ) + logger.warning( + "P2PSession %s: abort_ack timed out for kv_request_id=%s", + self._peer_id, + req_id, + ) + for req_id in to_remove: + self._inbound.pop(req_id) + + results = self._completed_loads + self._completed_loads = [] + return results + + def close(self) -> list[tuple[int, str]]: + """Tear down. Returns ``(job_id, kv_request_id)`` for pending loads.""" + failed = [(req.job_id, req.kv_request_id) for req in self._inbound.values()] + self._inbound.clear() + self._completed_loads.clear() + return failed diff --git a/vllm/v1/kv_offload/tiering/p2p/session/protocol.py b/vllm/v1/kv_offload/tiering/p2p/session/protocol.py new file mode 100644 index 0000000000..3a2ab5fd88 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/session/protocol.py @@ -0,0 +1,247 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +P2P KV cache sharing protocol constants and documentation. + +Protocol Overview +================= + +Two session types communicate over a bidirectional message channel: +- P2PClientSession (requests blocks from a server) +- P2PServerSession (serves blocks to a client) + +Connection Lifecycle +-------------------- + +1. Client opens a connection and sends ConnectMsg with its identity, + RDMA metadata, memory layout, and config fingerprint. +2. Server validates block_len and config_fingerprint, registers the + RDMA peer, and replies with ConnectAckMsg. +3. Client receives ConnectAckMsg and transitions to ready state + (flushes any queued messages). +4. Either side may send DisconnectMsg to gracefully close. + +Block Transfer Flow (happy path) +--------------------------------- + +1. Client sends FetchMsg with a kv_request_id and lists of + block keys + remote indexes where it wants the data written. +2. Server matches requested blocks against locally stored blocks: + - Blocks already available are transferred immediately via RDMA. + - Blocks not yet available are recorded as "demanded" and + transferred when the server later stores them. +3. When all blocks for a kv_request_id are transferred, the server + sends TransferDoneMsg (success=True) to the client. +4. Client reports the load job as complete. + +Abort Flow (timeout path) +-------------------------- + +1. If the client times out waiting for TransferDoneMsg, it sends + AbortFetchMsg to cancel the request. +2. Server cancels inflight transfers for that kv_request_id and + replies with AbortAckMsg. +3. Client receives AbortAckMsg and reports the load job as failed. +4. If AbortAckMsg itself times out, the client fails the job anyway. + +Message Format +-------------- + +All messages are dicts serialized with msgpack. Every message has a +TYPE_KEY key identifying its type. Additional fields depend on the +message type (see per-message class docstrings below). + +Security +-------- + +Sessions wrap all incoming message handling in try/except to guard +against malformed messages from adversarial or buggy peers. Invalid +messages are logged and dropped without crashing the session. + +The config_fingerprint field in ConnectMsg ensures peers have +compatible model configurations (model, dtype, block sizes). Mismatches +are rejected during the handshake. +""" + +TYPE_KEY = "type" + + +# --------------------------------------------------------------------------- +# Validation helpers +# --------------------------------------------------------------------------- + + +def _require(msg: dict, key: str, typ: type, *, name: str = "") -> None: + """Raise ValueError if msg[key] is missing or not isinstance(typ).""" + val = msg.get(key) + if not isinstance(val, typ): + label = name or key + raise ValueError(f"{label}: expected {typ.__name__}, got {type(val).__name__}") + + +def _require_pos_int(msg: dict, key: str, *, name: str = "") -> None: + """Raise ValueError if msg[key] is not a positive int.""" + val = msg.get(key) + if not isinstance(val, int) or val <= 0: + label = name or key + raise ValueError(f"{label}: expected positive int, got {val!r}") + + +def _require_non_neg_int(msg: dict, key: str, *, name: str = "") -> None: + """Raise ValueError if msg[key] is not a non-negative int.""" + val = msg.get(key) + if not isinstance(val, int) or val < 0: + label = name or key + raise ValueError(f"{label}: expected non-negative int, got {val!r}") + + +def _require_list(msg: dict, key: str, *, name: str = "") -> None: + """Raise ValueError if msg[key] is not a list.""" + val = msg.get(key) + if not isinstance(val, list): + label = name or key + raise ValueError(f"{label}: expected list, got {type(val).__name__}") + + +# --------------------------------------------------------------------------- +# Message classes +# --------------------------------------------------------------------------- + + +class ConnectMsg: + """Client → Server: initial handshake request. + + Fields: + PEER_ID: Local peer identity string. + AGENT_METADATA: RDMA agent metadata (opaque bytes). + BASE_ADDR: Base memory address of the KV block region. + NUM_BLOCKS: Number of blocks in the KV block region. + BLOCK_LEN: Size in bytes of each block (must match between peers). + CONFIG_FINGERPRINT: SHA-256 prefix of the model configuration. + Peers with different fingerprints are incompatible. + """ + + TYPE = "connect" + PEER_ID = "peer_id" + AGENT_METADATA = "agent_metadata" + BASE_ADDR = "base_addr" + NUM_BLOCKS = "num_blocks" + BLOCK_LEN = "block_len" + CONFIG_FINGERPRINT = "config_fingerprint" + + @staticmethod + def validate(msg: dict) -> None: + """Raise ValueError if any field has an invalid type or value.""" + _require(msg, ConnectMsg.PEER_ID, str) + _require(msg, ConnectMsg.AGENT_METADATA, bytes) + _require_non_neg_int(msg, ConnectMsg.BASE_ADDR) + _require_pos_int(msg, ConnectMsg.NUM_BLOCKS) + _require_pos_int(msg, ConnectMsg.BLOCK_LEN) + + +class ConnectAckMsg: + """Server → Client: handshake acknowledgement. + + Fields: + PEER_ID: Server's peer identity string. + """ + + TYPE = "connect_ack" + PEER_ID = "peer_id" + + @staticmethod + def validate(msg: dict) -> None: + """Raise ValueError if any field has an invalid type or value.""" + _require(msg, ConnectAckMsg.PEER_ID, str) + + +class DisconnectMsg: + """Either → Either: graceful connection close. + + No additional fields beyond TYPE_KEY. + """ + + TYPE = "disconnect" + + +class FetchMsg: + """Client → Server: request blocks by key. + + Fields: + KV_REQUEST_ID: Identifies this block transfer request. + BLOCK_HASHES: List of block keys (OffloadKey bytes). + BLOCK_INDEXES: List of remote block indexes (same length as BLOCK_HASHES). + """ + + TYPE = "fetch" + KV_REQUEST_ID = "kv_request_id" + BLOCK_HASHES = "block_hashes" + BLOCK_INDEXES = "block_indexes" + + @staticmethod + def validate(msg: dict) -> None: + """Raise ValueError if any field has an invalid type or value.""" + _require(msg, FetchMsg.KV_REQUEST_ID, str) + _require_list(msg, FetchMsg.BLOCK_HASHES) + _require_list(msg, FetchMsg.BLOCK_INDEXES) + hashes = msg[FetchMsg.BLOCK_HASHES] + indexes = msg[FetchMsg.BLOCK_INDEXES] + if len(hashes) != len(indexes): + raise ValueError( + f"block_hashes/block_indexes length mismatch: " + f"{len(hashes)} vs {len(indexes)}" + ) + for idx in indexes: + if not isinstance(idx, int) or idx < 0: + raise ValueError(f"block_indexes: invalid index {idx!r}") + + +class TransferDoneMsg: + """Server → Client: all blocks transferred for a request. + + Fields: + KV_REQUEST_ID: The request that completed. + SUCCESS: Whether the transfer completed successfully. + """ + + TYPE = "transfer_done" + KV_REQUEST_ID = "kv_request_id" + SUCCESS = "success" + + @staticmethod + def validate(msg: dict) -> None: + """Raise ValueError if any field has an invalid type or value.""" + _require(msg, TransferDoneMsg.KV_REQUEST_ID, str) + _require(msg, TransferDoneMsg.SUCCESS, bool) + + +class AbortFetchMsg: + """Client → Server: cancel a pending request. + + Fields: + KV_REQUEST_ID: The request to cancel. + """ + + TYPE = "abort_fetch" + KV_REQUEST_ID = "kv_request_id" + + @staticmethod + def validate(msg: dict) -> None: + """Raise ValueError if any field has an invalid type or value.""" + _require(msg, AbortFetchMsg.KV_REQUEST_ID, str) + + +class AbortAckMsg: + """Server → Client: acknowledge cancellation. + + Fields: + KV_REQUEST_ID: The request that was cancelled. + """ + + TYPE = "abort_ack" + KV_REQUEST_ID = "kv_request_id" + + @staticmethod + def validate(msg: dict) -> None: + """Raise ValueError if any field has an invalid type or value.""" + _require(msg, AbortAckMsg.KV_REQUEST_ID, str) diff --git a/vllm/v1/kv_offload/tiering/p2p/session/server.py b/vllm/v1/kv_offload/tiering/p2p/session/server.py new file mode 100644 index 0000000000..6f5059e550 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/session/server.py @@ -0,0 +1,601 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Server-role state machine for a single peer session. + +Owns block matching (supply vs. demand), inflight RDMA transfers, store-job +timeouts, abort-drain, and produces ``StoreResult`` for completed stores. +The session coordinator parses wire messages and dispatches typed +arguments here; this module never touches ``ControlConnection`` directly +— it emits via the ``send`` callback injected by the coordinator (which +gates on ConnectAck). + +Protocol violations the role can detect (today: duplicate ``FetchMsg`` +for the same ``kv_request_id``) are surfaced as ``ValueError`` so the +coordinator's ``_dispatch_message`` can reuse its existing +``_protocol_error`` path. +""" + +from __future__ import annotations + +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, NamedTuple + +from vllm.logger import init_logger +from vllm.v1.kv_offload.base import OffloadKey +from vllm.v1.kv_offload.tiering.p2p.session.protocol import ( + TYPE_KEY, + AbortAckMsg, + TransferDoneMsg, +) + +if TYPE_CHECKING: + from vllm.v1.kv_offload.tiering.base import JobId + from vllm.v1.kv_offload.tiering.p2p.data import DataTransport + +logger = init_logger(__name__) + +_STORE_TIMEOUT_S = 30.0 +_CANCEL_DRAIN_TIMEOUT_S = 10.0 + + +class StoreResult(NamedTuple): + """Result from a session poll, server side.""" + + job_id: int + success: bool + + +class _InflightXfer(NamedTuple): + """Metadata for a single inflight RDMA transfer, keyed by transfer_id.""" + + kv_request_id: str + block_count: int + # The set of store job IDs that contributed blocks to this transfer. + job_ids: set[int] + + +class _MatchResult(NamedTuple): + """Result of block matching: pairs ready for transfer.""" + + local_idxs: list[int] + remote_idxs: list[int] + # The set of store job IDs that contributed blocks + job_ids: set[int] + + +@dataclass +class _OutboundRequestState: + """Server-role state for a single peer fetch request. + + The owning ``kv_request_id`` is the dict key in + ``ServerRole._outbound`` and is not duplicated on the value. + """ + + demand_received: bool = False + available: dict[OffloadKey, tuple[int, int]] = field( + default_factory=dict + ) # key → (job_id, local_block_idx): blocks we have, awaiting demand + demanded: dict[OffloadKey, int] = field( + default_factory=dict + ) # key → remote_block_idx: blocks peer wants, awaiting supply + remaining: int = 0 # blocks that need to be transferred to client + finishing: bool = False # Signal finish request ASAP + # Job IDs that submit_store'd blocks for this request and have not + # yet emitted a StoreResult. The terminal-finalize helper drains + # this set; poll-done and poll-failed discard entries as their + # StoreResults fire. + pending_job_ids: set[int] = field(default_factory=set) + + def add_stored_blocks( + self, + block_hashes: Sequence[OffloadKey], + block_ids: Sequence[int], + job_id: int, + ) -> _MatchResult: + """Add locally-stored blocks. Returns matched pairs.""" + self.pending_job_ids.add(job_id) + local_idxs: list[int] = [] + remote_idxs: list[int] = [] + for block_hash, local_idx in zip(block_hashes, block_ids): + remote_idx = self.demanded.pop(block_hash, None) + if remote_idx is not None: + local_idxs.append(local_idx) + remote_idxs.append(remote_idx) + else: + self.available[block_hash] = (job_id, local_idx) + return _MatchResult( + local_idxs=local_idxs, + remote_idxs=remote_idxs, + job_ids={job_id} if local_idxs else set(), + ) + + def add_fetch_demand( + self, + block_hashes: Sequence[OffloadKey], + block_indexes: Sequence[int], + ) -> _MatchResult: + """Register the peer's fetch demand. Returns matched pairs.""" + self.demand_received = True + self.remaining = len(block_hashes) + + local_idxs: list[int] = [] + remote_idxs: list[int] = [] + job_ids: set[int] = set() + for block_hash, remote_idx in zip(block_hashes, block_indexes): + stored_entry = self.available.pop(block_hash, None) + if stored_entry is not None: + stored_job_id, local_idx = stored_entry + local_idxs.append(local_idx) + remote_idxs.append(remote_idx) + job_ids.add(stored_job_id) + else: + self.demanded[block_hash] = remote_idx + return _MatchResult( + local_idxs=local_idxs, + remote_idxs=remote_idxs, + job_ids=job_ids, + ) + + +class ServerRole: + """Server-side store/serve state machine for one peer session. + + The coordinator owns the connection and the send-gating; this role + is given a ``send`` callback, the ``DataTransport``, and the + ``peer_id`` for transport calls and log messages. + """ + + def __init__( + self, + peer_id: str, + transport: DataTransport, + send: Callable[[dict], None], + ) -> None: + self._peer_id = peer_id + self._transport = transport + self._send = send + + self._outbound: dict[str, _OutboundRequestState] = {} + # transfer_id → xfer. Mutate ONLY via _inflight_add / _inflight_pop + # so the per-request count below stays in sync. + self._inflight: dict[int, _InflightXfer] = {} + # kv_request_id → number of entries in _inflight for that id. + # Kept in sync with _inflight; entries that hit zero are removed + # so `kv_request_id in self._inflight_per_req` is an exact + # "has any inflight transfer" predicate (O(1) replacement for + # the previous O(N) scan). + self._inflight_per_req: dict[str, int] = {} + self._store_jobs: dict[int, float] = {} # job_id → submitted_at + self._pending_aborts: dict[str, float] = {} # kv_request_id → start + # StoreResults queued by _finalize_outbound for the next poll + # tick to surface. Mirrors the deferred-result pattern used for + # load timeouts. + self._pending_store_results: list[StoreResult] = [] + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def add_stored_blocks( + self, + kv_request_id: str, + keys: Sequence[OffloadKey], + block_ids: Sequence[int], + job_id: JobId, + ) -> None: + """New blocks stored locally — match against pending fetch demand.""" + self._store_jobs[job_id] = time.monotonic() + req = self._outbound.setdefault(kv_request_id, _OutboundRequestState()) + result = req.add_stored_blocks(keys, block_ids, job_id) + if result.local_idxs and req.demand_received: + self._submit_transfer(kv_request_id, result) + + def on_fetch( + self, + kv_request_id: str, + block_hashes: Sequence[OffloadKey], + block_indexes: Sequence[int], + ) -> None: + """Handle a FetchMsg from the peer. + + Raises ``ValueError`` on a duplicate fetch for the same + ``kv_request_id``; the coordinator's dispatch loop turns that + into a protocol-error disconnect. + """ + logger.debug( + "P2PSession %s: fetch RECEIVED kv_request_id=%s blocks=%d", + self._peer_id, + kv_request_id, + len(block_hashes), + ) + existing = self._outbound.get(kv_request_id) + if existing is not None and existing.demand_received: + # A second fetch for the same kv_request_id would overwrite + # `remaining` and leak inflight bookkeeping. Treat as a + # protocol violation. + raise ValueError(f"duplicate fetch for kv_request_id={kv_request_id}") + req = self._outbound.setdefault(kv_request_id, _OutboundRequestState()) + result = req.add_fetch_demand(block_hashes, block_indexes) + if result.local_idxs: + self._submit_transfer(kv_request_id, result) + # Prefiller-first mode: finish_request may have run before + # fetch arrived. If so, finalize once we know what was + # demanded — fully satisfied → success, else early-fail. + if req.finishing and not self._has_inflight_for(kv_request_id): + self._finalize_outbound(kv_request_id) + + def on_abort_fetch(self, kv_request_id: str) -> None: + """Handle an AbortFetchMsg from the peer.""" + # Abort for an unknown id may be a benign race/duplicate or a + # real protocol violation; we don't track completed ids, so warn. + if kv_request_id not in self._outbound and not self._has_inflight_for( + kv_request_id + ): + logger.warning( + "P2PSession %s: abort_fetch for unknown kv_request_id=%s " + "(no outbound or inflight state); benign race or stale", + self._peer_id, + kv_request_id, + ) + # Idempotent: receiving AbortFetchMsg again before we've sent the + # ack just triggers another drain attempt without resetting the + # deadline. + self._pending_aborts.setdefault(kv_request_id, time.monotonic()) + self._drain_abort(kv_request_id) + + def finish(self, kv_request_id: str) -> None: + """Mark an outbound request finishing. + + No more submit_store calls will arrive for this id. Any blocks + the peer demanded but we never stored will never come; tell the + peer to stop waiting (TransferDoneMsg success=False) instead of + letting it hit _LOAD_TIMEOUT_S. + + If the decoder hasn't sent fetch yet (no demand received), + defer — on_fetch will finalize once demand arrives. + + If inflight transfers exist for this id, defer — the last + completing transfer in collect_results will fire the message. + """ + req = self._outbound.get(kv_request_id) + if req is None: + return + req.finishing = True + if not req.demand_received: + return + if self._has_inflight_for(kv_request_id): + return + # Remaining > 0 here: if it had hit 0, the poll-done success + # branch would have already popped _outbound and we'd have + # returned at `req is None` above. Helper derives success from + # remaining and emits StoreResult(success=False) for any + # leftover pending jobs. + self._finalize_outbound(kv_request_id) + + def collect_results(self) -> list[StoreResult]: + """Drain timeouts, deferred results, and transport completions.""" + results: list[StoreResult] = self._timeout_pending_store_jobs() + + if self._pending_store_results: + results.extend(self._pending_store_results) + self._pending_store_results.clear() + + poll_result = self._transport.poll() + + for tid in poll_result.done: + xfer = self._inflight_pop(tid) + if xfer is None: + # Bug signal: transport reported a transfer we have no + # bookkeeping for. Likely a double-completion in the + # transport or a stale removal in the session. The + # attached job(s) still live in _store_jobs and will be + # surfaced as failures by _timeout_pending_store_jobs + # after _STORE_TIMEOUT_S, but log loudly so the + # underlying bug is findable. + logger.error( + "P2PSession %s: transport reported done for unknown " + "transfer_id=%d; attached job(s) will fail via " + "store-timeout instead of completing now", + self._peer_id, + tid, + ) + continue + req = self._outbound.get(xfer.kv_request_id) + for job_id in xfer.job_ids: + if self._store_jobs.pop(job_id, None) is None: + # Already reported (timeout, cancellation, etc.) — + # don't double-emit a contradictory success result. + continue + results.append(StoreResult(job_id=job_id, success=True)) + if req is not None: + req.pending_job_ids.discard(job_id) + if req is not None and req.demand_received: + req.remaining -= xfer.block_count + assert req.remaining >= 0, ( + f"remaining went negative for kv_request_id={xfer.kv_request_id}" + ) + if req.remaining == 0: + self._finalize_outbound(xfer.kv_request_id, success=True) + elif req.finishing and not self._has_inflight_for(xfer.kv_request_id): + self._finalize_outbound(xfer.kv_request_id, success=False) + + failed_kv_request_ids: set[str] | None = None + for tid in poll_result.failed: + xfer = self._inflight_pop(tid) + if xfer is None: + # See the matching error log in the done branch above. + logger.error( + "P2PSession %s: transport reported failed for unknown " + "transfer_id=%d; attached job(s) will fail via " + "store-timeout instead of completing now", + self._peer_id, + tid, + ) + continue + if failed_kv_request_ids is None: + failed_kv_request_ids = set() + failed_kv_request_ids.add(xfer.kv_request_id) + req_for_xfer = self._outbound.get(xfer.kv_request_id) + for job_id in xfer.job_ids: + if self._store_jobs.pop(job_id, None) is None: + # Already reported (timeout, cancellation, etc.) — + # don't double-emit. + continue + results.append(StoreResult(job_id=job_id, success=False)) + if req_for_xfer is not None: + req_for_xfer.pending_job_ids.discard(job_id) + req = self._outbound.pop(xfer.kv_request_id, None) + if req is not None and req.demand_received: + self._send( + { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: xfer.kv_request_id, + TransferDoneMsg.SUCCESS: False, + } + ) + + # Cancel other inflight for the same failed kv_request_ids + if failed_kv_request_ids: + ids_to_cancel = [ + tid + for tid, xfer in self._inflight.items() + if xfer.kv_request_id in failed_kv_request_ids + ] + for tid in ids_to_cancel: + self._inflight_pop(tid) + self._transport.cancel(ids_to_cancel) + + return results + + def collect_idle_timeouts(self) -> list[StoreResult]: + """Run only the store-job timeout sweep. + + Used by the coordinator's no-conn poll path: a pending session + cannot have inflight transfers (no peer registered yet), so we + skip the transport poll and the deferred-result drain. + """ + return self._timeout_pending_store_jobs() + + def drain_pending_aborts(self) -> None: + """Re-attempt every parked abort once per poll tick.""" + if not self._pending_aborts: + return + for kv_request_id in list(self._pending_aborts): + self._drain_abort(kv_request_id) + + def close(self) -> list[int]: + """Tear down. Cancels inflight, returns failed store job ids.""" + failed_stores = list(self._store_jobs.keys()) + self._store_jobs.clear() + if self._inflight: + self._transport.cancel(list(self._inflight.keys())) + self._inflight.clear() + self._inflight_per_req.clear() + self._outbound.clear() + self._pending_aborts.clear() + self._pending_store_results.clear() + return failed_stores + + # ------------------------------------------------------------------ + # Internal — inflight bookkeeping + # ------------------------------------------------------------------ + + def _has_inflight_for(self, kv_request_id: str) -> bool: + return kv_request_id in self._inflight_per_req + + def _inflight_add(self, tid: int, xfer: _InflightXfer) -> None: + """Insert an inflight transfer and bump the per-request count.""" + self._inflight[tid] = xfer + self._inflight_per_req[xfer.kv_request_id] = ( + self._inflight_per_req.get(xfer.kv_request_id, 0) + 1 + ) + + def _inflight_pop(self, tid: int) -> _InflightXfer | None: + """Pop an inflight transfer and decrement the per-request count. + + Removes the per-request entry once the count hits zero so the + dict stays bounded and `_has_inflight_for` remains exact. + """ + xfer = self._inflight.pop(tid, None) + if xfer is None: + return None + new_count = self._inflight_per_req.get(xfer.kv_request_id, 0) - 1 + if new_count > 0: + self._inflight_per_req[xfer.kv_request_id] = new_count + else: + self._inflight_per_req.pop(xfer.kv_request_id, None) + return xfer + + # ------------------------------------------------------------------ + # Internal — finalize / abort drain + # ------------------------------------------------------------------ + + def _finalize_outbound( + self, + kv_request_id: str, + success: bool | None = None, + ) -> None: + """Pop the outbound state and emit terminal results. + + Called when no further work will happen for this kv_request_id + on the server side: either request_finish has fired and there + are no inflight transfers, or the last inflight just completed + while finishing. + + If ``success`` is None, derive it from ``req.remaining == 0``. + The same flag is used for both the peer's TransferDoneMsg and + the StoreResult(s) emitted for any leftover pending job_ids. + """ + req = self._outbound.pop(kv_request_id, None) + if req is None: + return + if success is None: + success = req.remaining == 0 + for job_id in req.pending_job_ids: + self._store_jobs.pop(job_id, None) + self._pending_store_results.append( + StoreResult(job_id=job_id, success=success) + ) + self._send( + { + TYPE_KEY: TransferDoneMsg.TYPE, + TransferDoneMsg.KV_REQUEST_ID: kv_request_id, + TransferDoneMsg.SUCCESS: success, + } + ) + + def _drain_abort(self, kv_request_id: str) -> None: + """One drain attempt for a pending abort. + + Stops accepting more blocks for ``kv_request_id``, then asks the + transport to cancel any matching inflight transfers in + ``mode="wait"``. Sends ``AbortAckMsg`` once nothing remains + inflight, or after ``_CANCEL_DRAIN_TIMEOUT_S`` falls back to + ``mode="immediate"`` and acks anyway. + """ + self._outbound.pop(kv_request_id, None) + ids = [ + tid + for tid, xfer in self._inflight.items() + if xfer.kv_request_id == kv_request_id + ] + if not ids: + self._finalize_abort(kv_request_id) + return + + started_at = self._pending_aborts.get(kv_request_id) + expired = ( + started_at is not None + and time.monotonic() - started_at >= _CANCEL_DRAIN_TIMEOUT_S + ) + if expired: + for tid in ids: + self._inflight_pop(tid) + self._transport.cancel(ids, mode="immediate") + logger.warning( + "P2PSession %s: cancel drain timed out for kv_request_id=%s," + " force-canceled %d transfers", + self._peer_id, + kv_request_id, + len(ids), + ) + self._finalize_abort(kv_request_id) + return + + still = self._transport.cancel(ids, mode="wait") + # Tids the transport successfully released are gone from its + # _inflight; mirror that in session bookkeeping so they don't + # block the drain forever waiting for a poll() event that will + # never come. + still_set = set(still) + for tid in ids: + if tid not in still_set: + self._inflight_pop(tid) + if not still: + self._finalize_abort(kv_request_id) + + def _finalize_abort(self, kv_request_id: str) -> None: + self._pending_aborts.pop(kv_request_id, None) + self._send( + { + TYPE_KEY: AbortAckMsg.TYPE, + AbortAckMsg.KV_REQUEST_ID: kv_request_id, + } + ) + + # ------------------------------------------------------------------ + # Internal — transfers and store-job timeouts + # ------------------------------------------------------------------ + + def _submit_transfer(self, kv_request_id: str, result: _MatchResult) -> None: + logger.debug( + "P2PSession %s: NIXL write_blocks CALL kv_request_id=%s " + "local_idxs=%d remote_idxs=%d", + self._peer_id, + kv_request_id, + len(result.local_idxs), + len(result.remote_idxs), + ) + transfer_id = self._transport.write_blocks( + self._peer_id, result.local_idxs, result.remote_idxs + ) + if transfer_id is not None: + logger.debug( + "P2PSession %s: NIXL write_blocks SUBMITTED kv_request_id=%s " + "transfer_id=%d blocks=%d", + self._peer_id, + kv_request_id, + transfer_id, + len(result.local_idxs), + ) + self._inflight_add( + transfer_id, + _InflightXfer( + kv_request_id=kv_request_id, + block_count=len(result.local_idxs), + job_ids=result.job_ids, + ), + ) + else: + logger.warning( + "P2PSession %s: write_blocks failed for %s (%d blocks)", + self._peer_id, + kv_request_id, + len(result.local_idxs), + ) + # The matched blocks were popped from req.demanded / + # req.available, but no inflight will satisfy them, so + # remaining will never reach 0 on its own. Mark the + # request as finishing so the existing terminal paths + # clean up: if other inflight is in flight, the last one + # to drain will fire _finalize_outbound(success=False) + # via the elif branch in collect_results. If + # nothing else is in flight, finalize now so the peer + # and the local store jobs don't wait for finish_request + # or for _STORE_TIMEOUT_S / _LOAD_TIMEOUT_S. + req = self._outbound.get(kv_request_id) + if req is not None: + req.finishing = True + if not self._has_inflight_for(kv_request_id): + self._finalize_outbound(kv_request_id, success=False) + + def _timeout_pending_store_jobs(self) -> list[StoreResult]: + if not self._store_jobs: + return [] + deadline = time.monotonic() - _STORE_TIMEOUT_S + timed_out: list[int] | None = None + for jid, submitted_at in self._store_jobs.items(): + if submitted_at <= deadline: + if timed_out is None: + timed_out = [] + timed_out.append(jid) + if timed_out is None: + return [] + results: list[StoreResult] = [] + for jid in timed_out: + del self._store_jobs[jid] + results.append(StoreResult(job_id=jid, success=False)) + logger.warning("P2PSession %s: store job %d timed out", self._peer_id, jid) + return results diff --git a/vllm/v1/kv_offload/tiering/p2p/session/session.py b/vllm/v1/kv_offload/tiering/p2p/session/session.py new file mode 100644 index 0000000000..fc5f1feab5 --- /dev/null +++ b/vllm/v1/kv_offload/tiering/p2p/session/session.py @@ -0,0 +1,440 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +P2PSession — bidirectional session combining client + server roles. + +A single P2PSession per remote peer handles BOTH directions of the P2P +protocol on one ControlConnection: it can request blocks from the peer +("client" role, in :mod:`.client`) AND serve blocks to the peer +("server" role, in :mod:`.server`). This module is the thin coordinator +that owns the connection, the handshake, send-gating, and the message +dispatch — each parsed message is forwarded to the corresponding role. + +Wire protocol is unchanged. Both sides advertise their NIXL metadata +via ConnectMsg when their session is connected; the peer's ConnectMsg +triggers transport.add_remote_peer; ConnectAckMsg confirms the peer +received our ConnectMsg, after which queued outgoing messages are flushed. +""" + +from __future__ import annotations + +import contextlib +from collections.abc import Sequence +from typing import TYPE_CHECKING, NamedTuple + +from vllm.logger import init_logger +from vllm.v1.kv_offload.base import OffloadKey +from vllm.v1.kv_offload.tiering.p2p.control.base import ControlConnection +from vllm.v1.kv_offload.tiering.p2p.session.client import ClientRole, LoadResult +from vllm.v1.kv_offload.tiering.p2p.session.protocol import ( + TYPE_KEY, + AbortAckMsg, + AbortFetchMsg, + ConnectAckMsg, + ConnectMsg, + DisconnectMsg, + FetchMsg, + TransferDoneMsg, +) +from vllm.v1.kv_offload.tiering.p2p.session.server import ( + ServerRole, + StoreResult, +) + +if TYPE_CHECKING: + from vllm.v1.kv_offload.tiering.base import JobId + from vllm.v1.kv_offload.tiering.p2p.data import DataTransport + +logger = init_logger(__name__) + +# Cap on consecutive non-protocol dispatch exceptions before we tear +# down the session. Protocol violations (ValueError) disconnect on the +# first occurrence; this threshold protects against repeated internal +# bugs that may indicate a peer-induced bad state. Reset on any +# successful dispatch. +_MAX_CONSECUTIVE_DISPATCH_ERRORS = 5 + + +class SessionPollResult(NamedTuple): + """Result of one P2PSession.poll() tick. + + `loads`/`stores` are the same per-role results the manager has always + consumed. `new_fetch_ids` reports kv_request_ids whose FetchMsg + arrived this tick — the manager uses them to bind kv_request_id → + session and replay any submit_store batches parked while no peer had + asked yet. Reporting (rather than calling back into the manager + mid-dispatch) keeps the dependency strictly top-down. + """ + + loads: list[LoadResult] + stores: list[StoreResult] + new_fetch_ids: list[str] + + +class P2PSession: + """Bidirectional session — coordinator over ClientRole + ServerRole. + + Lifecycle: + - Constructor with conn=None ⇒ pending. Accepts add_stored_blocks + but cannot send (used by the prefiller to buffer blocks before + the decoder connects). + - Constructor with conn != None ⇒ connected. Sends our own ConnectMsg + immediately; the peer's ConnectMsg arrives in poll() and is + dispatched to _on_connect (which calls transport.add_remote_peer + and replies with ConnectAckMsg). Outgoing sends are queued until + ConnectAckMsg confirms our metadata reached the peer. + - attach_connection(conn) on a pending session ⇒ same as above, + starting from pending. + """ + + def __init__( + self, + peer_id: str, + local_id: str, + transport: DataTransport, + local_block_len: int, + conn: ControlConnection | None = None, + ) -> None: + self.peer_id = peer_id + self._local_id = local_id + self._transport = transport + self._local_block_len = local_block_len + self._conn: ControlConnection | None = None + + self._send_ready = False # True after the peer acked our ConnectMsg + # Msgs waiting to be sent on connection establishment + self._queued: list[dict] = [] + + # Consecutive non-protocol dispatch errors. Reset on success. + self._dispatch_error_count: int = 0 + + # kv_request_ids whose FetchMsg arrived during the current poll + # tick. Drained and returned in the next poll() result so the + # manager can bind kv_request_id → session and replay any + # submit_store batches parked before the binding existed. + self._new_fetch_ids: list[str] = [] + + self._client = ClientRole(peer_id=peer_id, send=self._send) + self._server = ServerRole(peer_id=peer_id, transport=transport, send=self._send) + + if conn is not None: + self.attach_connection(conn) + + # ------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------ + + @property + def alive(self) -> bool: + # Pending sessions (awaiting connection) are alive — only a + # closed real connection counts as dead. + return self._conn is None or self._conn.alive + + @property + def connected(self) -> bool: + return self._conn is not None + + @property + def ready(self) -> bool: + """True after the peer acked our ConnectMsg (we may send freely).""" + return self._send_ready + + # ------------------------------------------------------------------ + # Connection lifecycle + # ------------------------------------------------------------------ + + def attach_connection(self, conn: ControlConnection) -> None: + """Attach a connection to a pending session and announce ourselves. + + Symmetric: every side advertises its NIXL metadata on connect, so + whichever peer receives a session first can register the other. + """ + if self._conn is not None: + raise ValueError(f"P2PSession {self.peer_id}: already connected") + self._conn = conn + self._send_connect() + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def request_blocks( + self, + job_id: JobId, + kv_request_id: str, + keys: Sequence[bytes], + block_ids: Sequence[int], + ) -> None: + """Send fetch to the peer.""" + self._client.request_blocks( + job_id, kv_request_id, keys, block_ids, send_ready=self._send_ready + ) + + def add_stored_blocks( + self, + kv_request_id: str, + keys: Sequence[OffloadKey], + block_ids: Sequence[int], + job_id: JobId, + ) -> None: + """New blocks stored locally — match against pending fetch demand.""" + self._server.add_stored_blocks(kv_request_id, keys, block_ids, job_id) + + def finish_request(self, kv_request_id: str) -> None: + """Called when the request is finishing locally. + + Cancels any inbound load (client role) and finalizes any + outbound serving (server role) for this id. Roles that aren't + active for this id are silent no-ops. + """ + self._client.cancel(kv_request_id) + self._server.finish(kv_request_id) + + def poll(self) -> SessionPollResult: + """Process incoming messages, drive transfers, apply timeouts.""" + if self._conn is None: + # Pending session — store-job timeouts still apply so buffered + # jobs that never get picked up are surfaced as failures. + return SessionPollResult( + loads=[], + stores=self._server.collect_idle_timeouts(), + new_fetch_ids=[], + ) + + for msg in self._conn.recv(): + self._on_message(msg) + + loads = self._client.collect_results() + stores = self._server.collect_results() + self._server.drain_pending_aborts() + + new_fetch_ids = self._new_fetch_ids + self._new_fetch_ids = [] + return SessionPollResult( + loads=loads, stores=stores, new_fetch_ids=new_fetch_ids + ) + + def close(self) -> tuple[list[tuple[int, str]], list[int]]: + """Shut down. Returns (failed_loads, failed_stores). + + failed_loads: list of (job_id, kv_request_id) pairs. + failed_stores: list of job_ids. + """ + failed_loads = self._client.close() + failed_stores = self._server.close() + + if self._conn is not None: + with contextlib.suppress(Exception): + self._conn.send({TYPE_KEY: DisconnectMsg.TYPE}) + self._conn.close() + self._conn = None + + return failed_loads, failed_stores + + # ------------------------------------------------------------------ + # Message dispatch + # ------------------------------------------------------------------ + + def _on_message(self, msg: dict) -> None: + msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else msg + try: + self._dispatch_message(msg) + except ValueError as exc: + # Protocol contract violation from the peer — *Msg.validate() + # and handler-level checks raise ValueError. Retrying won't + # help and may corrupt session state, so disconnect now. + self._protocol_error(f"malformed {msg_type!r}: {exc}") + return + except Exception as exc: + # Anything else is most likely an internal bug rather than a + # peer fault. Log loudly with a traceback so it doesn't + # disappear, but don't kill the session on a single hiccup. + # Disconnect only if errors keep arriving — that pattern is + # consistent with a peer wedging us into a broken state. + self._dispatch_error_count += 1 + logger.exception( + "P2PSession %s: error handling message %r (count=%d): %s", + self.peer_id, + msg_type, + self._dispatch_error_count, + exc, + ) + if self._dispatch_error_count >= _MAX_CONSECUTIVE_DISPATCH_ERRORS: + self._protocol_error( + f"too many consecutive dispatch errors " + f"({self._dispatch_error_count})" + ) + return + self._dispatch_error_count = 0 + + def _protocol_error(self, reason: str) -> None: + """Log a protocol violation and disconnect. + + Best-effort sends ``DisconnectMsg`` so the peer learns why we're + going away, then marks the connection dead. The manager reaps + the session on the next poll via ``alive``. + """ + logger.error( + "P2PSession %s: protocol error: %s — disconnecting", + self.peer_id, + reason, + ) + if self._conn is not None: + with contextlib.suppress(Exception): + self._conn.send({TYPE_KEY: DisconnectMsg.TYPE}) + self._conn.mark_dead() + + def _dispatch_message(self, msg: dict) -> None: + # Drop messages buffered before disconnect: a poll batch can + # contain msg-after-DisconnectMsg, and dispatching them would + # mutate state on a dead session. + if self._conn is not None and not self._conn.alive: + return + msg_type = msg.get(TYPE_KEY) if isinstance(msg, dict) else None + if msg_type == ConnectMsg.TYPE: + self._on_connect(msg) + elif msg_type == ConnectAckMsg.TYPE: + ConnectAckMsg.validate(msg) + self._on_connect_ack() + elif msg_type == FetchMsg.TYPE: + FetchMsg.validate(msg) + kv_request_id = msg[FetchMsg.KV_REQUEST_ID] + block_hashes = [ + OffloadKey(bh if isinstance(bh, bytes) else bytes(bh)) + for bh in msg[FetchMsg.BLOCK_HASHES] + ] + block_indexes = msg[FetchMsg.BLOCK_INDEXES] + # Run the server-role state machine inline as today — + # add_fetch_demand records demand against any blocks we've + # already seen in `available`. Report the kv_request_id so + # the manager (after poll() returns) can replay any parked + # submit_store batches; their add_stored_blocks calls hit + # the demand recorded here and submit transfers immediately. + self._server.on_fetch(kv_request_id, block_hashes, block_indexes) + self._new_fetch_ids.append(kv_request_id) + elif msg_type == AbortFetchMsg.TYPE: + AbortFetchMsg.validate(msg) + self._server.on_abort_fetch(msg[AbortFetchMsg.KV_REQUEST_ID]) + elif msg_type == TransferDoneMsg.TYPE: + TransferDoneMsg.validate(msg) + self._client.on_transfer_done( + msg[TransferDoneMsg.KV_REQUEST_ID], + msg[TransferDoneMsg.SUCCESS], + ) + elif msg_type == AbortAckMsg.TYPE: + AbortAckMsg.validate(msg) + self._client.on_abort_ack(msg[AbortAckMsg.KV_REQUEST_ID]) + elif msg_type == DisconnectMsg.TYPE: + if self._conn is not None: + self._conn.mark_dead() + else: + logger.warning( + "P2PSession %s: unknown message type %r", self.peer_id, msg_type + ) + + # ------------------------------------------------------------------ + # Handshake + # ------------------------------------------------------------------ + + def _on_connect(self, msg: dict) -> None: + # Validation failures here mean an incompatible or malicious peer. + # Mark the connection dead so the manager reaps the session; + # don't call add_remote_peer or send connect_ack. + if self._send_ready: + # We've already received connect_ack, so the handshake is + # complete. A second connect from the peer is a protocol + # violation — re-registering would corrupt transport state. + self._protocol_error("duplicate connect after handshake") + return + try: + ConnectMsg.validate(msg) + if msg[ConnectMsg.BLOCK_LEN] != self._local_block_len: + raise ValueError( + f"block_len mismatch from {self.peer_id}: " + f"remote={msg[ConnectMsg.BLOCK_LEN]}, " + f"local={self._local_block_len}" + ) + remote_fp = msg.get(ConnectMsg.CONFIG_FINGERPRINT, "") + local_fp = self._transport.config_fingerprint + if local_fp and remote_fp and remote_fp != local_fp: + raise ValueError( + f"config fingerprint mismatch from {self.peer_id}: " + f"remote={remote_fp!r}, local={local_fp!r}" + ) + self._transport.add_remote_peer( + self.peer_id, + agent_metadata=msg[ConnectMsg.AGENT_METADATA], + base_addr=msg[ConnectMsg.BASE_ADDR], + num_blocks=msg[ConnectMsg.NUM_BLOCKS], + block_len=msg[ConnectMsg.BLOCK_LEN], + ) + except ValueError as exc: + logger.error("P2PSession %s: rejecting peer connect: %s", self.peer_id, exc) + if self._conn is not None: + self._conn.mark_dead() + return + + if self._conn is not None: + self._conn.send( + { + TYPE_KEY: ConnectAckMsg.TYPE, + ConnectAckMsg.PEER_ID: self._local_id, + } + ) + + def _on_connect_ack(self) -> None: + if self._queued: + logger.debug( + "P2PSession %s: connect_ack received, flushing %d queued msg(s)", + self.peer_id, + len(self._queued), + ) + self._send_ready = True + for queued in self._queued: + self._do_send(queued) + self._queued.clear() + + # ------------------------------------------------------------------ + # Send helpers + # ------------------------------------------------------------------ + + def _send_connect(self) -> None: + """Send our ConnectMsg announcing local NIXL metadata.""" + assert self._conn is not None + self._conn.send( + { + TYPE_KEY: ConnectMsg.TYPE, + ConnectMsg.PEER_ID: self._local_id, + ConnectMsg.AGENT_METADATA: self._transport.get_agent_metadata(), + ConnectMsg.BASE_ADDR: self._transport.base_addr, + ConnectMsg.NUM_BLOCKS: self._transport.num_blocks, + ConnectMsg.BLOCK_LEN: self._transport.block_len, + ConnectMsg.CONFIG_FINGERPRINT: self._transport.config_fingerprint, + } + ) + + def _send(self, msg: dict) -> None: + if self._conn is None or not self._send_ready: + logger.debug( + "P2PSession %s: queueing %s (ready=%s queue_depth=%d)", + self.peer_id, + msg.get(TYPE_KEY), + self._send_ready, + len(self._queued) + 1, + ) + self._queued.append(msg) + return + self._do_send(msg) + + def _do_send(self, msg: dict) -> None: + if self._conn is None: + return + try: + self._conn.send(msg) + logger.debug("P2PSession %s: sent %s", self.peer_id, msg.get(TYPE_KEY)) + except Exception: + logger.warning( + "P2PSession %s: failed to send %s", + self.peer_id, + msg.get(TYPE_KEY), + )