[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders

- Delete the snapshot package, CLI command, and its launch tests
- Delete CPU workers, cpu_communicator, and the hw_agnostic layer package
- Delete the EC CPU connector, keeping the EC base glue intact
- Delete dead composite/triton_flashinfer backends and third_party/flashmla
- Delete dead video decoders, video-bound tests, and orphan utils
- Remove the deep_gemm_mega_moe option and dead allreduce fusion pair

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 09:27:06 +07:00
co-authored by Hinata
parent 2b544b7ba0
commit bd70a61c20
88 changed files with 4 additions and 20247 deletions
File diff suppressed because it is too large Load Diff
@@ -1,111 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Any
from vllm.model_executor.warmup import jit_warmup_cutedsl_helper
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
CuTeDSLLaunchSpec,
VllmCuTeDSLJitKernel,
)
class _TestCuTeDSLKernel(VllmCuTeDSLJitKernel["_TestCuTeDSLKernel.CompileKey"]):
@dataclass(frozen=True)
class CompileKey:
variant: int
@staticmethod
def kernel(compile_key: CompileKey) -> tuple[str, int]:
return "entry", compile_key.variant
def dispatch(self, *, variant: int) -> CompileKey:
return self.CompileKey(variant=variant)
def get_warmup_keys(self) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(variant=(1, 2))
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
return (f"fake-{compile_key.variant}",)
@kernel_launcher
def __call__(
self,
payload: str,
*,
variant: int,
) -> CuTeDSLLaunchSpec[CompileKey]:
compile_key = self.dispatch(variant=variant)
return compile_key, (payload,)
def test_cutedsl_launcher_reuses_compiled_executor(monkeypatch) -> None:
compile_calls: list[tuple[Any, tuple[Any, ...]]] = []
launch_calls: list[tuple[int, tuple[Any, ...]]] = []
def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any:
compile_calls.append((entry, args))
variant = entry[1]
def executor(*runtime_args: Any) -> tuple[int, tuple[Any, ...]]:
launch_calls.append((variant, runtime_args))
return variant, runtime_args
return executor
monkeypatch.setattr(
jit_warmup_cutedsl_helper,
"compile_cutedsl",
compile_cutedsl,
)
kernel = _TestCuTeDSLKernel()
compile_key = kernel.CompileKey(variant=1)
kernel.compile(compile_key)
kernel.compile(compile_key)
assert compile_calls == [(("entry", 1), ("fake-1",))]
assert kernel("runtime", variant=1) == (1, ("runtime",))
assert kernel("other", variant=2) == (2, ("other",))
assert compile_calls == [
(("entry", 1), ("fake-1",)),
(("entry", 2), ("fake-2",)),
]
assert launch_calls == [(1, ("runtime",)), (2, ("other",))]
def test_cutedsl_launcher_runs_post_launch_finalizer(monkeypatch) -> None:
events: list[str] = []
def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any:
def executor(*runtime_args: Any) -> None:
events.append("launch")
return executor
class FinalizedKernel(_TestCuTeDSLKernel):
@kernel_launcher
def __call__(
self,
payload: str,
*,
variant: int,
) -> CuTeDSLLaunchSpec[_TestCuTeDSLKernel.CompileKey]:
compile_key = self.dispatch(variant=variant)
def finalize() -> str:
events.append("finalize")
return "output"
return compile_key, (payload,), "unused", finalize
monkeypatch.setattr(
jit_warmup_cutedsl_helper,
"compile_cutedsl",
compile_cutedsl,
)
assert FinalizedKernel()("runtime", variant=1) == "output"
assert events == ["launch", "finalize"]
@@ -1,113 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end check of the Transformers backend's hw-agnostic layers.
Serves a tiny Llama through `vllm_runner` with and without
`VLLM_USE_HW_AGNOSTIC` and checks the layer providers and logprobs. Needs a
real engine, so it runs in the GPU job. The CPU-only unit tests for the
resolution logic (`layers._resolve`) are in `test_layer_registry.py`.
"""
import pytest
import torch
@pytest.fixture(scope="module")
def tiny_llama_path(tmp_path_factory):
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
tokenizer) so vLLM can load it like any local checkpoint."""
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
config = LlamaConfig(
vocab_size=tokenizer.vocab_size,
hidden_size=128,
intermediate_size=256,
num_hidden_layers=2,
num_attention_heads=4,
rms_norm_eps=1e-6,
hidden_act="silu",
)
torch.manual_seed(0)
model = LlamaForCausalLM(config)
path = tmp_path_factory.mktemp("tiny_llama")
model.save_pretrained(path)
tokenizer.save_pretrained(path)
return str(path)
# Registered names of the layers the backend can
# currently route to hw-agnostic implementations.
_COVERED_LAYERS = ("rms_norm", "silu_and_mul")
def _layer_providers(model) -> dict[str, str]:
"""Map each covered layer type present in the model to the provider its
implementation came from (``hw_agnostic`` or ``vllm``).
"""
def provider_of(module) -> str | None:
for cls in type(module).__mro__:
if "hw_agnostic.layers" in cls.__module__:
return "hw_agnostic"
if ".model_executor.layers." in cls.__module__:
return "vllm"
return None
providers: dict[str, str] = {}
for module in model.modules():
name = getattr(module, "name", None)
if (
name in _COVERED_LAYERS
and name not in providers
and (prov := provider_of(module)) is not None
):
providers[name] = prov
return providers
def _serve(vllm_runner, model_path, prompts):
"""Serve the model through the backend; return (layer_providers, logprobs)."""
with vllm_runner(
model_path,
model_impl="transformers",
max_model_len=64,
enforce_eager=True,
gpu_memory_utilization=0.3,
) as runner:
assert runner.llm.llm_engine.model_config.using_transformers_backend()
providers = runner.apply_model(_layer_providers)[0]
outputs = runner.generate_greedy_logprobs(
prompts, max_tokens=32, num_logprobs=5
)
return providers, outputs
def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path):
"""Serving the tiny model with hw-agnostic layers matches the vLLM baseline."""
# spawn: `layers` resolves its symbols at import time, so the worker must
# re-import it with VLLM_USE_HW_AGNOSTIC already set (fork would inherit
# the parent's already-resolved module).
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
# apply_model pickles the introspection function.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
from ..utils import check_logprobs_close
prompts = ["The capital of France is", "vLLM is"]
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
# Both replaceable layers present in a Llama block must be vLLM's here.
assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"}
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"}
check_logprobs_close(
outputs_0_lst=vllm_outputs,
outputs_1_lst=hw_outputs,
name_0="vllm",
name_1="hw_agnostic",
)
View File
-233
View File
@@ -1,233 +0,0 @@
# EPD Correctness Test
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
## What It Tests
- **Baseline**: Single vLLM instance serving a multimodal model
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features.
## Files
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
- `test_epd_correctness.py` - Python test script (compares outputs)
## Usage
### Multimodal Prompts (Default)
```bash
cd vllm
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs the test with actual multimodal (image) prompts.
### Text-Only Prompts
```bash
cd vllm
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs a quick test with text-only prompts to verify the setup works.
### Custom Configuration
```bash
# Use specific GPUs
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific ports
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific model
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific storage path
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
## How It Works
### NIXL EC failure isolation (1E + 1PD)
```bash
PATH="$PWD/.venv/bin:$PATH" CUDA_VISIBLE_DEVICES=0,1 \
.venv/bin/python -m pytest \
tests/v1/ec_connector/integration/test_nixl_failure.py -v -s
```
Requires two CUDA GPUs and NIXL. `MODEL` overrides the default
`Qwen/Qwen2.5-VL-3B-Instruct`. The test starts real E/PD servers with CUDA
graphs enabled and uses the proxy's rewrite helper. Only the failing request's
control endpoint is replaced: a ZMQ peer returns `NACK_MISSING` and verifies
that the consumer actually requested the encoding.
Metadata-only requests must return a request-level error; requests retaining
the image must succeed through local fallback. After each case, a fresh
metadata-only image request must succeed through the real NIXL transfer path.
No server restart, proxy retry, cache hit, or timeout race can mask the result.
This is a correctness test, not a throughput benchmark.
### Mooncake EC (1E + 1PD)
```bash
PYTHON_BIN="$PWD/.venv/bin/python" \
bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
```
Requires two GPUs and Mooncake TransferEngine. TCP is the default transport;
no RDMA-capable network hardware is required.
The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA.
The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1.
`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport
environment variables should be set by the caller.
Three black-box cases check fixed short answers and compare with the baseline:
- One image: read the STOP sign.
- Two different images (including a local file): identify flowers and birds.
- The same image twice: read both STOP signs.
By default, all three requests run concurrently for two rounds, exercising
shared hashes across requests and reuse after completion. Every response is
compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override.
Prefix caching is disabled and CUDA graphs remain enabled. This is a small
correctness suite, not a performance benchmark or failure-injection suite.
`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output.
`SKIP_BASELINE=1` reuses a reference generated with the same model and test
configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer.
Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs.
The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and
selected for changes to EC, multimodal processing, scheduler/model-runner
integration, the proxy, or these tests (subject to normal PR CI approval).
It reuses the CI image's Python packages through a system-site-packages venv
and installs the CUDA-compatible Mooncake wheel. It uses loopback networking
and TCP only, without RDMA devices or peer-memory setup.
The job fails on startup errors, request errors, or answer mismatches. On
failure, the script prints the last 100 lines of each server/proxy log to
the CI job log; full files remain under `LOG_PATH` while the container exists.
### Step 1: Baseline
1. Start single vLLM instance on GPU
2. Run test prompts (multimodal or text-only)
3. Save outputs to `.vllm_epd_baseline.txt`
4. Shutdown instance
### Step 2: EPD (1E + 1PD)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
### Step 3: EPD (1E + 1P + 1D)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
## Test Scenarios
### Multimodal Prompts (--use_mm_prompts)
Tests encoder cache transfer:
- Single image query
- Multiple images in one request
- Mixed image and text
- Image with detailed questions
### Text-Only Prompts (default)
Quick sanity check:
- Simple text queries
- Text-only explanations
- Verifies proxy routing works
## Expected Behavior
### ✅ Test Passes When
- All disagg outputs match baseline outputs exactly
- No errors during instance startup
- Encoder cache is properly saved and loaded
- Proxy correctly routes requests
### ❌ Test Fails When
- Outputs differ between baseline and disagg
- Server startup fails
- Encoder cache not found (should fall back to local execution)
- Proxy routing errors
## Notes
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
- Encoder cache should enable exact output reproduction
- Test cleans up all instances and cache files after completion
- Safe to run multiple times (idempotent)
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
## Requirements
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
- Internet access (for accessing vllm test images)
## Debugging
### Check Logs
Logs and baseline output are saved in `/tmp/` by default.
Can be customized by changing the environment variables.
### Check Encoder Cache
```bash
# Verify cache files are created
ls -la $EC_SHARED_STORAGE_PATH/
# Should see directories with mm_hash names
# Each containing encoder_cache.safetensors
```
### Manual Testing
Run individual components:
```bash
# Baseline only
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode baseline \
--baseline_file test_output.txt \
--use_mm_prompts
# Disagg only (requires baseline output file!)
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode disagg \
--baseline_file test_output.txt \
--use_mm_prompts
```
Binary file not shown.

Before

Width:  |  Height:  |  Size: 821 KiB

@@ -1,505 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# EPD (Encoder-Prefill-Decode) Correctness Test
#
# This script tests that EPD disaggregation produces the same outputs as baseline.
# It runs:
# 1. Baseline: Single vLLM instance
# 2. EPD: 1E + 1PD setup
# 3. Baseline for (E + P + D): 1P + 1D vLLM instances disagg
# 4. EPD: 1E + 1P + 1D setup
# For GPU usage
# set -xe
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
# Model to test
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-10240}"
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
# Set 1 to use multimodal prompts; else to use text-only
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
USE_TWO_IMAGE_PROMPT="${USE_TWO_IMAGE_PROMPT:-1}"
TEST_FLAGS=()
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
TEST_FLAGS+=(--use_mm_prompts)
fi
if [[ "$USE_TWO_IMAGE_PROMPT" != "1" ]]; then
TEST_FLAGS+=(--skip_two_image_prompt)
fi
# GPU configuration
GPU_E="${GPU_E:-0}"
GPU_P="${GPU_P:-1}"
GPU_D="${GPU_D:-2}"
GPU_SINGLE="${GPU_SINGLE:-$GPU_P}"
GPU_PD="${GPU_PD:-$GPU_P}"
# Device platform and affinity environment variable
DEVICE_PLATFORM="${DEVICE_PLATFORM:-cuda}"
if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then
if [[ "${DEVICE_PLATFORM,,}" == "xpu" ]]; then
DEVICE_AFFINITY_ENV="ZE_AFFINITY_MASK"
else
DEVICE_AFFINITY_ENV="CUDA_VISIBLE_DEVICES"
fi
fi
# Port
ENCODE_PORT="${ENCODE_PORT:-19534}"
PREFILL_PORT="${PREFILL_PORT:-19535}"
DECODE_PORT="${DECODE_PORT:-19536}"
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
ENDPOINT_PORT="${ENDPOINT_PORT:-10001}"
# Storage path for encoder cache
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache_test}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}"
# Output file for baseline comparison and logs
LOG_PATH="${LOG_PATH:-/tmp}"
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_baseline.txt}"
BASELINE_PD_FILE="${BASELINE_PD_FILE:-/tmp/vllm_epd_baseline.txt}"
mkdir -p "$LOG_PATH"
# Trap the SIGINT signal (triggered by Ctrl+C)
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
# Wait for server to be ready
wait_for_server() {
local port=$1
timeout "$TIMEOUT_SECONDS" bash -c "
until curl -s localhost:${port}/v1/chat/completions > /dev/null; do
sleep 1
done" && return 0 || return 1
}
# Cleanup function
cleanup_instances() {
echo "Cleaning up any running vLLM instances..."
pkill -f "vllm serve" || true
pkill -f "disagg_epd_proxy.py" || true
sleep 2
}
# Function to run baseline (single instance)
run_baseline() {
echo "================================"
echo "Running BASELINE (single instance)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
local PORT=$ENDPOINT_PORT
# Start baseline instance
echo "Starting baseline instance on GPU $GPU_SINGLE, port $PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_SINGLE" vllm serve "$MODEL" \
--port "$PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.9 \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
> "$LOG_PATH"/baseline.log 2>&1 &
local BASELINE_PID=$!
# Wait for baseline to start
echo "Waiting for baseline instance to start..."
wait_for_server "$PORT"
curl http://127.0.0.1:"$PORT"/v1/models
echo ""
# Run test in baseline mode
echo "Running baseline..."
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PORT" \
--model_name "$MODEL" \
--mode baseline \
--baseline_file "$BASELINE_FILE" \
"${TEST_FLAGS[@]}"
# Cleanup baseline
echo "Stopping baseline instance..."
kill $BASELINE_PID 2>/dev/null || true
sleep 2
cleanup_instances
}
# Function to run EPD with 1E + 1PD
run_epd_1e_1pd() {
echo "================================"
echo "Running EPD (1E + 1PD)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
mkdir -p "$EC_SHARED_STORAGE_PATH"
local ENCODE_PORT=$ENCODE_PORT
local PREFILL_DECODE_PORT=$PREFILL_DECODE_PORT
local PROXY_PORT=$ENDPOINT_PORT
declare -a PIDS=()
# Start encoder instance
echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
--port "$ENCODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.01 \
--enable-request-id-headers \
--no-enable-prefix-caching \
--max-num-batched-tokens 114688 \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$LOG_PATH"/1e1pd_encoder.log 2>&1 &
PIDS+=($!)
# Start prefill+decode instance
echo "Starting PD instance on GPU $GPU_PD, port $PREFILL_DECODE_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_PD" vllm serve "$MODEL" \
--port "$PREFILL_DECODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_consumer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$LOG_PATH"/1e1pd_pd.log 2>&1 &
PIDS+=($!)
# Wait for instances to start
echo "Waiting for encoder instance..."
wait_for_server "$ENCODE_PORT"
echo "Waiting for PD instance..."
wait_for_server "$PREFILL_DECODE_PORT"
# Start proxy
echo "Starting EPD proxy on port $PROXY_PORT"
python "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
--host "0.0.0.0" \
--port "$PROXY_PORT" \
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
--prefill-servers-urls "disable" \
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
> "$LOG_PATH"/1e1pd_proxy.log 2>&1 &
PIDS+=($!)
# Wait for proxy
echo "Waiting for proxy..."
wait_for_server "$PROXY_PORT"
curl http://127.0.0.1:"$PROXY_PORT"/v1/models
curl http://127.0.0.1:"$PROXY_PORT"/health
echo ""
echo "All EPD (1E+1PD) services are up!"
# Run test in disagg mode
echo "Running EPD (1E+1PD) correctness test..."
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PROXY_PORT" \
--model_name "$MODEL" \
--mode disagg \
--baseline_file "$BASELINE_FILE" \
"${TEST_FLAGS[@]}"
# Cleanup
echo "✓✓ 1E+1PD Correctness Test finished"
echo "Stopping EPD (1E+1PD) instances..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
sleep 2
cleanup_instances
}
# Function to run baseline for 1E + 1P + 1D (PD disagg)
run_baseline_1p_1d() {
echo "================================"
echo "Running PD BASELINE (1P + 1D)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
mkdir -p "$EC_SHARED_STORAGE_PATH"
local PREFILL_PORT=$PREFILL_PORT
local DECODE_PORT=$DECODE_PORT
local PROXY_PORT=$ENDPOINT_PORT
declare -a PIDS=()
# Start prefill instance
echo "Starting prefill instance on GPU $GPU_P, port $PREFILL_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_P" \
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
vllm serve "$MODEL" \
--port "$PREFILL_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_producer"
}' \
> "$LOG_PATH"/1p1d_prefill.log 2>&1 &
PIDS+=($!)
# Start decode instance
echo "Starting decode instance on GPU $GPU_D, port $DECODE_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_D" \
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
vllm serve "$MODEL" \
--port "$DECODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_consumer"
}' \
> "$LOG_PATH"/1p1d_decode.log 2>&1 &
PIDS+=($!)
# Wait for instances to start
echo "Waiting for prefill instance..."
wait_for_server "$PREFILL_PORT"
echo "Waiting for decode instance..."
wait_for_server "$DECODE_PORT"
# Start proxy
echo "Starting EPD proxy on port $PROXY_PORT"
python "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
--host "0.0.0.0" \
--port "$PROXY_PORT" \
--prefiller-ports "$PREFILL_PORT" \
--decoder-ports "$DECODE_PORT" \
> "$LOG_PATH"/1p1d_proxy.log 2>&1 &
PIDS+=($!)
# Wait for proxy
echo "Waiting for proxy..."
wait_for_server "$PROXY_PORT"
curl http://127.0.0.1:"$PROXY_PORT"/healthcheck
echo ""
echo "All PD (1P+1D) services are up!"
# Run test in baseline mode
echo "Running PD disagg baseline..."
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PROXY_PORT" \
--model_name "$MODEL" \
--mode baseline_pd \
--baseline_file "$BASELINE_PD_FILE" \
"${TEST_FLAGS[@]}"
# Cleanup
echo "Stopping PD (1P+1D) instances..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
sleep 2
cleanup_instances
}
# Function to run EPD with 1E + 1P + 1D
run_epd_1e_1p_1d() {
echo "================================"
echo "Running EPD (1E + 1P + 1D)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
mkdir -p "$EC_SHARED_STORAGE_PATH"
local ENCODE_PORT=$ENCODE_PORT
local PREFILL_PORT=$PREFILL_PORT
local DECODE_PORT=$DECODE_PORT
local PROXY_PORT=$ENDPOINT_PORT
declare -a PIDS=()
# Start encoder instance
echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
--port "$ENCODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.01 \
--enable-request-id-headers \
--no-enable-prefix-caching \
--max-num-batched-tokens 114688 \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$LOG_PATH"/1e1p1d_encoder.log 2>&1 &
PIDS+=($!)
# Start prefill instance
echo "Starting prefill instance on GPU $GPU_P, port $PREFILL_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_P" \
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
vllm serve "$MODEL" \
--port "$PREFILL_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_consumer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_producer"
}' \
> "$LOG_PATH"/1e1p1d_prefill.log 2>&1 &
PIDS+=($!)
# Start decode instance
echo "Starting decode instance on GPU $GPU_D, port $DECODE_PORT"
env "$DEVICE_AFFINITY_ENV=$GPU_D" \
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
vllm serve "$MODEL" \
--port "$DECODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--enable-request-id-headers \
--max-num-seqs "$MAX_NUM_SEQS" \
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
--kv-transfer-config '{
"kv_connector": "NixlConnector",
"kv_role": "kv_consumer"
}' \
> "$LOG_PATH"/1e1p1d_decode.log 2>&1 &
PIDS+=($!)
# Wait for instances to start
echo "Waiting for encoder instance..."
wait_for_server "$ENCODE_PORT"
echo "Waiting for prefill instance..."
wait_for_server "$PREFILL_PORT"
echo "Waiting for decode instance..."
wait_for_server "$DECODE_PORT"
# Start proxy
echo "Starting EPD proxy on port $PROXY_PORT"
python "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
--host "0.0.0.0" \
--port "$PROXY_PORT" \
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
--prefill-servers-urls "http://localhost:$PREFILL_PORT" \
--decode-servers-urls "http://localhost:$DECODE_PORT" \
> "$LOG_PATH"/1e1p1d_proxy.log 2>&1 &
PIDS+=($!)
# Wait for proxy
echo "Waiting for proxy..."
wait_for_server "$PROXY_PORT"
curl http://127.0.0.1:"$PROXY_PORT"/v1/models
curl http://127.0.0.1:"$PROXY_PORT"/health
echo ""
echo "All EPD (1E+1P+1D) services are up!"
# Run test in disagg mode
echo "Running EPD (1E+1P+1D) correctness test..."
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PROXY_PORT" \
--model_name "$MODEL" \
--mode disagg \
--baseline_file "$BASELINE_PD_FILE" \
"${TEST_FLAGS[@]}"
# Cleanup
echo "✓✓ 1E+1P+1D Correctness Test finished"
echo "Stopping EPD (1E+1P+1D) instances..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
sleep 2
cleanup_instances
}
# Main execution
echo "================================"
echo "EPD Correctness Test Suite"
echo "Model: $MODEL"
echo "================================"
# Step 1: Run baseline
run_baseline
# Step 2: Test 1E + 1PD
run_epd_1e_1pd
# Step 3: Test baseline 1P + 1D
run_baseline_1p_1d
# # Step 4: Test 1E + 1P + 1D
run_epd_1e_1p_1d
# Cleanup output file
rm -f "$BASELINE_FILE"
rm -f "$BASELINE_PD_FILE"
echo "================================"
echo "✓✓ All EPD correctness tests finished!"
echo "================================"
@@ -1,323 +0,0 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine):
# 1) Single-GPU baseline (multimodal) -> saves baseline JSON
# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline
#
# Usage (from repo root):
# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
#
# Env:
# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct)
# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1)
# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT
# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1)
# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp)
# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity
# TIMEOUT_SECONDS wait_for_server timeout (default 1200)
# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE
# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)
# MAX_MODEL_LEN context length (default 16384)
# E_CUDAGRAPH_MM_ENCODER enable encoder CUDA graphs (default 0)
# E_MM_ENCODER_ATTN_BACKEND optional encoder attention backend override
set -euo pipefail
GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
cd "$GIT_ROOT" || exit 1
export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}"
PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}"
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}")
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)
fi
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
E_ATTN_ARGS=()
if [[ -n "${E_MM_ENCODER_ATTN_BACKEND:-}" ]]; then
E_ATTN_ARGS+=(--mm-encoder-attn-backend "$E_MM_ENCODER_ATTN_BACKEND")
fi
case "${E_CUDAGRAPH_MM_ENCODER:-0}" in
0) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":false}' ;;
1) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":true}' ;;
*) echo "E_CUDAGRAPH_MM_ENCODER must be 0 or 1"; exit 1 ;;
esac
GPU_SINGLE="${GPU_SINGLE:-0}"
GPU_E="${GPU_E:-0}"
GPU_PD="${GPU_PD:-1}"
ENCODE_PORT="${ENCODE_PORT:-19534}"
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
ENDPOINT_PORT="${ENDPOINT_PORT:-10002}"
BASELINE_PORT="${BASELINE_PORT:-10003}"
EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}"
EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}"
MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}"
export EC_MOONCAKE_RESERVATION_HOST
export EC_MOONCAKE_RESERVATION_PORT
export MOONCAKE_EC_PROTOCOL
if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then
# TransferEngine may otherwise auto-select RDMA on hosts with an HCA.
export MC_FORCE_TCP=1
else
unset MC_FORCE_TCP
fi
LOG_PATH="${LOG_PATH:-/tmp}"
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}"
PIDS=()
mkdir -p "$LOG_PATH"
VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve)
ENC_EC_JSON=$("$PYTHON_BIN" <<PY
import json, os
print(json.dumps({
"ec_connector": "ECMooncakeConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
},
}, separators=(",", ":")))
PY
)
PD_EC_JSON=$("$PYTHON_BIN" <<PY
import json, os
print(json.dumps({
"ec_connector": "ECMooncakeConnector",
"ec_role": "ec_consumer",
"ec_ip": os.environ["EC_MOONCAKE_RESERVATION_HOST"],
"ec_port": int(os.environ.get("EC_MOONCAKE_RESERVATION_PORT", "19019")),
"ec_connector_extra_config": {
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
},
}, separators=(",", ":")))
PY
)
EC_MOONCAKE_RESERVATION_ADDR=$("$PYTHON_BIN" <<'PY'
import os
from vllm.utils.network_utils import make_zmq_path
print(make_zmq_path(
"tcp", os.environ["EC_MOONCAKE_RESERVATION_HOST"],
int(os.environ["EC_MOONCAKE_RESERVATION_PORT"]),
))
PY
)
wait_for_server() {
local port=$1
local pid=$2
local deadline=$((SECONDS + TIMEOUT_SECONDS))
while ((SECONDS < deadline)); do
kill -0 "$pid" 2>/dev/null || return 1
if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
cleanup_instances() {
if ((${#PIDS[@]} == 0)); then
return
fi
echo "Cleaning up tracked vLLM / proxy processes..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
for _ in {1..10}; do
local alive=0
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
alive=1
fi
done
((alive == 0)) && break
sleep 1
done
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
kill -KILL "$pid" 2>/dev/null || true
fi
wait "$pid" 2>/dev/null || true
done
PIDS=()
}
finish() {
local status=$?
cleanup_instances
if ((status != 0)); then
for log in "${LOG_PATH}"/mooncake_epd_*.log; do
[[ -f "$log" ]] || continue
echo "=== $log (last 100 lines) ==="
tail -100 "$log"
done
fi
exit "$status"
}
trap finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
run_baseline() {
echo "================================"
echo "BASELINE (single vLLM, MM if enabled)"
echo "================================"
cleanup_instances
local PORT=$BASELINE_PORT
echo "Starting baseline on GPU $GPU_SINGLE port $PORT"
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$PORT" \
--gpu-memory-utilization 0.75 \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--no-enable-prefix-caching \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
>"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 &
local BASELINE_PID=$!
PIDS+=("$BASELINE_PID")
echo "Waiting for baseline..."
wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; }
curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true
echo ""
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PORT" \
--model_name "$MODEL" \
--mode baseline \
--baseline_file "$BASELINE_FILE" \
"${TEST_ARGS[@]}"
cleanup_instances
}
run_epd_mooncake() {
echo "================================"
echo "EPD 1E + 1PD with ECMooncakeConnector"
echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT"
echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL"
echo "================================"
cleanup_instances
echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"
VLLM_SERVER_DEV_MODE=1 CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$ENCODE_PORT" \
--gpu-memory-utilization 0.35 \
--mm-tensor-ipc torch_shm \
--enable-request-id-headers \
--no-enable-prefix-caching \
--max-num-batched-tokens "$MAX_MODEL_LEN" \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
--ec-transfer-config "$ENC_EC_JSON" \
--compilation-config "$E_COMPILATION_CONFIG" \
"${E_ATTN_ARGS[@]}" \
--worker-extension-cls tests.v1.ec_connector.integration.test_epd_correctness.EncoderGraphTestWorkerExtension \
>"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &
local ENCODER_PID=$!
PIDS+=("$ENCODER_PID")
echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT"
CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$PREFILL_DECODE_PORT" \
--gpu-memory-utilization 0.75 \
--mm-tensor-ipc torch_shm \
--enable-mm-embeds \
--enable-request-id-headers \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--no-enable-prefix-caching \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
--ec-transfer-config "$PD_EC_JSON" \
>"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 &
local PD_PID=$!
PIDS+=("$PD_PID")
echo "Waiting for encoder..."
wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; }
echo "Waiting for PD..."
wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; }
echo "Starting EPD proxy on $ENDPOINT_PORT"
"$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
--host "0.0.0.0" \
--port "$ENDPOINT_PORT" \
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
--prefill-servers-urls "disable" \
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
--ec-consumer-zmq-addrs \
"$EC_MOONCAKE_RESERVATION_ADDR" \
>"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 &
local PROXY_PID=$!
PIDS+=("$PROXY_PID")
echo "Waiting for proxy..."
wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; }
curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true
echo ""
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$ENDPOINT_PORT" \
--model_name "$MODEL" \
--mode disagg \
--baseline_file "$BASELINE_FILE" \
"${TEST_ARGS[@]}"
curl -fsS "http://localhost:${ENCODE_PORT}/collective_rpc" \
-H 'Content-Type: application/json' \
-d '{"method":"encoder_graph_stats"}' \
>"${LOG_PATH}/encoder_graph_stats.json"
"$PYTHON_BIN" - "${LOG_PATH}/encoder_graph_stats.json" "${E_CUDAGRAPH_MM_ENCODER:-0}" <<'PY'
import json
import sys
with open(sys.argv[1]) as f:
results = json.load(f)["results"]
assert results, "No encoder workers returned graph statistics"
for stats in results:
if sys.argv[2] == "1":
assert stats["is_captured"], stats
assert stats["graph_hits"] > 0, stats
else:
assert stats["graph_hits"] == 0, stats
print("Encoder graph statistics:", results)
PY
cleanup_instances
}
echo "================================"
echo "1E + 1PD ECMooncake end-to-end correctness"
echo "MODEL=$MODEL"
echo "================================"
if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then
run_baseline
else
echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE"
[[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; }
fi
run_epd_mooncake
echo "================================"
echo "PASS: Mooncake EC EPD matches baseline"
echo "Logs: ${LOG_PATH}/mooncake_epd_*.log"
echo "================================"
@@ -1,395 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EPD Correctness Test.
Tests that EPD (Encoder-Prefill-Decode) disaggregation produces the same
outputs as a baseline single instance.
Usage:
# Baseline mode (saves outputs):
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode baseline \
--baseline_file .vllm_epd_baseline.txt
# Disagg mode (compares outputs):
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode disagg \
--baseline_file .vllm_epd_baseline.txt
"""
import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING
import openai
import requests
from vllm.assets.image import ImageAsset
from vllm.multimodal.utils import encode_image_url
if TYPE_CHECKING:
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
class EncoderGraphTestWorkerExtension:
"""Expose encoder replay evidence through the existing test RPC endpoint."""
model_runner: "MMEncoderModelRunner"
def encoder_graph_stats(self):
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
runner = self.model_runner
assert isinstance(runner, MMEncoderModelRunner)
manager = runner.model_state.encoder_runner.cudagraph_manager
if manager is None:
return {"is_captured": False, "graph_hits": 0, "graph_misses": 0}
return {"is_captured": manager.is_captured(), **manager.get_cumulative_stats()}
MAX_OUTPUT_LEN = 256
# Sample prompts with multimodal content
image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720))
image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720))
image_local_path = f"{os.path.dirname(os.path.abspath(__file__))}/hato.jpg"
SAMPLE_PROMPTS_MM: list[dict] = [
{
"messages": [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": encode_image_url(image_1)},
},
{"type": "text", "text": "What's in this image?"},
],
}
],
"description": "Single image query",
},
{
"messages": [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": encode_image_url(image_2)},
},
{
"type": "image_url",
"image_url": {"url": f"file://{image_local_path}"},
},
{"type": "text", "text": "Describe these 2 images in detail."},
],
}
],
"description": "2 images with detailed query",
},
]
# Text-only prompts for mixed testing
SAMPLE_PROMPTS_TEXT: list[dict] = [
{
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"description": "Simple text-only query",
},
{
"messages": [
{"role": "user", "content": "Explain quantum computing in simple terms."}
],
"description": "Text-only explanation request",
},
]
def check_vllm_server(url: str, timeout=5, retries=10) -> bool:
"""Check if the vLLM server is ready.
Args:
url: The URL to check (usually /health or /healthcheck endpoint)
timeout: Timeout in seconds for each request
retries: Number of retries if the server is not ready
Returns:
True if the server is ready, False otherwise
"""
for attempt in range(retries):
try:
response = requests.get(url, timeout=timeout)
if response.status_code == 200:
print(f"Server is ready at {url}")
return True
else:
print(
f"Attempt {attempt + 1}/{retries}: Server returned "
f"status code {response.status_code}"
)
except requests.exceptions.RequestException as e:
print(f"Attempt {attempt + 1}/{retries}: Error connecting: {e}")
time.sleep(2) # Wait before retrying
return False
def run_chat_completion(
base_url: str,
model_name: str,
messages: list,
max_tokens: int = MAX_OUTPUT_LEN,
) -> str:
"""Run a chat completion request.
Args:
base_url: Base URL of the vLLM server
model_name: Name of the model
messages: Messages for chat completion
max_tokens: Maximum tokens to generate
Returns:
Generated text content
"""
with openai.OpenAI(
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
) as client:
completion = client.chat.completions.create(
model=model_name,
messages=messages,
max_tokens=max_tokens,
temperature=0.0,
seed=42,
)
content = completion.choices[0].message.content
assert content, "Expected a nonempty completion"
return content
def main():
"""Main test function."""
parser = argparse.ArgumentParser(
description="EPD correctness test - compare disagg vs baseline"
)
parser.add_argument(
"--service_url",
type=str,
required=True,
help="The vLLM service URL (e.g., http://localhost:8000)",
)
parser.add_argument(
"--model_name",
type=str,
required=True,
help="Model name",
)
parser.add_argument(
"--mode",
type=str,
default="baseline",
choices=["baseline", "baseline_pd", "disagg"],
help="Mode: baseline/baseline_pd (saves outputs) or disagg (compares outputs)",
)
parser.add_argument(
"--baseline_file",
type=str,
default=".vllm_epd_baseline.txt",
help="File to save/load baseline outputs",
)
parser.add_argument(
"--use_mm_prompts",
action="store_true",
help="Use multimodal prompts (default: use text-only for quick testing)",
)
parser.add_argument(
"--skip_two_image_prompt",
action="store_true",
help="Skip the two-image multimodal prompt",
)
parser.add_argument(
"--mm_smoke_test",
action="store_true",
help="Use three short, fixed-answer image cases, including duplicate images",
)
parser.add_argument("--concurrency", type=int, default=1)
parser.add_argument("--repeat", type=int, default=1)
args = parser.parse_args()
if args.concurrency < 1 or args.repeat < 1:
parser.error("--concurrency and --repeat must be positive")
if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt):
parser.error("--mm_smoke_test requires all multimodal prompts")
print(f"Service URL: {args.service_url}")
print(f"Model: {args.model_name}")
print(f"Mode: {args.mode}")
print(f"Output file: {args.baseline_file}")
print(f"Use MM prompts: {args.use_mm_prompts}")
# Determine health check endpoint
if args.mode == "baseline":
health_check_url = f"{args.service_url}/health"
elif args.mode == "baseline_pd":
# Nixl toy proxy use /healthcheck
health_check_url = f"{args.service_url}/healthcheck"
else:
# Disagg EPD proxy uses /health
health_check_url = f"{args.service_url}/health"
if not os.path.exists(args.baseline_file):
raise ValueError(
f"In disagg mode, the output file {args.baseline_file} from "
"baseline does not exist. Run baseline mode first."
)
# Check if server is ready
if not check_vllm_server(health_check_url):
raise RuntimeError(f"vLLM server at {args.service_url} is not ready!")
# Select prompts to use
if args.use_mm_prompts:
test_prompts = (
SAMPLE_PROMPTS_MM[:1] if args.skip_two_image_prompt else SAMPLE_PROMPTS_MM
)
print("Using multimodal prompts")
else:
test_prompts = SAMPLE_PROMPTS_TEXT
print("Using text-only prompts for quick testing")
if args.mm_smoke_test:
image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0]
image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2]
cases = [
(
"Single image",
[image],
"What word is on the red road sign? Reply with only that word "
"in uppercase.",
"STOP",
),
(
"Two different images",
image_pair,
"Do these pictures contain both flowers and birds? "
"Reply with only YES or NO.",
"YES",
),
(
"Same image twice",
[image, image],
"Read the red road sign in each image, in image order. Reply "
"with only the two uppercase words separated by a comma and a space.",
"STOP, STOP",
),
]
test_prompts = [
{
"description": description,
"expected": expected,
"messages": [
{
"role": "user",
"content": [*images, {"type": "text", "text": question}],
}
],
}
for description, images, question, expected in cases
]
# Run completions
service_url = f"{args.service_url}/v1"
output_strs = {}
def complete(prompt_data):
output = run_chat_completion(
base_url=service_url,
model_name=args.model_name,
messages=prompt_data["messages"],
max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN,
)
if args.mm_smoke_test:
output = output.strip()
assert output == prompt_data["expected"], (
f"{prompt_data['description']}: expected {prompt_data['expected']!r}, "
f"got {output!r}"
)
return output
# Each round includes concurrent requests sharing image hashes; later rounds
# exercise reuse after previous requests have finished.
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
for repeat in range(args.repeat):
outputs = executor.map(complete, test_prompts)
for prompt_data, output_str in zip(test_prompts, outputs):
key = prompt_data["description"]
if args.repeat > 1:
key = f"{key} (round {repeat + 1})"
output_strs[key] = output_str
print(f"{key}: {output_str}")
if args.mode in ("baseline", "baseline_pd"):
# Baseline mode: Save outputs
print(f"\nSaving baseline outputs to {args.baseline_file}")
try:
with open(args.baseline_file, "w") as json_file:
json.dump(output_strs, json_file, indent=4)
print("✅ Baseline outputs saved successfully")
except OSError as e:
print(f"Error writing to file: {e}")
raise
else:
# Disagg mode: Load and compare outputs
print(f"\nLoading baseline outputs from {args.baseline_file}")
baseline_outputs = None
try:
with open(args.baseline_file) as json_file:
baseline_outputs = json.load(json_file)
except OSError as e:
print(f"Error reading from file: {e}")
raise
# Verify outputs match
print("\nComparing disagg outputs with baseline...")
assert isinstance(baseline_outputs, dict), "Baseline outputs should be a dict"
assert len(baseline_outputs) == len(output_strs), (
f"Length mismatch: baseline has {len(baseline_outputs)}, "
f"disagg has {len(output_strs)}"
)
all_match = True
for key, baseline_output in baseline_outputs.items():
assert key in output_strs, f"{key} not in disagg outputs"
disagg_output = output_strs[key]
if baseline_output == disagg_output:
print(f"✅ {key}: MATCH")
else:
print(f"❌ {key}: MISMATCH")
print(f" Baseline: {baseline_output}")
print(f" Disagg: {disagg_output}")
all_match = False
assert all_match, "❌❌Disagg outputs do not match baseline!❌❌"
if all_match:
print("\n✅ All outputs match! Test PASSED")
if __name__ == "__main__":
main()
@@ -1,389 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for EmbeddingCache.
Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere.
"""
import threading
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
def _cache(num_blocks: int = 8) -> EmbeddingCache:
return EmbeddingCache(num_blocks)
# ── alloc ────────────────────────────────────────────────────────────────────
def test_alloc_returns_unique_ascending_block_ids():
cache = _cache()
entry = cache.alloc("a", 3)
assert entry is not None
assert len(entry.block_ids) == 3
assert len(set(entry.block_ids)) == 3
assert all(0 <= i < 8 for i in entry.block_ids)
# Ascending order lets consumers detect which blocks are adjacent.
assert list(entry.block_ids) == sorted(entry.block_ids)
def test_alloc_returns_tuple():
cache = _cache()
entry = cache.alloc("a", 2)
assert isinstance(entry.block_ids, tuple)
def test_alloc_returns_none_when_full():
cache = _cache(num_blocks=4)
cache.alloc("a", 4)
assert cache.alloc("b", 1) is None
def test_alloc_asserts_duplicate_key():
cache = _cache()
cache.alloc("a", 2)
with pytest.raises(AssertionError, match="duplicate alloc"):
cache.alloc("a", 2)
def test_alloc_asserts_n_blocks_exceeds_capacity():
cache = _cache(num_blocks=4)
with pytest.raises(AssertionError, match="capacity"):
cache.alloc("a", 5)
def test_alloc_zero_blocks():
cache = _cache()
entry = cache.alloc("a", 0)
assert entry is not None
assert entry.block_ids == ()
def test_alloc_starts_not_ready():
cache = _cache()
entry = cache.alloc("a", 2)
assert not entry.ready
assert not entry.evictable
# ── get ──────────────────────────────────────────────────────────────────────
def test_get_missing_returns_none():
cache = _cache()
assert cache.get("a") is None
def test_get_returns_entry():
cache = _cache()
alloc_entry = cache.alloc("a", 2)
got = cache.get("a")
assert got is alloc_entry
def test_get_returns_not_ready_entry():
cache = _cache()
cache.alloc("a", 2)
entry = cache.get("a")
assert entry is not None
assert not entry.ready
# ── mark_ready ───────────────────────────────────────────────────────────────
def test_mark_ready_transitions():
cache = _cache()
cache.alloc("a", 2)
entry = cache.get("a")
assert not entry.ready
cache.mark_ready("a")
assert entry.ready
assert entry.evictable
def test_mark_ready_asserts_already_ready():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
with pytest.raises(AssertionError, match="already-ready"):
cache.mark_ready("a")
def test_mark_ready_asserts_missing_key():
cache = _cache()
with pytest.raises(KeyError):
cache.mark_ready("nonexistent")
# ── pin / unpin ──────────────────────────────────────────────────────────────
def test_pin_asserts_not_ready():
cache = _cache()
cache.alloc("a", 2)
with pytest.raises(AssertionError, match="not-ready"):
cache.pin("a")
def test_pin_unpin_basic():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
entry = cache.get("a")
assert not entry.evictable
cache.unpin("a")
assert entry.evictable
def test_nested_pin():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.pin("a")
cache.unpin("a")
entry = cache.get("a")
assert not entry.evictable # still pinned (count=1)
cache.unpin("a")
assert entry.evictable
def test_unpin_asserts_on_unpinned():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
with pytest.raises(AssertionError, match="unpinned"):
cache.unpin("a")
# ── has_held_entries ─────────────────────────────────────────────────────────
def test_has_held_entries_empty_cache():
cache = _cache()
assert cache.has_held_entries() is False
def test_has_held_entries_true_while_not_ready():
cache = _cache()
cache.alloc("a", 2) # not ready → held
assert cache.has_held_entries() is True
cache.mark_ready("a") # ready + unpinned → settled
assert cache.has_held_entries() is False
def test_has_held_entries_true_while_pinned():
cache = _cache()
cache.alloc("a", 2)
cache.mark_ready("a")
assert cache.has_held_entries() is False
cache.pin("a") # pinned → held
assert cache.has_held_entries() is True
cache.unpin("a")
assert cache.has_held_entries() is False
# ── eviction ─────────────────────────────────────────────────────────────────
def test_eviction_frees_ready_unpinned_fifo():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.alloc("b", 2)
cache.mark_ready("b")
# Pool full. Alloc should evict "a" (oldest).
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is None # evicted
assert cache.get("b") is not None # still there
def test_eviction_skips_pinned():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.alloc("b", 2)
cache.mark_ready("b")
# "a" is pinned, so "b" should be evicted instead.
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is not None # pinned, survived
assert cache.get("b") is None # evicted
cache.unpin("a")
def test_eviction_skips_not_ready():
cache = _cache(num_blocks=4)
cache.alloc("a", 2) # not ready
cache.alloc("b", 2)
cache.mark_ready("b")
# "a" is not ready (non-evictable), "b" is evictable.
entry_c = cache.alloc("c", 2)
assert entry_c is not None
assert cache.get("a") is not None # not ready, survived
assert cache.get("b") is None # evicted
cache.mark_ready("a") # clean up
def test_eviction_returns_none_if_all_non_evictable():
cache = _cache(num_blocks=4)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.pin("a")
cache.alloc("b", 2) # not ready
# Both entries are non-evictable (a=pinned, b=not-ready).
assert cache.alloc("c", 1) is None
# Fail-fast: nothing was evicted.
assert cache.get("a") is not None
assert cache.get("b") is not None
cache.unpin("a")
def test_fail_fast_no_wasted_eviction():
cache = _cache(num_blocks=6)
cache.alloc("a", 2)
cache.mark_ready("a") # evictable: 2 blocks
cache.alloc("b", 2) # not ready
cache.alloc("c", 2) # not ready
# free=0, evictable=2, need 3 -> cannot succeed
assert cache.alloc("d", 3) is None
# "a" was NOT evicted despite being evictable
assert cache.get("a") is not None
def test_eviction_multiple_entries():
cache = _cache(num_blocks=6)
cache.alloc("a", 2)
cache.mark_ready("a")
cache.alloc("b", 2)
cache.mark_ready("b")
cache.alloc("c", 2)
cache.mark_ready("c")
# Need 4 blocks: must evict both "a" and "b".
entry_d = cache.alloc("d", 4)
assert entry_d is not None
assert cache.get("a") is None
assert cache.get("b") is None
assert cache.get("c") is not None
# ── thread safety ────────────────────────────────────────────────────────────
def test_concurrent_pin_unpin():
cache = _cache(num_blocks=8)
cache.alloc("a", 4)
cache.mark_ready("a")
errors: list[Exception] = []
barrier = threading.Barrier(8)
def worker():
barrier.wait()
try:
for _ in range(500):
cache.pin("a")
cache.unpin("a")
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(8)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, errors
entry = cache.get("a")
assert entry.evictable # all pins balanced
def test_concurrent_alloc_evict():
cache = _cache(num_blocks=4)
errors: list[Exception] = []
barrier = threading.Barrier(4)
counter = {"i": 0}
counter_lock = threading.Lock()
def worker():
barrier.wait()
try:
for _ in range(200):
with counter_lock:
key = f"k{counter['i']}"
counter["i"] += 1
entry = cache.alloc(key, 1)
if entry is not None:
cache.mark_ready(key)
except Exception as exc:
errors.append(exc)
threads = [threading.Thread(target=worker) for _ in range(4)]
for t in threads:
t.start()
for t in threads:
t.join()
assert not errors, errors
# ── discard ──────────────────────────────────────────────────────────────────
def test_discard_not_ready_returns_blocks():
cache = EmbeddingCache(num_blocks=4)
entry = cache.alloc("h", 3)
assert entry is not None
cache.discard("h")
assert cache.get("h") is None
# All blocks reclaimed: a fresh 4-block alloc now fits.
assert cache.alloc("h2", 4) is not None
def test_discard_rejects_ready_entry():
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1)
cache.mark_ready("h")
with pytest.raises(AssertionError):
cache.discard("h")
def test_discard_rejects_absent_key():
cache = EmbeddingCache(num_blocks=4)
with pytest.raises(KeyError):
cache.discard("nope")
# ── pin_if_ready ─────────────────────────────────────────────────────────────
def test_pin_if_ready_pins_and_returns_entry():
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 2)
cache.mark_ready("h")
entry = cache.pin_if_ready("h")
assert entry is not None and entry.ready and len(entry.block_ids) == 2
assert cache.alloc("h2", 4) is None # pinned ⇒ not evictable
def test_pin_if_ready_none_when_absent():
cache = EmbeddingCache(num_blocks=4)
assert cache.pin_if_ready("nope") is None
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
"""The producer needs "present but not ready" told apart from "absent":
the first is NACKed as retryable, the second as a miss."""
cache = EmbeddingCache(num_blocks=4)
cache.alloc("h", 1) # not-ready
entry = cache.pin_if_ready("h")
assert entry is not None and not entry.ready
assert cache.alloc("h2", 4) is None # not-ready ⇒ still not evictable
@@ -1,466 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.config.ec_transfer import ECRole
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N = 16
_BS = 64
class _Pos:
def __init__(self, offset, length):
self.offset = offset
self.length = length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, identifier=None):
self.mm_hash = mm_hash
self.identifier = identifier if identifier is not None else mm_hash
self.mm_position = _Pos(0, length)
self.modality = "image"
class _Request:
_counter = 0
def __init__(self, features, request_id=None):
self.mm_features = features
if request_id is None:
_Request._counter += 1
self.request_id = f"req_{_Request._counter}"
else:
self.request_id = request_id
class _WorkerOutput:
"""Stand-in for ECConnectorOutput carrying a completion report.
`saves` are mm_hashes; `loads` are transfer ids. One entry per reporting
rank, so a list holding the same id twice is two ranks reporting it.
"""
def __init__(self, *, saves=None, loads=None):
self.ec_connector_worker_meta = ECCPUWorkerMetadata(
completed_saves=list(saves or []),
completed_loads=list(loads or []),
)
def _make_scheduler(
monkeypatch,
num_blocks=_N,
*,
ec_role: ECRole = "ec_both",
tensor_parallel_size=1,
) -> ECCPUScheduler:
region = ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_blocks,
block_size_bytes=_BS,
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
return ECCPUScheduler(
create_ec_vllm_config(
ec_role=ec_role,
tensor_parallel_size=tensor_parallel_size,
)
)
def _load_ids(meta) -> list[int]:
"""Transfer ids the scheduler dispatched in this step's metadata."""
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
def _load_blocks(meta, mm_hash: str) -> list[int]:
"""Block ids the scheduler dispatched for one load."""
return meta.loads[mm_hash][1]
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
"""Pre-populate a ready cache entry backed by real blocks."""
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
s._cache.mark_ready(mm_hash)
def test_offload_reuse_cycle(monkeypatch):
s = _make_scheduler(monkeypatch)
# Two blocks, so the reload also pins down that every allocated block id
# comes back in meta.loads, not just the first.
req = _Request([_Feature("h1", length=2)])
# Step A: first sight — allocate and save.
assert s.has_cache_item("h1") is False
assert s.ensure_cache_available(req, 0) is True
s.update_state_after_alloc(req, 0)
meta_a = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_a.saves
assert meta_a.loads == {}
# Not ready until the worker reports the save memcpy complete.
assert s.has_cache_item("h1") is False
# Worker reports the save copy finished → entry becomes ready.
s.update_connector_output(_WorkerOutput(saves=["h1"]))
assert s.has_cache_item("h1") is True
# Step C: reload now works — entry is ready at update_state_after_alloc time.
s.update_state_after_alloc(req, 0)
meta_c = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta_c.loads
assert _load_blocks(meta_c, "h1") == meta_a.saves["h1"]
s.shutdown()
def test_has_cache_item_false_when_not_consumer(monkeypatch):
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
assert s.has_cache_item("anything") is False
s.shutdown()
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
"""The connector must key the encoder cache on feature.identifier (what
has_cache_item is called with), NOT feature.mm_hash."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "ENC_KEY" in meta.saves
assert "PROC_KEY" not in meta.saves
# Not ready until the completion report.
assert s.has_cache_item("ENC_KEY") is False
# After the report it becomes ready — keyed on identifier.
s.update_connector_output(_WorkerOutput(saves=["ENC_KEY"]))
assert s.has_cache_item("ENC_KEY") is True
assert s.has_cache_item("PROC_KEY") is False
s.shutdown()
def test_cpu_region_fifo_eviction(monkeypatch):
# Region holds exactly 2 one-block encodings.
s = _make_scheduler(monkeypatch, num_blocks=2)
# Fill with A and B, marking each ready via a completion report.
for h in ("A", "B"):
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
s.update_connector_output(_WorkerOutput(saves=[h]))
assert s.has_cache_item("A") is True
assert s.has_cache_item("B") is True
# Allocate C: must evict A (oldest) to make room.
s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_cache_item("A") is False # evicted
assert s.has_cache_item("B") is True
def test_multiple_mm_items_per_request(monkeypatch):
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)])
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(req, 1)
meta = s.build_connector_meta(scheduler_output=None)
assert set(meta.saves) == {"h1", "h2"}
# Worker reports both saves complete.
s.update_connector_output(_WorkerOutput(saves=["h1", "h2"]))
assert s.has_cache_item("h1") is True
assert s.has_cache_item("h2") is True
# Now reload both.
s.update_state_after_alloc(req, 0)
s.update_state_after_alloc(req, 1)
meta2 = s.build_connector_meta(scheduler_output=None)
assert set(meta2.loads) == {"h1", "h2"}
s.shutdown()
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
_seed_cached(s, "b", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
assert "b" not in meta.loads
s.shutdown()
def test_repeated_reload_same_step_loads_once(monkeypatch):
"""The same mm_hash requested twice in one step must appear in
meta.loads exactly once."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert list(meta.loads.keys()).count("a") == 1
s.shutdown()
def test_load_not_emitted_for_uncached_entry(monkeypatch):
"""A feature that was never saved must not appear in meta.loads."""
s = _make_scheduler(monkeypatch)
s.update_state_after_alloc(_Request([_Feature("missing")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert meta.loads == {}
s.shutdown()
def test_load_pin_protects_blocks_until_completion(monkeypatch):
"""Loaded blocks stay pinned until the worker reports the load memcpy
complete. Observable: a save that needs eviction cannot reclaim a
still-pinned loaded entry, but can once the completion report arrives."""
# 2 blocks, both occupied by "a" (ready).
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
# Load "a" — pins it.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "a" in meta.loads
(transfer_id,) = _load_ids(meta)
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "b" not in meta2.saves # can't evict pinned "a"
# The one participant reports the load complete → "a" unpins. Now "b" can
# evict it.
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
meta3 = s.build_connector_meta(scheduler_output=None)
assert "b" in meta3.saves
s.shutdown()
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
blocks currently held by other entries are reclaimable."""
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
meta = s.build_connector_meta(scheduler_output=None)
return mm_hash in meta.saves
def test_load_needs_every_participant_before_unpin(monkeypatch):
"""With two participating ranks, one report must not release the pin.
This is the multi-rank case: each rank copies the same CPU blocks into its
own GPU memory, so the blocks stay in use until the last rank is done.
"""
s = _make_scheduler(monkeypatch, num_blocks=2, tensor_parallel_size=2)
_seed_cached(s, "a", n_blocks=2)
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
(transfer_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
# First rank reports: still in use by the second, so "a" must hold.
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
assert _can_evict(s, "b", 2) is False
assert s.has_pending_push_work() is True
# Second rank reports: now it is genuinely free.
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
assert s.has_pending_push_work() is False
assert _can_evict(s, "c", 2) is True
s.shutdown()
def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
"""A straggling report from one dispatch must not release the pin taken by
a subsequent dispatch of the same mm_hash.
This is what the transfer id is for: mm_hash identifies a cache entry, not
a transfer, so it cannot tell the two dispatches apart.
"""
s = _make_scheduler(monkeypatch, num_blocks=2, tensor_parallel_size=2)
_seed_cached(s, "a", n_blocks=2)
# First dispatch, fully reported by both ranks → pin released.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
(first_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
s.update_connector_output(_WorkerOutput(loads=[first_id, first_id]))
assert s.has_pending_push_work() is False
# Second dispatch of the same hash takes a fresh pin.
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
(second_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
assert second_id != first_id
# A replayed report for the first dispatch must be ignored entirely.
s.update_connector_output(_WorkerOutput(loads=[first_id, first_id]))
assert s.has_pending_push_work() is True
assert _can_evict(s, "b", 2) is False
s.shutdown()
def test_region_full_skips_save_and_never_blocks(monkeypatch):
"""When the region is fully occupied by pinned entries, new saves are
silently skipped and ensure_cache_available never blocks."""
s = _make_scheduler(monkeypatch, num_blocks=1)
_seed_cached(s, "pinned", n_blocks=1)
s._cache.pin("pinned")
req = _Request([_Feature("new", length=1)])
assert s.ensure_cache_available(req, 0) is True
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "new" not in meta.saves
assert s.has_cache_item("new") is False
s._cache.unpin("pinned")
s.shutdown()
def test_producer_only_never_emits_loads(monkeypatch):
"""A producer-only scheduler must never populate meta.loads, even when
entries are ready."""
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
req = _Request([_Feature("h1", length=1)])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.saves
assert meta.loads == {}
s.update_connector_output(_WorkerOutput(saves=["h1"]))
# Even if we re-encounter the same feature, no load.
s.update_state_after_alloc(req, 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert meta2.loads == {}
s.shutdown()
def test_consumer_only_never_emits_saves(monkeypatch):
"""A consumer-only scheduler must never populate meta.saves."""
s = _make_scheduler(monkeypatch, ec_role="ec_consumer")
_seed_cached(s, "a", n_blocks=1)
req = _Request([_Feature("a", length=1)])
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert meta.saves == {}
assert "a" in meta.loads
s.shutdown()
def test_consumer_only_has_cache_item(monkeypatch):
"""Consumer-only: has_cache_item works normally for ready entries."""
s = _make_scheduler(monkeypatch, ec_role="ec_consumer")
assert s.has_cache_item("a") is False
_seed_cached(s, "a", n_blocks=1)
assert s.has_cache_item("a") is True
s.shutdown()
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
"""An entry already in the cache (from a prior save) must not trigger
a second allocation or appear in meta.saves again."""
s = _make_scheduler(monkeypatch)
req = _Request([_Feature("h1", length=1)])
# First encounter → save.
s.update_state_after_alloc(req, 0)
meta1 = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta1.saves
# Second encounter (same step or later) → no save.
s.update_state_after_alloc(req, 0)
meta2 = s.build_connector_meta(scheduler_output=None)
assert "h1" not in meta2.saves
s.shutdown()
# ── completion reporting (update_connector_output) ──────────────────────────
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
no-op — no cache mutation, no crash."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
class _Foreign:
ec_connector_worker_meta = object()
s.update_connector_output(_Foreign()) # must not raise
assert s.has_cache_item("a") is True
s.shutdown()
def test_update_connector_output_ignores_unknown_report(monkeypatch):
"""Reports for a never-seen save hash or a transfer id this scheduler never
dispatched are dropped rather than mutating unrelated state."""
s = _make_scheduler(monkeypatch)
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
assert s.has_cache_item("gone") is False
s.shutdown()
# ── has_pending_push_work ───────────────────────────────────────────────────
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
"""A dispatched-but-unconfirmed save keeps push work pending until the
completion report marks it ready."""
s = _make_scheduler(monkeypatch)
assert s.has_pending_push_work() is False
s.update_state_after_alloc(_Request([_Feature("h1", length=1)]), 0)
s.build_connector_meta(scheduler_output=None)
assert s.has_pending_push_work() is True # save not yet confirmed
s.update_connector_output(_WorkerOutput(saves=["h1"]))
assert s.has_pending_push_work() is False
s.shutdown()
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
"""A dispatched-but-unconfirmed load keeps push work pending until the
unpin completion report arrives."""
s = _make_scheduler(monkeypatch)
_seed_cached(s, "a", n_blocks=1)
assert s.has_pending_push_work() is False # ready + unpinned
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
meta = s.build_connector_meta(scheduler_output=None)
assert s.has_pending_push_work() is True # pinned, awaiting unpin
s.update_connector_output(_WorkerOutput(loads=_load_ids(meta)))
assert s.has_pending_push_work() is False
s.shutdown()
# ── shutdown ────────────────────────────────────────────────────────────────
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
"""After shutdown the scheduler serves no items and the region is
cleaned up."""
s = _make_scheduler(monkeypatch, num_blocks=2)
_seed_cached(s, "a", n_blocks=2)
assert s.has_cache_item("a") is True
s.shutdown()
assert s.has_cache_item("a") is False # consumer role disabled
@@ -1,71 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
def _cfg():
ec = MagicMock()
ec.is_ec_producer = True
ec.is_ec_consumer = True
cfg = MagicMock()
cfg.ec_transfer_config = ec
return cfg
def test_scheduler_role_builds_only_scheduler(monkeypatch):
fake_sched = MagicMock()
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
assert c.connector_scheduler is fake_sched
assert c.connector_worker is None
c.has_cache_item("x")
fake_sched.has_cache_item.assert_called_once_with("x")
def test_worker_role_builds_only_worker(monkeypatch):
fake_worker = MagicMock()
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker)
c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER)
assert c.connector_worker is fake_worker
assert c.connector_scheduler is None
@pytest.mark.parametrize("role", [ECConnectorRole.SCHEDULER, ECConnectorRole.WORKER])
def test_v1_model_runner_is_rejected(monkeypatch, role):
"""The V1 runner never reports save completions, so refuse to start."""
monkeypatch.setattr(
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
)
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: MagicMock())
cfg = _cfg()
cfg.use_v2_model_runner = False
with pytest.raises(ValueError, match="V2 model runner"):
ECCPUConnector(cfg, role)
def test_factory_registered():
cls = ECConnectorFactory._registry["ECCPUConnector"]()
assert cls is ECCPUConnector
def test_request_finished_forwards_to_scheduler(monkeypatch):
from unittest.mock import MagicMock
import vllm.distributed.ec_transfer.ec_connector.cpu.connector as conn_mod # noqa: F401
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
fake_sched = MagicMock()
fake_sched.request_finished.return_value = (False, {"h1": {"peer_port": 1}})
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
req = MagicMock()
assert c.request_finished(req) == (False, {"h1": {"peer_port": 1}})
fake_sched.request_finished.assert_called_once_with(req)
@@ -1,396 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECSharedRegion (the mmap substrate)."""
import contextlib
import ctypes
import errno
import mmap
import os
import uuid
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.config import ParallelConfig
from vllm.distributed.ec_transfer.ec_connector.cpu import common
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
_wait_for_file_size,
)
def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion:
return ECSharedRegion(
engine_id=str(uuid.uuid4()),
num_blocks=num_blocks,
block_size_bytes=block_size_bytes,
)
@pytest.fixture
def region() -> ECSharedRegion:
r = _make_region()
yield r
r.cleanup()
@contextlib.contextmanager
def _region(**kwargs):
"""Context manager: create one region, clean up on exit."""
r = _make_region(**kwargs)
try:
yield r
finally:
r.cleanup()
def _page_residency(mmap_obj: mmap.mmap, length: int) -> list[bool]:
"""Return Linux page-residency bits for a writable mmap."""
try:
libc = ctypes.CDLL(None, use_errno=True)
mincore = libc.mincore
except AttributeError:
pytest.skip("mincore is unavailable")
page_count = (length + mmap.PAGESIZE - 1) // mmap.PAGESIZE
mincore.argtypes = [
ctypes.c_void_p,
ctypes.c_size_t,
ctypes.POINTER(ctypes.c_ubyte),
]
mincore.restype = ctypes.c_int
vector = (ctypes.c_ubyte * page_count)()
address = ctypes.addressof(ctypes.c_ubyte.from_buffer(mmap_obj))
result = mincore(ctypes.c_void_p(address), length, vector)
if result != 0:
raise OSError(ctypes.get_errno())
return [bool(value & 1) for value in vector]
# ── mmap sharing between two instances ───────────────────────────────────────
def test_second_instance_opens_existing_file_and_shares_memory():
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
try:
r2 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
try:
assert r1._is_creator
assert not r2._is_creator
# Both map the same physical pages; writes via r1 are visible via r2.
r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8)
assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40]
finally:
r2.cleanup()
finally:
r1.cleanup()
def test_dense_dp_engines_get_distinct_regions():
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
so the region must be keyed by data_parallel_index."""
instance_id = str(uuid.uuid4())
regions = []
try:
for dp_index in range(2):
parallel_config = ParallelConfig(
data_parallel_size=2, data_parallel_rank=dp_index
)
parallel_config.reconfigure_for_independent_dp_rank()
cfg = MagicMock()
cfg.instance_id = instance_id
cfg.parallel_config = parallel_config
cfg.model_config.dtype = torch.float16
cfg.model_config.get_inputs_embeds_size.return_value = 8
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
regions.append(common.create_ec_shared_region(cfg))
assert all(r._is_creator for r in regions)
assert regions[0]._mmap_path != regions[1]._mmap_path
finally:
for r in regions:
r.cleanup()
def test_only_creator_unlinks_file_on_cleanup():
"""Critical contract: if the non-creator unlinks, the creator's mmap path
becomes a dangling backing file and a third opener would create a new one
out from under the creator."""
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
path = r1._mmap_path
r2 = ECSharedRegion(
engine_id=instance_id,
num_blocks=4,
block_size_bytes=64,
)
# Non-creator goes away first — file must still be on disk for r1.
r2.cleanup()
assert os.path.exists(path), "non-creator cleanup must not unlink the file"
# Creator goes away — file is removed.
r1.cleanup()
assert not os.path.exists(path), "creator cleanup must unlink the file"
# ── _wait_for_file_size ──────────────────────────────────────────────────────
def test_wait_for_file_size_returns_when_already_big_enough(tmp_path):
"""The fast path: file already at expected size — return immediately."""
p = tmp_path / "f.bin"
p.write_bytes(b"\x00" * 128)
fd = os.open(str(p), os.O_RDONLY)
try:
_wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise
finally:
os.close(fd)
def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path):
p = tmp_path / "f.bin"
p.write_bytes(b"")
fd = os.open(str(p), os.O_RDONLY)
try:
with pytest.raises(TimeoutError):
_wait_for_file_size(fd, expected_size=4096, timeout=0.05)
finally:
os.close(fd)
# ── MADV_POPULATE_WRITE pre-faulting / fallback ──────────────────────────────
def test_madvise_success_selects_madvise_population(monkeypatch):
"""A successful probe must keep using MADV_POPULATE_WRITE — the fallback
helper must never be invoked on a kernel that accepts the advice.
Spies on both helpers so we can verify call counts: 1 probe call over a
single page + 1 populate call over the whole flat region, fallback 0 times.
"""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
madvise_calls: list[tuple[int, int, int]] = []
fallback_calls: list[tuple[int, int]] = []
def _spy_madvise(mm, off, ln):
madvise_calls.append((off, ln, id(mm)))
def _spy_fallback(mm, off, ln):
fallback_calls.append((off, ln))
monkeypatch.setattr(esr, "_madvise_populate_write", _spy_madvise)
monkeypatch.setattr(esr, "_fallback_populate_write", _spy_fallback)
num_blocks, block_size_bytes = 4, mmap.PAGESIZE
with _region(num_blocks=num_blocks, block_size_bytes=block_size_bytes):
total = num_blocks * block_size_bytes
mmap_id = madvise_calls[0][2]
assert madvise_calls == [
(0, mmap.PAGESIZE, mmap_id), # probe
(0, total, mmap_id), # flat pre-fault of the whole region
]
assert fallback_calls == [], (
"native-path constructor must not invoke the fallback helper"
)
def test_madvise_einval_selects_fallback_for_whole_region(monkeypatch):
"""An EINVAL probe must select the fallback for the whole flat region.
This is the regression under test: on Linux < 5.14 the constant is absent
from the `mmap` module, so advice=23 reaches a kernel that rejects it and
every ECSharedRegion construction used to abort with EINVAL.
"""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
fallback_calls: list[tuple[int, int]] = []
real_fallback = esr._fallback_populate_write
def _raise_einval(mm, off, ln):
raise OSError(errno.EINVAL, "simulated unsupported kernel")
def _spy_fallback(mm, off, ln):
fallback_calls.append((off, ln))
return real_fallback(mm, off, ln)
monkeypatch.setattr(esr, "_madvise_populate_write", _raise_einval)
monkeypatch.setattr(esr, "_fallback_populate_write", _spy_fallback)
num_blocks, block_size_bytes = 4, mmap.PAGESIZE
with _region(num_blocks=num_blocks, block_size_bytes=block_size_bytes) as r:
assert fallback_calls == [(0, num_blocks * block_size_bytes)]
# Region is still fully usable after taking the fallback path.
r.blocks[0, :4] = torch.tensor([1, 2, 3, 4], dtype=torch.int8)
assert r.blocks[0, :4].tolist() == [1, 2, 3, 4]
def test_non_creator_does_not_probe_or_populate(monkeypatch):
"""Only the creator pre-faults. A second opener must not probe the advice
nor re-touch pages the creator already populated."""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
instance_id = str(uuid.uuid4())
r1 = ECSharedRegion(
engine_id=instance_id, num_blocks=4, block_size_bytes=mmap.PAGESIZE
)
try:
calls: list[str] = []
monkeypatch.setattr(
esr,
"_madvise_populate_write",
lambda mm, off, ln: calls.append("madvise"),
)
monkeypatch.setattr(
esr,
"_fallback_populate_write",
lambda mm, off, ln: calls.append("fallback"),
)
r2 = ECSharedRegion(
engine_id=instance_id, num_blocks=4, block_size_bytes=mmap.PAGESIZE
)
try:
assert not r2._is_creator
assert calls == []
finally:
r2.cleanup()
finally:
r1.cleanup()
def test_fallback_populate_write_preserves_bytes_and_faults_pages():
"""Fallback preserves existing bytes and touches every target page.
`|= 0` rather than `= 0` matters here: a peer worker may already have
written EC data into the shared mmap before we pre-fault it.
"""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
size = 3 * mmap.PAGESIZE
if not hasattr(mmap, "MADV_DONTNEED"):
pytest.skip("MADV_DONTNEED is unavailable")
mmap_obj = mmap.mmap(
-1,
size,
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
try:
mmap_obj.madvise(mmap.MADV_DONTNEED, 0, size)
if any(_page_residency(mmap_obj, size)):
pytest.skip("kernel did not discard anonymous mmap pages")
mmap_obj[0] = 0xAB
assert _page_residency(mmap_obj, size) == [True, False, False]
esr._fallback_populate_write(mmap_obj, 0, size)
assert _page_residency(mmap_obj, size) == [True, True, True]
assert mmap_obj[0] == 0xAB
finally:
mmap_obj.close()
def test_madvise_unexpected_oserror_propagates(monkeypatch):
"""Only EINVAL triggers the fallback. Other OSErrors (e.g. EIO) must
propagate out of __init__, not be silently masked by the fallback branch.
"""
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
monkeypatch.setattr(
esr,
"_madvise_populate_write",
lambda mm, off, ln: (_ for _ in ()).throw(
OSError(errno.EIO, "simulated I/O failure")
),
)
engine_id = str(uuid.uuid4())
try:
with pytest.raises(OSError) as exc_info:
ECSharedRegion(
engine_id=engine_id, num_blocks=4, block_size_bytes=mmap.PAGESIZE
)
assert exc_info.value.errno == errno.EIO
finally:
# __init__ raised past the unlink-owning cleanup(), so drop the
# backing file here rather than leaking it into /dev/shm.
with contextlib.suppress(FileNotFoundError):
os.unlink(f"/dev/shm/vllm_ec_{engine_id}.mmap")
# ── pin_memory ───────────────────────────────────────────────────────────────
def test_pin_memory_success_sets_flag(region):
"""When cudaHostRegister returns 0, _is_pinned flips to True
and cleanup will correspondingly call cudaHostUnregister."""
fake_cudart = MagicMock()
success = MagicMock()
success.value = 0
fake_cudart.cudaHostRegister.return_value = success
fake_cudart.cudaHostUnregister.return_value = success
with (
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.cudart", return_value=fake_cudart),
):
region.pin_memory()
assert region._is_pinned is True
# cleanup must pair with cudaHostUnregister exactly once.
region.cleanup()
fake_cudart.cudaHostUnregister.assert_called_once()
def test_pin_memory_failure_leaves_flag_false():
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
cleanup must NOT call cudaHostUnregister on memory we never registered."""
r = _make_region()
try:
fake_cudart = MagicMock()
fail = MagicMock()
fail.value = 1 # non-zero == error
fake_cudart.cudaHostRegister.return_value = fail
with (
patch("torch.cuda.is_available", return_value=True),
patch("torch.cuda.cudart", return_value=fake_cudart),
):
r.pin_memory()
assert r._is_pinned is False
# Now run cleanup and verify cudaHostUnregister was NOT called.
r.cleanup()
fake_cudart.cudaHostUnregister.assert_not_called()
finally:
r.cleanup()
def test_pin_memory_noop_without_cuda(region):
"""pin_memory is a no-op when CUDA is not available."""
with patch("torch.cuda.is_available", return_value=False):
region.pin_memory()
assert region._is_pinned is False
# ── cleanup idempotency ───────────────────────────────────────────────────────
def test_cleanup_is_idempotent(region):
region.cleanup()
region.cleanup() # fixture calls a third time — must not raise
-285
View File
@@ -1,285 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for control-layer ABCs, ZmqClientConnection, and ECSession."""
from unittest.mock import MagicMock, patch
import pytest
import zmq
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
ControlConnection,
)
class _ConcreteConn(ControlConnection):
"""Minimal concrete subclass for ABC-instantiation tests."""
def __init__(self):
self._alive = True
self._sent = []
self._inbox = []
@property
def alive(self) -> bool:
return self._alive
def send(self, msg: bytes) -> None:
self._sent.append(msg)
def recv(self) -> list[bytes]:
msgs, self._inbox = self._inbox, []
return msgs
def mark_dead(self) -> None:
self._alive = False
def close(self) -> None:
self._alive = False
def test_control_connection_abc_cannot_be_instantiated_directly():
with pytest.raises(TypeError):
ControlConnection() # type: ignore[abstract]
def test_concrete_connection_send_recv():
conn = _ConcreteConn()
assert conn.alive
conn.send(b"hello")
conn._inbox.append(b"world")
assert conn.recv() == [b"world"]
assert conn._sent == [b"hello"]
def test_mark_dead_sets_alive_false():
conn = _ConcreteConn()
conn.mark_dead()
assert not conn.alive
def test_close_sets_alive_false():
conn = _ConcreteConn()
conn.close()
assert not conn.alive
# ── ZmqClientConnection ───────────────────────────────────────────────────────
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( # noqa: E402
ZmqClientConnection,
ZmqClientTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr # noqa: E402
def _conn(messages: list[list[bytes]], monitor=None) -> ZmqClientConnection:
"""Build a ZmqClientConnection whose dealer yields the given frame-lists."""
dealer = MagicMock()
dealer.recv_multipart.side_effect = [*messages, zmq.Again()]
return ZmqClientConnection(dealer=dealer, monitor=monitor)
def test_zmq_client_connection_recv_extracts_payloads_from_envelopes():
conn = _conn([[b"", b"msg1"], [b"", b"msg2"]])
assert conn.recv() == [b"msg1", b"msg2"]
def test_zmq_client_connection_recv_skips_frames_without_empty_delimiter():
# A frame that does not start with b"" is malformed and must be dropped.
conn = _conn([[b"", b"ok"], [b"no-delimiter", b"bad"]])
assert conn.recv() == [b"ok"]
def test_zmq_client_connection_recv_breaks_on_non_again_exception():
# A socket error should stop the drain and return whatever was collected.
dealer = MagicMock()
dealer.recv_multipart.side_effect = [[b"", b"first"], OSError("broken")]
conn = ZmqClientConnection(dealer=dealer, monitor=None)
assert conn.recv() == [b"first"]
def test_zmq_client_connection_send_prefixes_empty_delimiter():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.send(b"payload")
dealer.send_multipart.assert_called_once_with([b"", b"payload"])
def test_zmq_client_connection_mark_dead_does_not_close_socket():
# mark_dead signals liveness only; resource teardown is close()'s job.
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.mark_dead()
assert not conn.alive
dealer.close.assert_not_called()
def test_zmq_client_connection_close_without_monitor():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.close()
dealer.close.assert_called_once_with(linger=0)
assert not conn.alive
def test_zmq_client_connection_close_with_monitor_disables_then_closes_both():
# The disable_monitor → monitor.close → dealer.close sequence must be
# respected: closing the dealer before disabling the monitor can cause
# a zmq assertion in some versions.
dealer = MagicMock()
monitor = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
conn.close()
dealer.disable_monitor.assert_called_once()
monitor.close.assert_called_once_with(linger=0)
dealer.close.assert_called_once_with(linger=0)
assert not conn.alive
def test_zmq_client_connection_close_is_idempotent():
dealer = MagicMock()
conn = ZmqClientConnection(dealer=dealer, monitor=None)
conn.close()
conn.close() # must not raise
assert not conn.alive
# ── ZmqClientTransport ────────────────────────────────────────────────────────
def _make_transport() -> tuple[ZmqClientTransport, MagicMock]:
"""Return a ZmqClientTransport with a mocked ZMQ context."""
ctx = MagicMock()
ctx.socket.return_value = MagicMock()
with (
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context",
return_value=ctx,
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket",
return_value=MagicMock(),
),
patch(
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path",
return_value="tcp://host:0",
),
):
transport = ZmqClientTransport()
transport._ctx = ctx
return transport, ctx
def _inject_conn(transport: ZmqClientTransport, addr: PeerAddr) -> ZmqClientConnection:
"""Inject a ZmqClientConnection directly into the transport pool."""
conn = ZmqClientConnection(dealer=MagicMock(), monitor=MagicMock())
transport._connections[addr] = conn
return conn
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_connect_caches_connection(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx_cls.return_value = MagicMock()
mock_ctx_cls.return_value.socket.return_value = MagicMock()
mock_sock.return_value = MagicMock()
transport = ZmqClientTransport()
addr: PeerAddr = ("host", 1234)
assert transport.connect(addr) is transport.connect(addr)
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_connect_creates_independent_conns_per_addr(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx_cls.return_value = MagicMock()
mock_ctx_cls.return_value.socket.return_value = MagicMock()
mock_sock.return_value = MagicMock()
transport = ZmqClientTransport()
conn_a = transport.connect(("host-a", 1111))
conn_b = transport.connect(("host-b", 2222))
assert conn_a is not conn_b
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_detects_disconnected_peer(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 1234)
conn = _inject_conn(transport, addr)
mock_recv_mon.side_effect = [{"event": zmq.EVENT_DISCONNECTED}, zmq.Again()]
dead = transport.poll_dead()
assert addr in dead
assert not conn.alive
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_skips_conn_without_monitor(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 9999)
conn = ZmqClientConnection(dealer=MagicMock(), monitor=None)
transport._connections[addr] = conn
dead = transport.poll_dead()
assert dead == []
mock_recv_mon.assert_not_called()
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
def test_zmq_client_transport_poll_dead_ignores_non_disconnect_events(mock_recv_mon):
transport, _ = _make_transport()
addr: PeerAddr = ("host", 1234)
conn = _inject_conn(transport, addr)
# A CONNECTED event should not trigger a dead notification.
mock_recv_mon.side_effect = [{"event": zmq.EVENT_CONNECTED}, zmq.Again()]
dead = transport.poll_dead()
assert dead == []
assert conn.alive
def test_zmq_client_transport_remove_closes_and_evicts_conn():
transport, _ = _make_transport()
addr: PeerAddr = ("host", 5678)
conn = _inject_conn(transport, addr)
removed = transport.remove(addr)
assert removed is conn
assert addr not in transport._connections
conn.dealer.close.assert_called()
def test_zmq_client_transport_remove_returns_none_for_unknown_addr():
transport, _ = _make_transport()
assert transport.remove(("ghost", 0)) is None
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
def test_zmq_client_transport_close_closes_all_connections(
mock_ctx_cls, mock_path, mock_sock
):
mock_ctx = MagicMock()
mock_ctx_cls.return_value = mock_ctx
mock_ctx.socket.return_value = MagicMock()
dealer_a, dealer_b = MagicMock(), MagicMock()
mock_sock.side_effect = [dealer_a, dealer_b]
transport = ZmqClientTransport()
transport.connect(("a", 1))
transport.connect(("b", 2))
transport.close()
dealer_a.close.assert_called()
dealer_b.close.assert_called()
assert transport._connections == {}
-184
View File
@@ -1,184 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for DataTransport ABC and NixlDataTransport."""
from unittest.mock import MagicMock, patch
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
DataTransport,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
serialize_mem_descriptor,
)
_NIXL_PATH = "vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl"
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_nixl_mock(prep_side_effect=None) -> MagicMock:
nixl = MagicMock()
nixl.get_agent_metadata.return_value = b"agent-meta"
nixl.get_reg_descs.return_value = MagicMock()
nixl.get_xfer_descs.return_value = MagicMock()
nixl.prep_xfer_dlist.side_effect = prep_side_effect or [77, 88]
nixl.add_remote_agent.return_value = "remote-agent-1"
nixl.get_new_notifs.return_value = {}
nixl.check_xfer_state.return_value = "PROC"
return nixl
def _make_transport(nixl_mock=None):
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
mock = nixl_mock or _make_nixl_mock()
with (
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=mock),
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
):
t = NixlDataTransport("agent", 0, 4, 64, 256)
t._nixl = mock
return t
# ── ABC ───────────────────────────────────────────────────────────────────────
def test_data_transport_abc_cannot_be_instantiated():
with pytest.raises(TypeError):
DataTransport() # type: ignore[abstract]
# ── NixlDataTransport construction ───────────────────────────────────────────
def test_nixl_data_transport_registers_memory_on_init():
nixl = _make_nixl_mock()
_make_transport(nixl)
nixl.register_memory.assert_called_once()
def test_nixl_data_transport_unwinds_registration_if_init_fails():
"""A failure after register_memory must not leave the region pinned.
__init__ propagates, so the caller never gets a transport to deregister.
"""
nixl = _make_nixl_mock(prep_side_effect=RuntimeError("no dlist"))
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
with (
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=nixl),
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
pytest.raises(RuntimeError, match="no dlist"),
):
NixlDataTransport("agent", 0, 4, 64, 256)
nixl.deregister_memory.assert_called_once_with(nixl.get_reg_descs.return_value)
def test_nixl_data_transport_raises_if_nixl_unavailable():
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
NixlDataTransport,
)
with (
patch(f"{_NIXL_PATH}.NixlWrapper", None),
patch(f"{_NIXL_PATH}.nixl_agent_config", None),
pytest.raises(RuntimeError, match="requires NIXL"),
):
NixlDataTransport("a", 0, 1, 64, 64)
# ── public accessors ──────────────────────────────────────────────────────────
def test_get_agent_metadata_returns_nixl_blob():
assert _make_transport().get_agent_metadata() == b"agent-meta"
def test_get_mem_descriptor_roundtrips_block_descs():
t = _make_transport()
raw = t.get_mem_descriptor()
assert isinstance(raw, bytes) and len(raw) > 0
# ── add_remote_peer ───────────────────────────────────────────────────────────
def test_add_remote_peer_returns_agent_name_and_stores_handle_internally():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent_name = t.add_remote_peer(b"meta", mem_desc)
assert agent_name == "remote-agent-1"
# Dlist handle stored internally; accessible via _peer_handles.
assert t._peer_handles["remote-agent-1"] == 88
def test_add_remote_peer_registers_remote_agent():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
t.add_remote_peer(b"fresh-meta", mem_desc)
nixl.add_remote_agent.assert_called_once_with(b"fresh-meta")
# ── remove_remote_peer ────────────────────────────────────────────────────────
def test_remove_remote_peer_delegates_to_nixl():
t = _make_transport()
t.remove_remote_peer("agent-x")
t._nixl.remove_remote_agent.assert_called_once_with("agent-x")
def test_remove_remote_peer_swallows_exceptions():
t = _make_transport()
t._nixl.remove_remote_agent.side_effect = RuntimeError("gone")
t.remove_remote_peer("agent-x") # must not raise
# ── post_read ─────────────────────────────────────────────────────────────────
def test_post_read_raises_on_index_count_mismatch():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent = t.add_remote_peer(b"meta", mem_desc)
with pytest.raises(ValueError, match="mismatch"):
t.post_read([0, 1], agent, [5], notif_msg=b"sid")
def test_post_read_calls_make_prepped_xfer_and_transfer():
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
t = _make_transport(nixl)
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
agent = t.add_remote_peer(b"meta", mem_desc)
nixl.make_prepped_xfer.return_value = 999
handle = t.post_read([0, 1], agent, [5, 6], notif_msg=b"sid")
nixl.make_prepped_xfer.assert_called_once()
nixl.transfer.assert_called_once_with(999)
assert handle == 999
# ── release_xfer_handle / deregister ─────────────────────────────────────────
def test_release_xfer_handle_swallows_exceptions():
t = _make_transport()
t._nixl.release_xfer_handle.side_effect = RuntimeError("stale")
t.release_xfer_handle(42) # must not raise
def test_deregister_swallows_exceptions():
t = _make_transport()
t._nixl.deregister_memory.side_effect = RuntimeError("already gone")
t.deregister() # must not raise
@@ -1,85 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the connector-agnostic EC connector metrics plumbing.
Covers only the base infra: no concrete connector type implements these
hooks yet, so every assertion here is about the documented no-op defaults
and the container contracts, mirroring the equivalent KV connector tests.
"""
from unittest.mock import Mock
import pytest
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
ECExampleConnector,
)
from vllm.distributed.ec_transfer.ec_connector.metrics import (
ECConnectorLogging,
ECConnectorProm,
ECConnectorStats,
)
from vllm.v1.outputs import ECConnectorOutput
pytestmark = pytest.mark.cpu_test
@pytest.fixture
def example_connector(tmp_path):
config = Mock(spec=VllmConfig)
config.ec_transfer_config = Mock()
config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path))
config.ec_transfer_config.is_ec_producer = True
config.ec_transfer_config.is_ec_consumer = False
return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER)
def test_base_stats_methods_raise_not_implemented():
"""The base container has no aggregation logic of its own."""
stats = ECConnectorStats()
with pytest.raises(NotImplementedError):
stats.reset()
with pytest.raises(NotImplementedError):
stats.aggregate(stats)
with pytest.raises(NotImplementedError):
stats.reduce()
with pytest.raises(NotImplementedError):
stats.is_empty()
def test_connector_hooks_default_to_none(example_connector):
"""No connector type overrides these yet, so every hook is a no-op."""
assert example_connector.get_ec_connector_stats() is None
assert ECExampleConnector.build_ec_connector_stats() is None
assert (
ECExampleConnector.build_prom_metrics(
vllm_config=Mock(spec=VllmConfig),
metric_types={},
labelnames=[],
per_engine_labelvalues={},
)
is None
)
def test_ec_connector_output_is_empty_reflects_stats_field():
output = ECConnectorOutput()
assert output.is_empty()
output.ec_connector_stats = ECConnectorStats(data={"x": 1})
assert not output.is_empty()
def test_logging_and_prom_are_no_ops_without_ec_transfer_config():
"""A normal run with no EC connector configured must stay silent."""
logging = ECConnectorLogging(ec_transfer_config=None)
logging.log() # Should not raise, nothing to log.
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = None
prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={})
assert prom.prom_metrics is None
prom.observe({}) # Should not raise.
@@ -1,747 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECExampleConnector."""
import os
from unittest.mock import Mock, patch
import pytest
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
ECExampleConnector,
ECExampleConnectorMetadata,
MMMeta,
)
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.v1.core.sched.output import SchedulerOutput
# ------------------ Mock Classes ------------------ #
class MockRequest:
def __init__(self, request_id, mm_hashes: list[str], token_counts: list[int]):
assert len(mm_hashes) == len(token_counts)
self.request_id = request_id
self._token_counts = token_counts
self.mm_features = []
for i, mm_hash in enumerate(mm_hashes):
feature = MultiModalFeatureSpec(
data=None,
modality="image",
identifier=mm_hash,
mm_position=PlaceholderRange(offset=0, length=self._token_counts[i]),
)
self.mm_features.append(feature)
def get_num_encoder_embeds(self, input_id: int) -> int:
assert input_id < len(self._token_counts)
return self._token_counts[input_id]
def test_metadata_preserves_repeated_item_positions():
request = MockRequest("request", ["A", "B", "A"], [4, 4, 4])
resolver = Mock()
resolver.fields_for.return_value = {"image_grid_thw"}
for feature in request.mm_features:
feature.data = Mock()
feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])}
items = collect_ec_item_metadata(request.mm_features, resolver)
assert items == {
"A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]},
"B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]},
}
def test_audio_metadata_survives_processor_cache_hit():
resolver = Mock()
resolver.fields_for.return_value = {"audio_num_tokens"}
feature = MultiModalFeatureSpec(
data=None,
modality="audio",
identifier="audio",
mm_position=PlaceholderRange(
offset=0, length=4, is_embed=torch.tensor([False, True, True, False])
),
)
assert collect_ec_item_metadata([feature], resolver) == {
"audio": {"metadata": {"audio_num_tokens": [2]}, "item_indices": [0]}
}
@pytest.fixture
def temp_storage(tmp_path):
"""Fixture providing temporary storage path."""
return str(tmp_path)
@pytest.fixture
def mock_vllm_config_producer(temp_storage):
"""Fixture providing mock VllmConfig for producer role."""
config = Mock(spec=VllmConfig)
config.ec_transfer_config = Mock()
config.ec_transfer_config.get_from_extra_config = Mock(return_value=temp_storage)
config.ec_transfer_config.is_ec_producer = True
return config
@pytest.fixture
def mock_vllm_config_consumer(temp_storage):
"""Fixture providing mock VllmConfig for consumer role."""
config = Mock(spec=VllmConfig)
config.ec_transfer_config = Mock()
config.ec_transfer_config.get_from_extra_config = Mock(return_value=temp_storage)
config.ec_transfer_config.is_ec_producer = False
return config
@pytest.fixture
def mock_request_with_3_mm():
"""Fixture providing mock Request with 3 multimodal items."""
request_id = "test_req_123"
mm_hashes = ["img_hash_1", "img_hash_2", "img_hash_3"]
token_counts = [100, 150, 200]
request = MockRequest(request_id, mm_hashes, token_counts)
return request
# ------------------ Unit Tests ------------------ #
class TestECExampleConnectorBasics:
"""Test basic EC connector functionality."""
def test_initialization_producer(self, mock_vllm_config_producer, temp_storage):
"""Test connector initializes correctly as producer."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
assert connector.role == ECConnectorRole.SCHEDULER
assert connector.is_producer
assert connector._storage_path == temp_storage
assert connector._mm_datas_need_loads == {}
def test_initialization_consumer(self, mock_vllm_config_consumer, temp_storage):
"""Test connector initializes correctly as consumer."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
assert connector.role == ECConnectorRole.WORKER
assert not connector.is_producer
assert connector._storage_path == temp_storage
def test_role_assignment(self, mock_vllm_config_producer):
"""Test role is correctly assigned."""
scheduler_connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
worker_connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
assert scheduler_connector.role == ECConnectorRole.SCHEDULER
assert worker_connector.role == ECConnectorRole.WORKER
class TestCacheExistence:
"""Test cache existence checking using has_cache_item() API."""
def test_has_cache_item_all_exist_3_items(
self,
mock_vllm_config_producer,
mock_vllm_config_consumer,
mock_request_with_3_mm,
):
"""Test has_cache_item returns True when all 3 caches exist."""
# Test for producer first
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
# Create cache files using save_caches (proper way)
encoder_cache: dict[str, torch.Tensor] = {}
for mm_feature in mock_request_with_3_mm.mm_features:
mm_hash = mm_feature.identifier
encoder_cache[mm_hash] = torch.randn(10, 768)
producer.save_caches(encoder_cache, mm_hash)
# Test using has_cache_item API
producer_result = [
producer.has_cache_item(mm_feature.identifier)
for mm_feature in mock_request_with_3_mm.mm_features
]
# Assert
assert len(producer_result) == 3
assert all(producer_result), f"Expected all True, got {producer_result}"
# Also test consumer can check if cache exists
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.SCHEDULER,
)
# Test using has_cache_item API
consumer_result = [
consumer.has_cache_item(mm_feature.identifier)
for mm_feature in mock_request_with_3_mm.mm_features
]
# Assert
assert len(consumer_result) == 3
assert all(consumer_result), f"Expected all True, got {consumer_result}"
def test_has_cache_item_none_exist(
self, mock_vllm_config_producer, mock_request_with_3_mm
):
"""Test has_caches returns False when no caches exist."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
# Test without creating any files
result = [
connector.has_cache_item(mm_feature.identifier)
for mm_feature in mock_request_with_3_mm.mm_features
]
# Assert
assert len(result) == 3
assert not any(result), f"Expected all False, got {result}"
def test_has_cache_item_partial_exist(
self, mock_vllm_config_producer, mock_request_with_3_mm
):
"""Test has_caches with some caches existing (1 of 3)."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
# Create only the second cache file
mm_hash_second = mock_request_with_3_mm.mm_features[1].identifier
encoder_cache = {mm_hash_second: torch.randn(10, 768)}
connector.save_caches(encoder_cache, mm_hash_second)
# Test
result = [
connector.has_cache_item(mm_feature.identifier)
for mm_feature in mock_request_with_3_mm.mm_features
]
# Assert
assert len(result) == 3
assert not result[0] # First doesn't exist
assert result[1] # Second exists
assert not result[2] # Third doesn't exist
def test_has_cache_item_miss_creates_no_folder(
self, mock_vllm_config_consumer, temp_storage
):
"""Test a cache miss leaves no empty folder in the shared storage."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.SCHEDULER,
)
assert not connector.has_cache_item("missing_hash")
assert os.listdir(temp_storage) == []
class TestStateManagement:
"""Test connector state management."""
def test_update_state_after_alloc_3_items(
self, mock_vllm_config_producer, mock_request_with_3_mm
):
"""Test state update after allocation for 3 MM items."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
# Initial state should be empty
assert len(connector._mm_datas_need_loads) == 0
# Update state for all 3 items (mock cache existence)
with patch.object(connector, "has_cache_item", return_value=True):
for i in range(3):
connector.update_state_after_alloc(mock_request_with_3_mm, index=i)
# Check state updated for all 3
assert len(connector._mm_datas_need_loads) == 3
assert "img_hash_1" in connector._mm_datas_need_loads
assert "img_hash_2" in connector._mm_datas_need_loads
assert "img_hash_3" in connector._mm_datas_need_loads
assert connector._mm_datas_need_loads["img_hash_1"] == 100
assert connector._mm_datas_need_loads["img_hash_2"] == 150
assert connector._mm_datas_need_loads["img_hash_3"] == 200
def test_build_connector_meta_3_items(
self, mock_vllm_config_producer, mock_request_with_3_mm
):
"""Test metadata building for 3 MM items."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
# Setup state for all 3 items (mock cache existence)
with patch.object(connector, "has_cache_item", return_value=True):
for i in range(3):
connector.update_state_after_alloc(mock_request_with_3_mm, index=i)
# Build metadata
scheduler_output = Mock(spec=SchedulerOutput)
metadata = connector.build_connector_meta(scheduler_output)
# Assert
assert isinstance(metadata, ECExampleConnectorMetadata)
assert len(metadata.mm_datas) == 3
assert metadata.mm_datas[0].mm_hash == "img_hash_1"
assert metadata.mm_datas[0].num_token == 100
assert metadata.mm_datas[1].mm_hash == "img_hash_2"
assert metadata.mm_datas[1].num_token == 150
assert metadata.mm_datas[2].mm_hash == "img_hash_3"
assert metadata.mm_datas[2].num_token == 200
# State should be cleared after building
assert len(connector._mm_datas_need_loads) == 0
def test_build_connector_meta_empty(self, mock_vllm_config_producer):
"""Test metadata building with empty state."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
scheduler_output = Mock(spec=SchedulerOutput)
metadata = connector.build_connector_meta(scheduler_output)
assert isinstance(metadata, ECExampleConnectorMetadata)
assert len(metadata.mm_datas) == 0
def test_state_cleared_after_metadata_build(
self, mock_vllm_config_producer, mock_request_with_3_mm
):
"""Test that state is properly cleared after building metadata."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
# Add state (mock cache existence)
with patch.object(connector, "has_cache_item", return_value=True):
for i in range(3):
connector.update_state_after_alloc(mock_request_with_3_mm, index=i)
assert len(connector._mm_datas_need_loads) == 3
# Build metadata (should clear state)
scheduler_output = Mock(spec=SchedulerOutput)
connector.build_connector_meta(scheduler_output)
# State should be empty
assert len(connector._mm_datas_need_loads) == 0
# Build again should return empty metadata
metadata2 = connector.build_connector_meta(scheduler_output)
assert len(metadata2.mm_datas) == 0
class TestCacheSaving:
"""Test encoder cache saving (producer only)."""
def test_save_caches_producer_3_items(
self, mock_vllm_config_producer, mock_request_with_3_mm, temp_storage
):
"""Test cache saving as producer for 3 different MM items."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
# Create and save 3 different caches
mm_hashes = [f.identifier for f in mock_request_with_3_mm.mm_features]
encoder_cache: dict[str, torch.Tensor] = {}
for mm_hash in mm_hashes:
encoder_cache[mm_hash] = torch.randn(10, 768)
connector.save_caches(encoder_cache, mm_hash)
# Verify all files exist using has_cache_item
result = [
connector.has_cache_item(mm_feature.identifier)
for mm_feature in mock_request_with_3_mm.mm_features
]
assert all(result), f"Not all caches were saved: {result}"
# Verify each file's content
for mm_hash in mm_hashes:
filename = connector._generate_filename_debug(mm_hash)
loaded = safetensors.torch.load_file(filename)
assert "ec_cache" in loaded
assert torch.allclose(loaded["ec_cache"], encoder_cache[mm_hash].cpu())
def test_save_caches_consumer_skips(self, mock_vllm_config_consumer):
"""Test cache saving is skipped for consumer."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
mm_hash = "test_hash_consumer"
encoder_cache = {mm_hash: torch.randn(10, 768)}
# Save should not raise but also not create file
connector.save_caches(encoder_cache, mm_hash)
# Verify file doesn't exist using has_cache_item
result = connector.has_cache_item(mm_hash)
assert not result, "Consumer should not save caches"
class TestCacheLoading:
"""Test encoder cache loading (consumer)."""
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
def test_start_load_caches_consumer_3_items(
self,
mock_vllm_config_producer,
mock_vllm_config_consumer,
mock_request_with_3_mm,
temp_storage,
):
"""Test consumer loads 3 caches from storage."""
# First, create producer to save caches
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
# Producer saves 3 caches
mm_hashes = [f.identifier for f in mock_request_with_3_mm.mm_features]
saved_caches = {}
for mm_hash in mm_hashes:
saved_caches[mm_hash] = torch.randn(10, 768)
producer.save_caches(saved_caches, mm_hash)
# Now consumer loads
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
# Setup metadata for all 3
metadata = ECExampleConnectorMetadata()
for mm_hash in mm_hashes:
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
consumer.bind_connector_metadata(metadata)
# Load
encoder_cache: dict[str, torch.Tensor] = {}
consumer.start_load_caches(encoder_cache=encoder_cache)
# Verify all 3 loaded
assert len(encoder_cache) == 3
for mm_hash in mm_hashes:
assert mm_hash in encoder_cache, f"{mm_hash} missing in encoder_cache"
assert encoder_cache[mm_hash].is_cuda, (
f"{mm_hash} cache is in {encoder_cache[mm_hash].device}"
)
assert torch.allclose(
encoder_cache[mm_hash].cpu(), saved_caches[mm_hash]
), f"{mm_hash} cache saved and loaded tesnor are not the same"
def test_start_load_caches_skip_existing(
self, mock_vllm_config_producer, mock_vllm_config_consumer, temp_storage
):
"""Test cache loading skips already cached items."""
# Setup: producer saves cache
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "existing_hash"
saved_cache = torch.randn(10, 768)
producer.save_caches({mm_hash: saved_cache}, mm_hash)
# Consumer setup
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
consumer.bind_connector_metadata(metadata)
# Pre-populate encoder_cache with different value
existing_cache = torch.randn(5, 512)
encoder_cache = {mm_hash: existing_cache}
# Load (should skip since already exists)
with patch("safetensors.torch.load_file") as mock_load:
consumer.start_load_caches(encoder_cache=encoder_cache)
# Should not call load_file since cache exists
mock_load.assert_not_called()
# Verify original cache unchanged
assert torch.equal(encoder_cache[mm_hash], existing_cache)
def test_start_load_caches_empty_metadata(self, mock_vllm_config_consumer):
"""Test loading with empty metadata does nothing."""
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
# Setup empty metadata
metadata = ECExampleConnectorMetadata()
consumer.bind_connector_metadata(metadata)
# Load (should not raise)
encoder_cache: dict[str, torch.Tensor] = {}
consumer.start_load_caches(encoder_cache=encoder_cache)
# Cache should remain empty
assert len(encoder_cache) == 0
def test_start_load_caches_pins_device_to_current_rank(
self, mock_vllm_config_producer, mock_vllm_config_consumer
):
"""Load must pin to the selected device under tensor parallelism."""
# Produce a real cache file so the load path has something to open.
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "rank_pinning_hash"
producer.save_caches({mm_hash: torch.zeros(4, 8)}, mm_hash)
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
consumer.bind_connector_metadata(metadata)
mock_platform = Mock()
mock_platform.device_type = "cuda"
mock_platform.is_cuda_alike.return_value = True
mock_platform.current_device.return_value = 2
# Simulate running on TP rank 2 without requiring a multi-GPU host.
captured: dict = {}
def fake_load_file(filename, device):
captured["device"] = device
return {"ec_cache": torch.zeros(4, 8)}
with (
patch("vllm.platforms.current_platform", mock_platform),
patch("safetensors.torch.load_file", side_effect=fake_load_file),
):
encoder_cache: dict[str, torch.Tensor] = {}
consumer.start_load_caches(encoder_cache=encoder_cache)
assert captured["device"] == "cuda:2", (
f"Expected device='cuda:2' (pinned to current rank), "
f"got {captured['device']!r}. A bare 'cuda' string makes "
"safetensors load onto cuda:0, which breaks non-rank-0 TP workers."
)
def test_start_load_caches_non_cuda_platform_uses_device_type(
self, mock_vllm_config_producer, mock_vllm_config_consumer
):
"""Non-CUDA platforms pass `device_type` through unchanged (no rank suffix)."""
producer = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "cpu_device_hash"
producer.save_caches({mm_hash: torch.zeros(2, 4)}, mm_hash)
consumer = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
consumer.bind_connector_metadata(metadata)
mock_platform = Mock()
mock_platform.device_type = "cpu"
mock_platform.is_cuda_alike.return_value = False
captured: dict = {}
def fake_load_file(filename, device):
captured["device"] = device
return {"ec_cache": torch.zeros(2, 4)}
with (
patch("vllm.platforms.current_platform", mock_platform),
patch("safetensors.torch.load_file", side_effect=fake_load_file),
):
encoder_cache: dict[str, torch.Tensor] = {}
consumer.start_load_caches(encoder_cache=encoder_cache)
assert captured["device"] == "cpu"
class TestFilenameGeneration:
"""Test filename and path generation."""
def test_generate_foldername(self, mock_vllm_config_producer, temp_storage):
"""Test folder name generation."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "test_folder_hash"
folder = connector._generate_foldername_debug(mm_hash)
assert folder == os.path.join(temp_storage, mm_hash)
assert os.path.isdir(folder) # Should be created
def test_generate_filename(self, mock_vllm_config_producer, temp_storage):
"""Test filename generation."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "test_file_hash"
filename = connector._generate_filename_debug(mm_hash)
expected = os.path.join(temp_storage, mm_hash, "encoder_cache.safetensors")
assert filename == expected
assert os.path.isdir(os.path.dirname(filename)) # Folder created
def test_generate_filename_consistency(self, mock_vllm_config_producer):
"""Test filename generation is consistent."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "consistency_hash"
filename1 = connector._generate_filename_debug(mm_hash)
filename2 = connector._generate_filename_debug(mm_hash)
assert filename1 == filename2
class TestMetadataBindingLifecycle:
"""Test metadata binding and clearing lifecycle."""
def test_bind_connector_metadata(self, mock_vllm_config_consumer):
"""Test binding connector metadata."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta("hash_1", 100))
connector.bind_connector_metadata(metadata)
assert connector._connector_metadata is metadata
def test_clear_connector_metadata(self, mock_vllm_config_consumer):
"""Test clearing connector metadata."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
connector.bind_connector_metadata(metadata)
connector.clear_connector_metadata()
assert connector._connector_metadata is None
def test_get_connector_metadata(self, mock_vllm_config_consumer):
"""Test getting connector metadata."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
connector.bind_connector_metadata(metadata)
retrieved = connector._get_connector_metadata()
assert retrieved is metadata
def test_get_connector_metadata_not_set(self, mock_vllm_config_consumer):
"""Test getting metadata when not set raises."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
with pytest.raises(AssertionError):
connector._get_connector_metadata()
class TestEdgeCases:
"""Test edge cases and error handling."""
def test_save_empty_cache(self, mock_vllm_config_producer):
"""Test saving empty tensor."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.WORKER,
)
mm_hash = "empty_hash"
encoder_cache = {mm_hash: torch.empty(0)}
# Should not raise
connector.save_caches(encoder_cache, mm_hash)
def test_load_nonexistent_cache(self, mock_vllm_config_consumer):
"""Test loading cache that doesn't exist raises error."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.WORKER,
)
metadata = ECExampleConnectorMetadata()
metadata.add_mm_data(MMMeta.make_meta("nonexistent_hash", 100))
connector.bind_connector_metadata(metadata)
encoder_cache: dict[str, torch.Tensor] = {}
# Should raise FileNotFoundError
with pytest.raises(FileNotFoundError):
connector.start_load_caches(encoder_cache=encoder_cache)
def test_has_cache_item_empty_request(self, mock_vllm_config_producer):
"""Test has_cache_item with a nonexistent identifier."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_producer,
role=ECConnectorRole.SCHEDULER,
)
result = connector.has_cache_item("nonexistent_hash")
assert result is False
@@ -1,131 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECOutputAggregator."""
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
from vllm.v1.outputs import (
EMPTY_MODEL_RUNNER_OUTPUT,
ECConnectorOutput,
KVConnectorOutput,
ModelRunnerOutput,
)
pytestmark = pytest.mark.cpu_test
class FakeWorkerMeta(ECConnectorWorkerMetadata):
"""Records merge order. `aggregate` returns a new object, as the base class
declares: an aggregator discarding the return value would lose the merge.
"""
def __init__(self, saves: list[str]):
self.saves = saves
def aggregate(self, other: "FakeWorkerMeta") -> "FakeWorkerMeta":
return FakeWorkerMeta(self.saves + other.saves)
class FakeConnectorStats(ECConnectorStats):
"""Records merge order, mirroring FakeWorkerMeta above."""
def __init__(self, saves: list[str]):
self.saves = saves
def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats":
return FakeConnectorStats(self.saves + other.saves)
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
return ModelRunnerOutput(
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
)
def test_aggregate_folds_every_rank_onto_output_rank():
"""EC work done on any rank reaches the scheduler via output_rank's output.
The middle rank reports no worker metadata or stats: it must neither seed
nor clobber either accumulator.
"""
outputs = [
_worker_output(
ECConnectorOutput(
finished_sending={"mm0"},
ec_connector_stats=FakeConnectorStats(["mm0"]),
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
)
),
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
_worker_output(
ECConnectorOutput(
ec_connector_stats=FakeConnectorStats(["mm2"]),
ec_connector_worker_meta=FakeWorkerMeta(["mm2"]),
)
),
]
result = ECOutputAggregator().aggregate(outputs, output_rank=2)
assert result is outputs[2]
assert result.ec_connector_output.finished_sending == {"mm0"}
assert result.ec_connector_output.finished_recving == {"mm1"}
assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"]
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
def test_aggregate_leaves_no_ec_output_when_no_worker_reported():
"""Empty per-worker reports must not reach the scheduler as an empty object."""
outputs = [_worker_output(ECConnectorOutput()), _worker_output(ECConnectorOutput())]
result = ECOutputAggregator().aggregate(outputs, output_rank=0)
assert result is outputs[0]
assert result.ec_connector_output is None
assert ECOutputAggregator().aggregate([None], output_rank=0) is None
def test_aggregate_does_not_write_through_the_shared_empty_output():
"""A rank with nothing to report yields the shared empty output singleton.
Folding another rank's metadata onto it must not write through to the
module-level object, which every later step would then carry.
"""
outputs = [
_worker_output(
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm0"]))
),
EMPTY_MODEL_RUNNER_OUTPUT,
]
result = ECOutputAggregator().aggregate(outputs, output_rank=1)
assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None
assert result is not EMPTY_MODEL_RUNNER_OUTPUT
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0"]
def test_chaining_with_kv_aggregator_preserves_both_outputs():
"""MultiprocExecutor chains both aggregators and keeps only the last result,
so each must merge onto the same output_rank output rather than replace it.
"""
outputs = [
_worker_output(ECConnectorOutput(finished_sending={"mm0"})),
_worker_output(None),
]
outputs[1].kv_connector_output = KVConnectorOutput(invalid_block_ids={7})
result = None
for aggregator in (
KVOutputAggregator(expected_finished_count=1),
ECOutputAggregator(),
):
result = aggregator.aggregate(outputs, output_rank=1)
assert result is outputs[1]
assert result.kv_connector_output.invalid_block_ids == {7}
assert result.ec_connector_output.finished_sending == {"mm0"}
@@ -1,129 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sanity tests for ec_transfer_params protocol plumbing.
No running engine required.
"""
from unittest.mock import MagicMock
import pytest
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.v1.request import Request, RequestStatus
# tests.v1.core.utils transitively imports the example KV connector, which
# requires the compiled CUDA flash-attention extensions; skip cleanly rather
# than erroring at collection when those extensions aren't built.
_core_utils = pytest.importorskip(
"tests.v1.core.utils",
reason="requires compiled CUDA flash-attention extensions",
exc_type=ImportError,
)
create_scheduler = _core_utils.create_scheduler
EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}}
def test_ec_transfer_params_routed_to_sampling_params_extra_args():
"""ec_transfer_params on the request must land in SamplingParams.extra_args."""
req = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hi"}],
max_tokens=5,
ec_transfer_params=EC_PARAMS,
)
sp = req.to_sampling_params(max_tokens=5, default_sampling_params={})
assert sp.extra_args is not None
assert sp.extra_args.get("ec_transfer_params") == EC_PARAMS
def test_request_output_add_propagates_ec_transfer_params():
"""RequestOutput.add() must carry ec_transfer_params forward to the caller."""
def _out(ec_params):
return RequestOutput(
request_id="r1",
prompt="p",
prompt_token_ids=[1],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="",
token_ids=[],
cumulative_logprob=0.0,
logprobs=None,
finish_reason=None,
)
],
finished=False,
ec_transfer_params=ec_params,
)
accumulated = _out(None)
accumulated.add(_out(EC_PARAMS), aggregate=True)
assert accumulated.ec_transfer_params == EC_PARAMS
def test_request_reads_ec_transfer_params_from_extra_args():
"""v1 Request must pull ec_transfer_params out of SamplingParams.extra_args."""
sp = SamplingParams(extra_args={"ec_transfer_params": EC_PARAMS})
req = Request(
request_id="r1",
prompt_token_ids=[1, 2, 3],
sampling_params=sp,
pooling_params=None,
)
assert req.ec_transfer_params == EC_PARAMS
def test_free_request_calls_ec_connector_and_surfaces_params():
"""_free_request must call ec_connector.request_finished() and return its params."""
sp = SamplingParams(max_tokens=1)
sp.update_from_generation_config({}, 50256)
request = Request(
request_id="test-req",
prompt_token_ids=[1, 2, 3],
sampling_params=sp,
pooling_params=None,
client_index=0,
)
scheduler = create_scheduler(use_ec_connector=True, ec_role="ec_producer")
scheduler.add_request(request)
request.status = RequestStatus.FINISHED_STOPPED
mock_ec = MagicMock()
mock_ec.request_finished.return_value = (False, EC_PARAMS)
scheduler.ec_connector = mock_ec
kv_params, ec_params = scheduler._free_request(request)
mock_ec.request_finished.assert_called_once_with(request)
assert ec_params == EC_PARAMS
assert kv_params is None
def test_free_request_without_ec_connector_returns_none():
"""When no EC connector is configured, ec_transfer_params must be None."""
sp = SamplingParams(max_tokens=1)
sp.update_from_generation_config({}, 50256)
request = Request(
request_id="test-req",
prompt_token_ids=[1, 2, 3],
sampling_params=sp,
pooling_params=None,
client_index=0,
)
scheduler = create_scheduler(use_ec_connector=True, ec_role="ec_producer")
scheduler.add_request(request)
request.status = RequestStatus.FINISHED_STOPPED
kv_params, ec_params = scheduler._free_request(request)
assert ec_params is None
assert kv_params is None
@@ -1,133 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder fan-out fairness in the disaggregated EPD proxy.
Exercises the REAL ``encoder_rr_assignment`` helper loaded from its
``examples/`` path -- no routing logic is re-implemented here, so a future
change to that function is what these tests exercise.
Regression target: the fan-out cursor used to restart at e_urls[0] on every
incoming request (``e_urls[i % len(e_urls)] for i in range(len(mm_items))``),
so single-item requests -- the common case -- always hit the first encoder
instance and left the rest idle. ``encoder_rr_assignment`` threads a cursor
across calls so consecutive requests keep advancing through every URL.
"""
import importlib.util
import sys
from collections import Counter
from pathlib import Path
import pytest
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
def _load_proxy_module():
path = Path(__file__).parents[4] / PROXY_REL
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_under_test", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
@pytest.fixture(scope="module")
def proxy():
return _load_proxy_module()
@pytest.fixture(scope="module")
def assign(proxy):
return proxy.encoder_rr_assignment
def _drive(assign, e_urls, counts):
"""Feed a sequence of request item-counts through the cursor, as
fanout_encoder_primer does one request at a time."""
cursor = 0
all_urls = []
for count in counts:
urls, cursor = assign(e_urls, cursor, count)
assert len(urls) == count
all_urls.append(urls)
return all_urls
@pytest.mark.parametrize("n_urls", [1, 2, 3, 5])
def test_full_url_space_is_covered_uniformly(assign, n_urls):
e_urls = [f"E{i}" for i in range(n_urls)]
# Three full cycles of single-item requests.
urls = _drive(assign, e_urls, counts=[1] * (n_urls * 3))
hits = Counter(u for req in urls for u in req)
assert set(hits) == set(e_urls)
assert max(hits.values()) == min(hits.values())
def test_single_item_requests_rotate_through_all_encoders(assign):
# Exact scenario from the bug report: 3 encoders, one image/request.
e_urls = ["E0", "E1", "E2"]
routed = _drive(assign, e_urls, counts=[1] * 6)
assert [urls[0] for urls in routed] == ["E0", "E1", "E2", "E0", "E1", "E2"]
def test_cursor_stays_contiguous_across_varying_item_counts(assign):
e_urls = ["E0", "E1", "E2"]
routed = _drive(assign, e_urls, counts=[2, 1, 3, 1])
assert routed == [
["E0", "E1"],
["E2"],
["E0", "E1", "E2"],
["E0"],
]
def test_single_encoder_always_resolves_to_it(assign):
urls, next_cursor = assign(["E0"], 0, 4)
assert urls == ["E0"] * 4
assert next_cursor == 0
def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy):
proxy.validate_ec_consumer_routing(["http://prefill"], [])
def test_mooncake_e_pd_routing_is_supported(proxy):
proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"])
def test_mooncake_independent_prefill_routing_fails_fast(proxy):
with pytest.raises(ValueError, match=r"supports E\+PD only"):
proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"])
def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
request = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
]
}
rewritten = proxy.rewrite_for_decode(
request,
{
0: {
"mm_hash": "proxy-uuid",
"ec_mm_hash": "engine-derived-hash",
"transfer_id": "transfer",
"image_grid_thw": [1, 2, 3],
}
},
)
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
assert rewritten["messages"][0]["content"][0]["image_embeds"] == {
"image_grid_thw": [1, 2, 3]
}
assert rewritten["ec_transfer_params"]["ec_items"] == [
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
]
@@ -1,96 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECCPUConnectorMetadata."""
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
def test_metadata_is_ec_connector_metadata():
"""ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata."""
meta = ECCPUConnectorMetadata()
assert isinstance(meta, ECConnectorMetadata)
def test_metadata_initialization():
"""ECCPUConnectorMetadata initializes with empty saves and loads dicts."""
meta = ECCPUConnectorMetadata()
assert hasattr(meta, "saves")
assert hasattr(meta, "loads")
assert meta.saves == {}
assert meta.loads == {}
def test_metadata_saves_dict_operations():
"""The saves dict supports standard dict operations."""
meta = ECCPUConnectorMetadata()
# Add multiple entries
meta.saves["mm_hash_1"] = [0, 1, 2]
meta.saves["mm_hash_2"] = [10, 11]
assert "mm_hash_1" in meta.saves
assert "mm_hash_2" in meta.saves
assert len(meta.saves) == 2
assert meta.saves["mm_hash_1"] == [0, 1, 2]
assert meta.saves["mm_hash_2"] == [10, 11]
def test_metadata_loads_carry_a_transfer_id_and_blocks():
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
report the transfer rather than the mm_hash."""
meta = ECCPUConnectorMetadata()
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
meta.loads["mm_hash_2"] = (1, [100])
assert len(meta.loads) == 2
assert meta.loads["mm_hash_1"] == (0, [5, 6, 7])
transfer_id, block_ids = meta.loads["mm_hash_2"]
assert transfer_id == 1
assert block_ids == [100]
def test_metadata_saves_and_loads_are_independent():
"""Saves and loads dicts are independent."""
meta = ECCPUConnectorMetadata()
meta.saves["key"] = [1, 2]
meta.loads["key"] = (0, [3, 4])
# Same key but different dicts should not interfere
assert meta.saves["key"] == [1, 2]
assert meta.loads["key"] == (0, [3, 4])
# Modifying one should not affect the other
meta.saves["other"] = [99]
assert "other" not in meta.loads
def test_metadata_block_indices_are_integers():
"""Block indices in saves/loads lists should be integers."""
meta = ECCPUConnectorMetadata()
meta.saves["hash"] = [0, 100, 999, 65535]
meta.loads["hash"] = (0, [1, 2, 3])
for idx in meta.saves["hash"]:
assert isinstance(idx, int)
for idx in meta.loads["hash"][1]:
assert isinstance(idx, int)
def test_metadata_multiple_instances_are_independent():
"""Different ECCPUConnectorMetadata instances should not share state."""
meta1 = ECCPUConnectorMetadata()
meta2 = ECCPUConnectorMetadata()
meta1.saves["x"] = [1]
meta2.saves["y"] = [2]
assert "x" in meta1.saves
assert "x" not in meta2.saves
assert "y" not in meta1.saves
assert "y" in meta2.saves
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import msgspec
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
EC_CONNECTOR_VERSION,
XferAck,
XferReq,
XferStatus,
compute_ec_compatibility_hash,
)
def test_xferreq_roundtrip():
req = XferReq(mm_hash="h1", compatibility_hash="c1", session_id="s1")
data = msgspec.msgpack.encode(req)
out = msgspec.msgpack.decode(data, type=XferReq)
assert out.mm_hash == "h1"
assert out.compatibility_hash == "c1"
assert out.connector_version == EC_CONNECTOR_VERSION
def test_xferack_roundtrip_ok():
ack = XferAck(
mm_hash="h1",
status=XferStatus.OK,
src_block_indices=[1, 2],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
out = msgspec.msgpack.decode(msgspec.msgpack.encode(ack), type=XferAck)
assert out.status == XferStatus.OK
assert out.src_block_indices == [1, 2]
def test_compat_hash_deterministic_and_sensitive():
a = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
b = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
c = compute_ec_compatibility_hash("0.1", "m", "float16", 128)
assert a == b
assert a != c
@@ -1,461 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import uuid
import pytest
import torch
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
from vllm.multimodal.inputs import (
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalSharedField,
)
_N, _BS, _HID, _ES = 16, 64, 32, 2
class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, data=None):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
self.data = data
self.modality = "image"
class _Request:
def __init__(self, features, params=None, req_id="r1"):
self.mm_features = features
self.request_id = req_id
self.ec_transfer_params = params
class _FakeResults:
def __init__(self):
self.completed = set()
self.tombstoned = set()
self.quarantined = set()
self.retryable = set()
self.settled = []
class _FakeSession:
"""Records started transfers; lets the test flip them to completed."""
def __init__(self):
self.started = []
self._results = _FakeResults()
def start_xfer(self, mm_hash, block_indices, deadline):
self.started.append(mm_hash)
def poll(self, messages, now):
pass
def take_results(self):
r, self._results = self._results, _FakeResults()
return r
def close(self):
pass
class _FakeTransport:
"""Stand-in for ZmqClientTransport so _poll_step runs without ZMQ."""
def poll(self):
return {}
def poll_dead(self):
return []
def close(self):
pass
_MISSING = object()
def _consumer_sched(monkeypatch):
def _region(cfg):
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
# below flip on the NIXL consumer state directly.
# No measured widths: a consumer takes each shape from the announcement.
s = ECCPUScheduler(
create_ec_vllm_config(ec_role="ec_consumer", encoder_output_widths={})
)
# Turn on NIXL consumer state without constructing real transports.
s._nixl_enabled = True
s._transport = _FakeTransport()
s._data = None
s._compat_hash = "c"
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
return s
def _params(mm_hash, length):
return {
mm_hash: {
"peer_host": "h",
"peer_port": 1,
"size_bytes": length * _HID * _ES,
"shape": [length, _HID],
}
}
def test_new_remote_read_defers_then_completes(monkeypatch):
"""The read is sized from the announced shape, not a local width."""
width = 4 * _HID
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
# still reserve a real not-ready cache entry so mark_ready works.
def _fake_start(mm_hash, info, size, shape):
assert size == width * _ES
assert shape == (1, width)
entry = s._cache.alloc(mm_hash, size // _BS, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
params = _params("h1", 1)
params["h1"]["size_bytes"] = width * _ES
params["h1"]["shape"] = [1, width]
req = _Request([_Feature("h1", 1)], params=params)
# Step 1: unseen remote item -> read started, request deferred.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
assert "h1" in fake.started
s.build_connector_meta(scheduler_output=None) # re-arms _first_in_batch
# Session reports completion this step.
fake._results.completed.add("h1")
s._sessions[("h", 1)] = fake
# Step 2: poll drains completion -> entry marked ready, request admitted.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert "h1" in s._step_completed
entry = s._cache.get("h1")
assert entry is not None and entry.ready
# The scheduler now allocates encoder input; the ready entry loads through
# the same local path as a natively cached encoding.
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.loads
assert meta.loads["h1"][2] == (1, width)
assert "h1" not in s._step_completed # cleared by promote
# Step 3: still cached -> admitted directly, no new transfer.
assert s.ensure_cache_available(req, 0) is True
s.shutdown()
@pytest.mark.parametrize("params", [None, {"x": 123}])
def test_no_remote_announcement_is_ready(monkeypatch, params):
s = _consumer_sched(monkeypatch)
assert s.ensure_cache_available(_Request([_Feature("x")], params=params), 0)
assert s.take_unavailable_requests() == set()
s.shutdown()
@pytest.mark.parametrize("payload", [None, "metadata", "pixels", "embeds", "shm"])
def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
"""A remote miss is fatal only when local model input is unavailable."""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size, shape):
entry = s._cache.alloc(mm_hash, 1, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
fields = {
"metadata": {"image_grid_thw": torch.tensor([1, 2, 2])},
"pixels": {"pixel_values": torch.ones(1, 3)},
"embeds": {"image_embeds": torch.ones(1, _HID)},
"shm": {"address": 123, "monotonic_id": 1},
}
data = (
MultiModalKwargsItem(
{
key: MultiModalFieldElem(value, MultiModalSharedField(batch_size=1))
for key, value in fields[payload].items()
}
)
if payload is not None
else None
)
req = _Request([_Feature("h1", 1, data)], params=_params("h1", 1))
# Step 1: start the read.
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
fake._results.tombstoned.add("h1")
s._sessions[("h", 1)] = fake
can_fallback = payload in ("pixels", "embeds", "shm")
assert s.ensure_cache_available(req, 0) is can_fallback
assert s._cache.get("h1") is None
assert "h1" not in s._in_flight
assert s.take_unavailable_requests() == (set() if can_fallback else {"r1"})
# Drained by the read, so the scheduler cannot abort it twice.
assert s.take_unavailable_requests() == set()
if can_fallback:
# A later scheduling step must not retry the failed remote source.
s.build_connector_meta(scheduler_output=None)
assert s.ensure_cache_available(req, 0)
assert fake.started == ["h1"]
s.shutdown()
@pytest.mark.parametrize("retry", [False, True], ids=["in-flight", "retryable"])
def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
"""Neither retries nor delayed scheduling may restart the request budget."""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
s._sessions[("h", 1)] = fake
def start(mm_hash, info, size, shape):
assert s._cache.alloc(mm_hash, 1, shape) is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", start)
now = [1000.0]
monkeypatch.setattr(time, "monotonic", lambda: now[0])
req = _Request([_Feature("h1")], params=_params("h1", 1))
assert not s.ensure_cache_available(req, 0)
for elapsed in (20, 40, 121):
now[0] = 1000.0 + elapsed
if retry:
fake._results.retryable.add("h1")
s.build_connector_meta(scheduler_output=None)
assert not s.ensure_cache_available(req, 0)
assert s.take_unavailable_requests() == ({"r1"} if elapsed > 60 else set())
assert len(fake.started) == (3 if retry else 1)
s.shutdown()
def test_retryable_read_re_requests_without_admitting(monkeypatch):
"""A not-ready NACK defers the request and re-requests the read.
Unlike a tombstone it never admits the request, because an admitted request
whose media was rewritten away upstream has no embedding and no way to
recompute one.
"""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size, shape):
entry = s._cache.alloc(mm_hash, 1, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
# Producer's save has not landed: retryable, not tombstoned.
fake._results.retryable.add("h1")
s._sessions[("h", 1)] = fake
# The entry is released and the read re-issued in the same step, so the
# request stays deferred rather than being admitted unfetched.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
assert "h1" not in s._tombstones
assert fake.started.count("h1") == 2
s.shutdown()
@pytest.mark.parametrize(
"fields",
[
{"size_bytes": 999},
{"size_bytes": None},
{"size_bytes": "big"},
{"size_bytes": _MISSING},
{"shape": _MISSING},
{"shape": [2, _HID]},
],
ids=[
"size-mismatch",
"null-size",
"text-size",
"no-size",
"no-shape",
"wrong-rows",
],
)
def test_invalid_announcement_fails_the_request(monkeypatch, fields):
"""An unusable announced size or shape fails the request, not the engine."""
s = _consumer_sched(monkeypatch)
announced = {**_params("h1", 1)["h1"], **fields}
announced = {k: v for k, v in announced.items() if v is not _MISSING}
req = _Request([_Feature("h1", 1)], params={"h1": announced})
assert s.ensure_cache_available(req, 0) is False
assert s.take_unavailable_requests() == {"r1"}
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_deferral_budget_is_per_request(monkeypatch):
"""Shared hashes have independent wait budgets and cancellation cleanup."""
s = _consumer_sched(monkeypatch)
monkeypatch.setattr(s._cache, "alloc", lambda key, n, shape=None: None)
params = _params("h1", 1)
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
now = [1000.0]
monkeypatch.setattr(time, "monotonic", lambda: now[0])
assert s.ensure_cache_available(old, 0) is False
now[0] += sched_mod._ADMIT_DEFER_TIMEOUT_S - 1
assert s.ensure_cache_available(new, 0) is False
assert s.take_unavailable_requests() == set()
now[0] += 2
# `old` is now past its budget; `new` is not.
assert s.ensure_cache_available(new, 0) is False
assert s.take_unavailable_requests() == set()
assert s.request_finished(new) == (False, None)
assert set(s._deferred_since) == {("old", "h1")}
assert s.ensure_cache_available(old, 0) is False
assert s.take_unavailable_requests() == {"old"}
assert "h1" not in s._in_flight
s.shutdown()
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
# A producer that reported placeholder metadata but had no cache entry
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
# must fall back to local compute rather than treating that as a size
# mismatch.
s = _consumer_sched(monkeypatch)
metadata_only = {"h1": {"image_grid_thw": [1, 2, 3]}}
req = _Request([_Feature("h1", 1)], params=metadata_only)
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_orphan_not_ready_entry_defers_no_realloc(monkeypatch):
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
# entry exists for the mm_hash, but the hash is tracked in none of
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
# quarantined/settling DMA and must not be re-allocated.
s = _consumer_sched(monkeypatch)
entry = s._cache.alloc("h1", 1)
assert entry is not None and not entry.ready
assert "h1" not in s._in_flight
assert "h1" not in s._step_completed
assert "h1" not in s._tombstones
# Spy on alloc: record calls, delegate to the real allocator (which
# asserts on a duplicate key). A re-alloc for "h1" is the bug.
real_alloc = s._cache.alloc
calls: list[str] = []
def _spy_alloc(key, n):
calls.append(key)
return real_alloc(key, n)
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Must not raise; the request is deferred while the DMA settles, the
# orphan entry is untouched, and alloc is never called again.
assert s.ensure_cache_available(req, 0) is False
assert calls == []
assert s._cache.get("h1") is entry
assert not entry.ready
assert s.take_unavailable_requests() == set()
s.shutdown()
def test_already_computed_feature_admits_without_transfer(monkeypatch):
# A feature entirely within num_computed_tokens is skipped by admit; no
# cache lookup/alloc or transfer should be triggered for it.
s = _consumer_sched(monkeypatch)
calls: list[str] = []
real_alloc = s._cache.alloc
def _spy_alloc(key, n):
calls.append(key)
return real_alloc(key, n)
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
# Feature spans [0, 4); with num_computed_tokens=4 it is fully computed.
req = _Request([_Feature("h1", 4)], params=_params("h1", 4))
assert s.ensure_cache_available(req, 4) is True
assert "h1" not in s._in_flight
assert calls == []
s.shutdown()
def test_local_ready_entry_admits_without_transfer(monkeypatch):
# A READY entry already present in the cache is a local hit: admit
# returns True and no new transfer is started for that mm_hash.
s = _consumer_sched(monkeypatch)
entry = s._cache.alloc("h1", 1)
assert entry is not None
s._cache.mark_ready("h1")
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
s.shutdown()
def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
# A mm_hash already tracked in _in_flight must defer the request rather
# than starting a duplicate transfer.
s = _consumer_sched(monkeypatch)
started: list[str] = []
def _spy_start_xfer(mm_hash, info, size, shape):
started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _spy_start_xfer)
s._in_flight.add("h1")
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
assert s.ensure_cache_available(req, 0) is False
assert started == []
s.shutdown()
@@ -1,146 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
from pathlib import Path
import pytest
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
import vllm.distributed.nixl_utils as nixl_utils
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.config.ec_transfer import ECTransferConfig
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
from vllm.distributed.nixl_utils import NixlWrapper
_N = 16
_BS = 64
def _region() -> ECSharedRegion:
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
def test_ec_enable_nixl_absent_from_extra_config_by_default():
cfg = ECTransferConfig()
assert cfg.get_from_extra_config("ec_enable_nixl", False) is False
def test_ec_enable_nixl_read_from_extra_config():
cfg = ECTransferConfig(
ec_connector="ECCPUConnector",
ec_role="ec_both",
ec_connector_extra_config={"ec_enable_nixl": True},
)
assert cfg.get_from_extra_config("ec_enable_nixl", False) is True
def test_gate_off_builds_no_nixl(monkeypatch):
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
# ec_enable_nixl defaults to False.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
assert s._nixl_enabled is False
assert getattr(s, "_data", None) is None
assert getattr(s, "_producer_session", None) is None
s.shutdown()
def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch):
"""Extra config is not type coerced, and bool("false") is True."""
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = "false"
s = ECCPUScheduler(cfg)
assert s._nixl_enabled is False
s.shutdown()
@pytest.mark.cpu_test
def test_missing_nixl_releases_shared_region(monkeypatch):
"""A failed constructor leaves no scheduler for the caller to shut down."""
# Import before patching to avoid caching the missing-NIXL values.
from vllm.distributed.ec_transfer.ec_connector.cpu.data import nixl # noqa: F401
region = _region()
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
monkeypatch.setattr(nixl_utils, "NixlWrapper", None)
monkeypatch.setattr(nixl_utils, "nixl_agent_config", None)
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
try:
with pytest.raises(RuntimeError, match="ec_enable_nixl requires NIXL"):
ECCPUScheduler(cfg)
assert not Path(region._mmap_path).exists()
assert region._fd is None
assert region._mmap_obj is None
finally:
region.cleanup()
@pytest.mark.cpu_test
def test_control_transport_failure_releases_nixl_and_shared_region(monkeypatch):
import vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq as zmq_mod
import vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl as data_mod
class RegisteredData:
registered = True
def deregister(self):
self.registered = False
data = RegisteredData()
error = RuntimeError("control transport failed")
def fail_control_transport():
raise error
region = _region()
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
monkeypatch.setattr(nixl_utils, "NixlWrapper", object())
monkeypatch.setattr(nixl_utils, "nixl_agent_config", object())
monkeypatch.setattr(data_mod, "NixlDataTransport", lambda **kwargs: data)
monkeypatch.setattr(zmq_mod, "ZmqClientTransport", fail_control_transport)
cfg = create_ec_vllm_config(ec_role="ec_consumer")
cfg.model_config.hf_config = None
cfg.model_config.get_inputs_embeds_size.return_value = 8
cfg.model_config.model = "test-model"
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
try:
with pytest.raises(RuntimeError, match="control transport failed") as exc:
ECCPUScheduler(cfg)
assert exc.value is error
assert not data.registered
assert not Path(region._mmap_path).exists()
assert region._fd is None
assert region._mmap_obj is None
finally:
region.cleanup()
@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package")
def test_gate_on_wires_data_transport_and_producer_session(monkeypatch):
# Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind
# in ProducerSession.start() cannot collide with another test/process.
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_HOST", "127.0.0.1")
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_PORT", "0")
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
cfg = create_ec_vllm_config(ec_role="ec_both")
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
s = ECCPUScheduler(cfg)
try:
assert s._nixl_enabled is True
assert s._data is not None
assert s._compat_hash is not None
assert s._producer_session is not None
assert s._peer_host == "127.0.0.1"
assert s._transport is not None # is_ec_consumer is also True
finally:
s.shutdown()
@@ -1,116 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import uuid
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
_N, _BS = 16, 64
class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
self.data = None
self.modality = "image"
class _Request:
def __init__(self, features, req_id="r1"):
self.mm_features = features
self.request_id = req_id
def _sched_gate_off(monkeypatch):
def _region(cfg):
return ECSharedRegion(
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
)
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
return ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
def test_request_finished_gate_off_returns_none(monkeypatch):
s = _sched_gate_off(monkeypatch)
assert s.request_finished(_Request([_Feature("h1")])) == (False, None)
s.shutdown()
def test_request_finished_producer_emits_params(monkeypatch):
s = _sched_gate_off(monkeypatch)
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
entry = s._cache.alloc("h1", 2, (2, 32))
assert entry is not None
s._cache.mark_ready("h1")
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert entry.evictable
assert params == {
"h1": {
"metadata": {},
"item_indices": [0],
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
"shape": [2, 32],
}
}
s.shutdown()
def test_request_finished_announces_not_ready_entry(monkeypatch):
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
still announced: the entry can't be evicted before it's ready, and a read
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
protecting it is not this method's job."""
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._cache.alloc("h1", 2, (2, 32)) # allocated but not marked ready
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert params == {
"h1": {
"metadata": {},
"item_indices": [0],
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
"shape": [2, 32],
}
}
s.shutdown()
def test_request_finished_skips_unallocated_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
# No alloc() for "h1" at all — e.g. the cache was full at save time.
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
# The item's placeholder metadata is still reported even though there's
# no cache entry to transfer (empty "metadata": no fields, no transfer).
assert params == {"h1": {"metadata": {}, "item_indices": [0]}}
s.shutdown()
-471
View File
@@ -1,471 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for ProducerXfer, ProducerSession, ConsumerXfer, ConsumerSession."""
import time
from unittest.mock import MagicMock
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
XferAck,
XferReq,
XferStatus,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
_CONSUMER_QUARANTINE_TIMEOUT_S,
ConsumerSession,
ConsumerXfer,
ProducerSession,
ProducerXfer,
XferState,
)
# ── helpers ───────────────────────────────────────────────────────────────────
def _make_data(xfer_state: str = "PROC") -> MagicMock:
data = MagicMock()
data.check_xfer_state.return_value = xfer_state
data.post_read.return_value = 99
return data
def _ok_ack(
mm_hash: str = "h1",
session_id: str = "sess-1",
src_indices: list | None = None,
) -> XferAck:
return XferAck(
mm_hash=mm_hash,
status=XferStatus.OK,
session_id=session_id,
src_block_indices=src_indices or [0, 1],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
def _xfer(
data: MagicMock | None = None,
deadline: float | None = None,
consumer_session_id: str = "sess-1",
) -> ConsumerXfer:
return ConsumerXfer(
mm_hash="h1",
block_indices=[10, 11],
addr=("host", 1234),
deadline=deadline if deadline is not None else time.monotonic() + 60,
data=data or _make_data(),
consumer_session_id=consumer_session_id,
)
def _started(
data: MagicMock | None = None, deadline: float | None = None
) -> ConsumerXfer:
"""ConsumerXfer that has already received an OK XferAck."""
x = _xfer(data, deadline)
x.handle_ack(_ok_ack(session_id="sess-1", src_indices=[5, 6]), agent_name="agent-1")
return x
# ── ProducerXfer ──────────────────────────────────────────────────────────────
def test_producer_xfer_not_expired_before_deadline():
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() + 60)
assert not x.is_expired()
def test_producer_xfer_expired_after_deadline():
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() - 1)
assert x.is_expired()
# ── ProducerSession ───────────────────────────────────────────────────────────
def _xfer_req(mm_hash: str = "h1", session_id: str = "sess-1") -> XferReq:
return XferReq(
mm_hash=mm_hash, compatibility_hash="hash-abc", session_id=session_id
)
def _make_producer_session(cache: EmbeddingCache | None = None) -> ProducerSession:
data = _make_data()
data.get_agent_metadata.return_value = b"meta"
data.get_mem_descriptor.return_value = b"desc"
return ProducerSession(
transport=MagicMock(),
data=data,
cache=cache or EmbeddingCache(num_blocks=8),
compat_hash="hash-abc",
)
def test_producer_session_grant_pins_ready_entry_and_returns_blocks():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
assert ack.status == XferStatus.OK
entry = cache.get("h1")
assert not entry.evictable # pinned by the grant
assert list(ack.src_block_indices) == list(entry.block_ids)
def test_producer_session_grant_nacks_missing_entry():
s = _make_producer_session()
ack = s._grant_or_nack(_xfer_req(mm_hash="nope"))
assert ack.status == XferStatus.NACK_MISSING
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
"""An announced entry whose save is still in flight is distinguished from a
miss, so the consumer retries instead of falling back to local compute."""
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2) # not ready
s = _make_producer_session(cache)
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
assert ack.status == XferStatus.NACK_NOT_READY
def test_producer_session_grant_nacks_version_mismatch():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
req = XferReq(
mm_hash="h1",
compatibility_hash="hash-abc",
session_id="sess-1",
connector_version=-1,
)
ack = s._grant_or_nack(req)
assert ack.status == XferStatus.NACK_VERSION
entry = cache.get("h1")
assert entry.evictable # nothing was pinned
def test_producer_session_grant_nacks_compat_hash_mismatch():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
req = XferReq(mm_hash="h1", compatibility_hash="wrong-hash", session_id="sess-1")
ack = s._grant_or_nack(req)
assert ack.status == XferStatus.NACK_INCOMPAT
entry = cache.get("h1")
assert entry.evictable # nothing was pinned
def test_producer_session_notif_unpins_via_cache():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
entry = cache.get("h1")
assert not entry.evictable # pinned
s._data.get_new_notifs.return_value = {"agent": [b"sess-1:h1"]}
s._drain_notifs()
assert entry.evictable # unpinned via cache
assert s._active_xfers == {}
def test_producer_session_sweep_timeouts_unpins_via_cache():
cache = EmbeddingCache(num_blocks=8)
cache.alloc("h1", 2)
cache.mark_ready("h1")
s = _make_producer_session(cache)
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
key = next(iter(s._active_xfers))
s._active_xfers[key].deadline = time.monotonic() - 1
s._sweep_timeouts()
assert cache.get("h1").evictable
assert s._active_xfers == {}
# ── ConsumerXfer.handle_ack ───────────────────────────────────────────────────
def test_consumer_xfer_handle_ack_ok_starts_read():
data = _make_data()
x = _xfer(data)
assert x.handle_ack(_ok_ack(), agent_name="agent-1") is True
assert x.transfer_handle == 99
def test_consumer_xfer_handle_ack_notif_msg_is_session_id_colon_mm_hash():
data = _make_data()
x = _xfer(data, consumer_session_id="my-sess")
x.handle_ack(_ok_ack(mm_hash="h1"), agent_name="agent-1")
_, kwargs = data.post_read.call_args
assert kwargs["notif_msg"] == b"my-sess:h1"
def test_consumer_xfer_handle_ack_forwards_correct_args():
data = _make_data()
x = _xfer(data)
x.handle_ack(_ok_ack(src_indices=[5, 6]), agent_name="agent-1")
args, _ = data.post_read.call_args
assert args[0] == [10, 11] # local block_indices
assert args[1] == "agent-1" # agent_name (not a raw handle)
assert args[2] == [5, 6] # remote src_block_indices
def test_consumer_xfer_handle_ack_resets_deadline():
original = time.monotonic() + 2
x = _xfer(deadline=original)
x.handle_ack(_ok_ack(), agent_name="agent-1")
assert x.deadline > original
def test_consumer_xfer_handle_ack_nack_returns_false():
data = _make_data()
x = _xfer(data)
assert (
x.handle_ack(XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING), "a")
is False
)
assert x.transfer_handle is None
data.post_read.assert_not_called()
# ── ConsumerXfer.poll — WAITING_ACK ──────────────────────────────────────────
def test_consumer_xfer_poll_waiting_ack_before_deadline():
assert _xfer().poll(time.monotonic()) == XferState.WAITING_ACK
def test_consumer_xfer_poll_waiting_ack_timeout():
assert (
_xfer(deadline=time.monotonic() - 1).poll(time.monotonic())
== XferState.ACK_TIMEOUT
)
# ── ConsumerXfer.poll — READING ───────────────────────────────────────────────
def test_consumer_xfer_poll_reading_proc():
assert _started().poll(time.monotonic()) == XferState.READING
def test_consumer_xfer_poll_reading_done_releases_handle():
data = _make_data(xfer_state="DONE")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.DONE
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
def test_consumer_xfer_poll_reading_unexpected_state_fails():
data = _make_data(xfer_state="ERR")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.READ_FAILED
data.release_xfer_handle.assert_called_once_with(99)
def test_consumer_xfer_poll_reading_exception_fails_gracefully():
data = _make_data()
data.check_xfer_state.side_effect = RuntimeError("nixl crash")
x = _started(data)
assert x.poll(time.monotonic()) == XferState.READ_FAILED
def test_consumer_xfer_poll_timeout_quarantines_without_releasing():
data = _make_data(xfer_state="PROC")
x = _started(data)
assert x.poll(time.monotonic() + 9999) == XferState.QUARANTINED
data.release_xfer_handle.assert_not_called()
assert x.transfer_handle == 99
# ── ConsumerXfer.poll — QUARANTINED ───────────────────────────────────────────
def test_consumer_xfer_quarantined_proc_stays():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999) # → QUARANTINED
assert x.poll(time.monotonic()) == XferState.QUARANTINED
data.release_xfer_handle.assert_not_called()
def test_consumer_xfer_quarantined_terminal_settles():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999) # → QUARANTINED
data.check_xfer_state.return_value = "DONE"
assert x.poll(time.monotonic()) == XferState.SETTLED
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
def test_consumer_xfer_quarantined_exception_settles():
data = _make_data(xfer_state="PROC")
x = _started(data)
x.poll(time.monotonic() + 9999)
data.check_xfer_state.side_effect = RuntimeError("gone")
assert x.poll(time.monotonic()) == XferState.SETTLED
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
"""Quarantine timeout should release handle and return READ_FAILED."""
data = _make_data(xfer_state="PROC")
x = _started(data)
# First poll transitions to QUARANTINED (deadline extended by 60s)
x.poll(time.monotonic() + 9999)
# Second poll with expired quarantine deadline should evict
expired_time = time.monotonic() + 9999 + _CONSUMER_QUARANTINE_TIMEOUT_S + 1
assert x.poll(expired_time) == XferState.READ_FAILED
data.release_xfer_handle.assert_called_once_with(99)
assert x.transfer_handle is None
# ── ConsumerXfer.cancel / release ────────────────────────────────────────────
def test_consumer_xfer_cancel_valid_in_waiting_ack():
_xfer().cancel() # must not raise
def test_consumer_xfer_cancel_raises_when_read_started():
with pytest.raises(AssertionError):
_started().cancel()
def test_consumer_xfer_release_releases_handle():
data = _make_data()
x = _started(data)
x.release()
data.release_xfer_handle.assert_called_once_with(99)
def test_consumer_xfer_release_noop_when_no_handle():
data = _make_data()
_xfer(data).release()
data.release_xfer_handle.assert_not_called()
# ── ConsumerSession ───────────────────────────────────────────────────────────
def _make_consumer_session(data=None):
zmq_conn = MagicMock()
zmq_conn.recv.return_value = []
return ConsumerSession(
addr=("host", 1234),
zmq_conn=zmq_conn,
transport=MagicMock(),
data=data or _make_data(),
compat_hash="hash-abc",
)
def test_consumer_session_has_stable_session_id():
s = _make_consumer_session()
assert isinstance(s._session_id, str) and len(s._session_id) > 0
assert s._session_id == s._session_id # stable
def test_consumer_session_start_xfer_sends_req_with_session_id():
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
raw = s._zmq.send.call_args[0][0]
req = msgspec.msgpack.decode(raw, type=XferReq)
assert req.mm_hash == "h1"
assert req.session_id == s._session_id
def test_consumer_session_poll_done_goes_to_completed():
import msgspec
data = _make_data(xfer_state="PROC")
s = _make_consumer_session(data)
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
ack = XferAck(
mm_hash="h1",
status=XferStatus.OK,
session_id=s._session_id,
src_block_indices=[5, 6],
agent_metadata=b"meta",
mem_descriptor=b"desc",
)
data.add_remote_peer.return_value = "agent-1"
raw_ack = msgspec.msgpack.encode(ack)
data.check_xfer_state.return_value = "DONE"
s.poll([raw_ack], time.monotonic())
results = s.take_results()
assert "h1" in results.completed
assert not results.tombstoned
def test_consumer_session_poll_nack_goes_to_tombstoned():
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING)
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
results = s.take_results()
assert "h1" in results.tombstoned
assert not results.completed
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
"""A not-ready NACK must not tombstone: tombstoning admits the request with
no embedding, which is fatal once the media has been rewritten away."""
import msgspec
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_NOT_READY)
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
results = s.take_results()
assert "h1" in results.retryable
assert not results.tombstoned
assert not results.completed
def test_consumer_session_ack_timeout_reports_retry_once():
"""A delayed producer ACK is retryable, and its result is drained once."""
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
s.poll([], time.monotonic())
r1 = s.take_results()
assert "h1" in r1.retryable
assert not r1.tombstoned
r2 = s.take_results()
assert not r2.retryable # cleared after first take
def test_consumer_session_on_peer_down_cancels_waiting_ack():
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 60)
s.on_peer_down()
results = s.take_results()
assert "h1" in results.retryable
assert not results.tombstoned
-54
View File
@@ -1,54 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for ECCPUConnector scheduler utilities."""
import msgspec
import pytest
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
build_block_descs,
deserialize_mem_descriptor,
serialize_mem_descriptor,
)
# ── build_block_descs ────────────────────────────────────────────────────────
@pytest.mark.parametrize("device_id,expected_dev", [(7, 7), (None, 0)])
def test_build_block_descs(device_id, expected_dev):
kwargs = dict(base_ptr=1000, num_blocks=4, block_size_bytes=256)
if device_id is not None:
kwargs["device_id"] = device_id
descs = build_block_descs(**kwargs)
assert len(descs) == 4
for i, (addr, size, dev) in enumerate(descs):
assert addr == 1000 + i * 256
assert size == 256
assert dev == expected_dev
def test_build_block_descs_zero_blocks_returns_empty():
assert build_block_descs(base_ptr=100, num_blocks=0, block_size_bytes=64) == []
# ── serialize / deserialize_mem_descriptor ───────────────────────────────────
def test_mem_descriptor_roundtrip():
descs = [(100, 64, 0), (164, 64, 0), (228, 64, 1)]
assert deserialize_mem_descriptor(serialize_mem_descriptor(descs)) == descs
@pytest.mark.parametrize(
"bad_value",
[
[(1, 2)], # 2-tuple instead of 3
[("a", "b", "c")], # strings, not ints
],
)
def test_mem_descriptor_rejects_malformed_payload(bad_value):
"""Malformed descriptor lists must fail to decode."""
encoder = msgspec.msgpack.Encoder()
bad_payload = encoder.encode(bad_value)
with pytest.raises((msgspec.DecodeError, msgspec.ValidationError)):
deserialize_mem_descriptor(bad_payload)
@@ -1,115 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the V2 GPU model runner's EC connector wrapper."""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorMetadata,
)
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
from vllm.v1.worker.ec_connector_model_runner_mixin import (
ECConnectorModelRunnerMixin,
)
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
pytestmark = pytest.mark.cpu_test
WORKER_META = object()
CONNECTOR_STATS = object()
def _scheduler_output() -> SimpleNamespace:
return SimpleNamespace(
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
)
def _connector(
encoder_cache: dict | None = None,
is_producer: bool = True,
is_consumer: bool = False,
) -> tuple[ActiveECConnector, MagicMock]:
fake = MagicMock(spec=ECConnectorBase)
fake.is_producer = is_producer
fake.is_consumer = is_consumer
fake.get_finished.return_value = (None, None)
fake.build_connector_worker_meta.return_value = WORKER_META
fake.get_ec_connector_stats.return_value = CONNECTOR_STATS
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
@pytest.mark.parametrize(
("is_producer", "is_consumer"), [(True, False), (True, True), (False, True)]
)
def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
"""An ec_both node is also a producer: it must offload what it just computed."""
encoder_cache = {"mm_old": None}
connector, fake = _connector(encoder_cache, is_producer, is_consumer)
with connector.maybe_get_output(_scheduler_output()):
encoder_cache["mm_new"] = None
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
assert saved == (["mm_new"] if is_producer else [])
assert fake.start_save_caches.called == is_producer
if is_producer:
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
assert fake.start_load_caches.called == is_consumer
def test_worker_meta_is_reported_on_context_exit():
"""Reported in the finally block, so is_empty() sees it only after the exit."""
connector, fake = _connector()
with connector.maybe_get_output(_scheduler_output()) as output:
assert output.ec_connector_worker_meta is None
assert output.ec_connector_worker_meta is WORKER_META
assert fake.clear_connector_metadata.called
def test_connector_stats_are_reported_on_context_exit():
"""Reported in the finally block, alongside the worker meta."""
connector, fake = _connector()
with connector.maybe_get_output(_scheduler_output()) as output:
assert output.ec_connector_stats is None
assert output.ec_connector_stats is CONNECTOR_STATS
def test_v1_producer_receives_existing_encoder_cache():
encoder_cache = {"mm_cached": None}
fake = MagicMock(spec=ECConnectorBase)
fake.is_producer = True
fake.is_consumer = False
fake.get_finished.return_value = (None, None)
module = "vllm.v1.worker.ec_connector_model_runner_mixin"
with (
patch(f"{module}.has_ec_transfer", return_value=True),
patch(f"{module}.get_ec_transfer", return_value=fake),
ECConnectorModelRunnerMixin.maybe_get_ec_connector_output(
_scheduler_output(), encoder_cache
),
):
pass
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
def test_no_forward_reports_without_running_the_model():
connector, _ = _connector()
output = connector.no_forward(_scheduler_output())
assert output.ec_connector_output.ec_connector_worker_meta is WORKER_META
empty = NO_OP_EC_CONNECTOR.no_forward(_scheduler_output())
assert empty is EMPTY_MODEL_RUNNER_OUTPUT
-81
View File
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared config builders for EC connector unit tests."""
import uuid
from unittest.mock import Mock
import torch
from vllm.config import ModelConfig, VllmConfig
from vllm.config.ec_transfer import ECRole, ECTransferConfig
from vllm.config.parallel import ParallelConfig
# The scheduler derives its per-load ack count from the PCP size only under the
# `mp` backend -- which also keeps the world size off the local GPU count, so
# TP > 1 stays expressible on a single-GPU host.
_EXECUTOR_BACKEND = "mp"
_PCP_SIZE = 1
_EMBED_WIDTH = 32
def create_ec_vllm_config(
*,
ec_role: ECRole = "ec_both",
tensor_parallel_size: int = 1,
rank: int = 0,
dtype: torch.dtype = torch.float16,
encoder_output_widths: dict[str, int] | None = None,
) -> Mock:
"""Build a `VllmConfig` stand-in for EC connector unit tests.
`ec_transfer_config` and `parallel_config` are the real config objects, so
role derivation and the rank arithmetic the connector depends on behave as
they do in production.
`model_config` supplies a dtype and embedding width without resolving an
HF model or inspecting its architecture.
Args:
ec_role: EC role this instance plays. `is_ec_producer` and
`is_ec_consumer` are derived from it, as in production.
tensor_parallel_size: TP degree.
rank: Global rank of this worker.
dtype: Encoder cache dtype.
encoder_output_widths: Per-modality encoder output width, as a
producer measures it at startup. Defaults to the embedding width.
Returns:
A `VllmConfig`-specced mock carrying the configs above.
"""
parallel_config = ParallelConfig(
tensor_parallel_size=tensor_parallel_size,
prefill_context_parallel_size=_PCP_SIZE,
distributed_executor_backend=_EXECUTOR_BACKEND,
rank=rank,
)
ec_transfer_config = ECTransferConfig(
# The role properties report False unless a connector is named.
ec_connector="ECCPUConnector",
ec_role=ec_role,
engine_id=str(uuid.uuid4()),
)
ec_transfer_config.mm_encoder_output_widths = (
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
if encoder_output_widths is None
else encoder_output_widths
)
model_config = Mock(spec=ModelConfig)
model_config.dtype = dtype
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = ec_transfer_config
vllm_config.parallel_config = parallel_config
vllm_config.model_config = model_config
vllm_config.instance_id = f"ec-test-{uuid.uuid4()}"
return vllm_config