[CI] Add e2e test for scale-out EC connector flow (#54973)

Signed-off-by: zhouyou9505 <zhouyou9505@gmail.com>
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
Co-authored-by: zhouyou9505 <zhouyou9505@gmail.com>
Co-authored-by: Cursor Grok 4.6 <noreply@cursor.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Nicolò Lucchesi
2026-09-09 07:38:32 -07:00
committed by GitHub
co-authored by zhouyou9505 Cursor Grok 4.6 OpenAI Codex
parent cc4210f671
commit 9ffb8cea96
5 changed files with 617 additions and 2 deletions
@@ -0,0 +1,16 @@
group: Disaggregated EC
depends_on:
- image-build
steps:
- label: ":nvidia: (L4) Scale-out EC Connector E2E"
key: scale-out-ec-e2e-2-gpus
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
source_file_dependencies:
- vllm/distributed/ec_transfer/
- vllm/entrypoints/scale_out/
- tests/entrypoints/scale_out/ec_integration/
commands:
- bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
@@ -0,0 +1,43 @@
# Scale-out EC connector e2e tests
End-to-end tests for the disaggregated multimodal (encoder-cache) flow over
the scale-out endpoints, in the bash-script style of
`tests/v1/kv_connector/nixl_integration`.
## Topology
```text
render (GPU-less) encode (EC producer)
/v1/chat/completions/render /inference/v1/generate (kwargs_data)
/v1/chat/completions/derender |
prefill (EC consumer)
/inference/v1/generate (mm_metadata + ec_transfer_params)
```
- The **render** server (`vllm launch render`) preprocesses the chat request
into token ids and multimodal features (`kwargs_data` + `mm_metadata`).
- The **encode** instance (`ec_role: ec_producer`, encode-only) runs the
vision encoder on `kwargs_data` and publishes embeddings through the
`ECExampleConnector` shared storage. Its response carries
`ec_transfer_params`.
- The **prefill** instance (`ec_role: ec_consumer`) receives metadata-only
features (`mm_metadata`, no `kwargs_data`) plus `ec_transfer_params`, loads
the embeddings through the EC connector, and generates output token ids.
- The **derender** endpoint turns the output token ids back into a chat
completion response.
## Running
```bash
bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
```
The script first collects baseline outputs from a single `vllm serve`
instance, then runs the scale-out topology and compares the derendered
outputs for exact equality. It also checks that metadata-only features are
rejected when `ec_transfer_params` is missing.
Defaults: `Qwen/Qwen3-VL-2B-Instruct`, encode on GPU 0, prefill on GPU 1.
Override via environment variables (`MODEL`, `GPU_E`, `GPU_PD`,
`EC_SHARED_STORAGE_PATH`, `PYTHON=.venv/bin/python`, ...); see the header of
the script for the full list.
@@ -0,0 +1,275 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Scale-out EC connector E2E test
#
# Tests the disaggregated multimodal flow over the scale-out endpoints,
# following the bash-script style of tests/v1/kv_connector/nixl_integration:
#
# 1. Baseline: a single vLLM instance serving /v1/chat/completions
# 2. Scale-out:
# render (GPU-less, `vllm launch render`)
# encode (EC producer, encode-only)
# prefill (EC consumer)
#
# The Python client renders each multimodal request once, sends the full
# kwargs_data to the encode instance, then sends metadata-only features
# (mm_metadata, no kwargs_data) plus ec_transfer_params to the prefill
# instance, and derenders the output tokens. Outputs are compared against
# the baseline for exact equality.
#
# Usage:
# bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
#
# Requires 2 GPUs by default (encode on GPU 0, prefill on GPU 1); the
# baseline reuses GPU 0 after the encode instance is stopped.
set -euo pipefail
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
# Model and engine configuration
MODEL="${MODEL:-Qwen/Qwen3-VL-2B-Instruct}"
MAX_MODEL_LEN="${MAX_MODEL_LEN:-8192}"
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}"
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
# GPU configuration. encode + prefill run concurrently on separate GPUs;
# the baseline reuses GPU_E after the scale-out instances are stopped.
GPU_E="${GPU_E:-0}"
GPU_PD="${GPU_PD:-1}"
GPU_SINGLE="${GPU_SINGLE:-$GPU_E}"
# Ports
RENDER_PORT="${RENDER_PORT:-19600}"
ENCODE_PORT="${ENCODE_PORT:-19601}"
PREFILL_PORT="${PREFILL_PORT:-19602}"
BASELINE_PORT="${BASELINE_PORT:-19603}"
# Storage path for the ECExampleConnector encoder cache
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_scale_out_test}"
# Python interpreter (override to .venv/bin/python for local runs)
PYTHON="${PYTHON:-python3}"
# Output files and logs
LOG_PATH="${LOG_PATH:-/tmp}"
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_scale_out_ec_baseline.txt}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}"
BASELINE_LOG="${LOG_PATH}/scale_out_ec_baseline.log"
RENDER_LOG="${LOG_PATH}/scale_out_ec_render.log"
ENCODE_LOG="${LOG_PATH}/scale_out_ec_encode.log"
PREFILL_LOG="${LOG_PATH}/scale_out_ec_prefill.log"
mkdir -p "$LOG_PATH"
# Trap the SIGINT signal (triggered by Ctrl+C)
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
wait_for_server() {
local port=$1
local pid=$2
local name=$3
local deadline=$((SECONDS + TIMEOUT_SECONDS))
while ((SECONDS < deadline)); do
if ! kill -0 "$pid" 2>/dev/null; then
echo "$name instance exited before becoming healthy"
return 1
fi
if curl --max-time 2 -fsS "http://localhost:${port}/health" \
> /dev/null 2>&1; then
return 0
fi
sleep 1
done
echo "Timed out waiting for $name instance on port $port"
return 1
}
dump_log() {
local name=$1
local log=$2
echo "--- $name log (last 200 lines)"
if [[ -f "$log" ]]; then
tail -n 200 "$log" || true
else
echo "Log file not found: $log"
fi
}
dump_scale_out_logs() {
dump_log "render" "$RENDER_LOG"
dump_log "encode" "$ENCODE_LOG"
dump_log "prefill" "$PREFILL_LOG"
}
cleanup_instances() {
echo "Cleaning up any running vLLM instances..."
pkill -TERM -f "vllm serve" || true
pkill -TERM -f "vllm launch render" || true
sleep 3
pkill -9 -f "vllm serve" || true
pkill -9 -f "vllm launch render" || true
sleep 2
}
# Step 1: baseline single instance
run_baseline() {
echo "================================"
echo "Running BASELINE (single instance)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
echo "Starting baseline instance on GPU $GPU_SINGLE, port $BASELINE_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \
--port "$BASELINE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.9 \
--max-num-seqs "$MAX_NUM_SEQS" \
> "$BASELINE_LOG" 2>&1 &
local BASELINE_PID=$!
echo "Waiting for baseline instance to start..."
if ! wait_for_server "$BASELINE_PORT" "$BASELINE_PID" "baseline"; then
dump_log "baseline" "$BASELINE_LOG"
return 1
fi
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
--mode baseline \
--service_url "http://localhost:$BASELINE_PORT" \
--model_name "$MODEL" \
--baseline_file "$BASELINE_FILE"; then
dump_log "baseline" "$BASELINE_LOG"
return 1
fi
echo "Stopping baseline instance..."
kill "$BASELINE_PID" 2>/dev/null || true
sleep 2
cleanup_instances
}
# Step 2: render + encode (EC producer) + prefill (EC consumer)
run_scale_out_ec() {
echo "================================"
echo "Running SCALE-OUT EC (render + encode + prefill)"
echo "================================"
cleanup_instances
rm -rf "$EC_SHARED_STORAGE_PATH"
mkdir -p "$EC_SHARED_STORAGE_PATH"
declare -a PIDS=()
# GPU-less render/derender server
echo "Starting render server on port $RENDER_PORT"
vllm launch render "$MODEL" \
--port "$RENDER_PORT" \
> "$RENDER_LOG" 2>&1 &
PIDS+=("$!")
# Encode-only EC producer instance. It runs the vision encoder and
# publishes embeddings through the ECExampleConnector shared storage.
echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_E" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
vllm serve "$MODEL" \
--port "$ENCODE_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--gpu-memory-utilization 0.01 \
--no-enable-prefix-caching \
--max-num-batched-tokens 114688 \
--max-num-seqs "$MAX_NUM_SEQS" \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$ENCODE_LOG" 2>&1 &
PIDS+=("$!")
# Prefill/decode EC consumer instance. It receives metadata-only
# features plus ec_transfer_params and loads the embeddings that the
# encode instance published.
echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT"
env CUDA_VISIBLE_DEVICES="$GPU_PD" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
vllm serve "$MODEL" \
--port "$PREFILL_PORT" \
--max-model-len "$MAX_MODEL_LEN" \
--enforce-eager \
--enable-mm-embeds \
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
--max-num-seqs "$MAX_NUM_SEQS" \
--ec-transfer-config '{
"ec_connector": "ECExampleConnector",
"ec_role": "ec_consumer",
"ec_connector_extra_config": {
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
}
}' \
> "$PREFILL_LOG" 2>&1 &
PIDS+=("$!")
echo "Waiting for render instance..."
if ! wait_for_server "$RENDER_PORT" "${PIDS[0]}" "render"; then
dump_scale_out_logs
return 1
fi
echo "Waiting for encode instance..."
if ! wait_for_server "$ENCODE_PORT" "${PIDS[1]}" "encode"; then
dump_scale_out_logs
return 1
fi
echo "Waiting for prefill instance..."
if ! wait_for_server "$PREFILL_PORT" "${PIDS[2]}" "prefill"; then
dump_scale_out_logs
return 1
fi
echo "All scale-out EC services are up!"
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
--mode disagg \
--render_url "http://localhost:$RENDER_PORT" \
--encode_url "http://localhost:$ENCODE_PORT" \
--prefill_url "http://localhost:$PREFILL_PORT" \
--model_name "$MODEL" \
--baseline_file "$BASELINE_FILE"; then
dump_scale_out_logs
return 1
fi
echo "Stopping scale-out EC instances..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
sleep 2
cleanup_instances
}
# Main execution
echo "================================"
echo "Scale-out EC E2E Test Suite"
echo "Model: $MODEL"
echo "================================"
run_baseline
run_scale_out_ec
rm -f "$BASELINE_FILE"
echo "================================"
echo "All scale-out EC e2e tests finished!"
echo "================================"
@@ -0,0 +1,276 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test for the scale-out EC connector flow (render -> encode -> prefill).
Mimics the bash-script style of ``tests/v1/kv_connector/nixl_integration``:
the instances are started by ``run_scale_out_ec_e2e_test.sh`` and this file is
the HTTP client that drives the disaggregated request path and compares its
output against a single-instance baseline.
Topology under test (started by the bash script):
render (GPU-less) -- /v1/chat/completions/render, /derender
encode (EC producer) -- /inference/v1/generate
prefill (EC consumer) -- /inference/v1/generate
The client renders each multimodal chat request once, sends the full
``kwargs_data`` to the encode instance, then sends metadata-only features
(``mm_metadata`` without ``kwargs_data``) plus the encode response's
``ec_transfer_params`` to the prefill instance, and finally derenders the
output token ids. The derendered text must match the baseline exactly.
Usage (from the bash script, or manually):
# Baseline mode (saves outputs):
python test_scale_out_ec_e2e.py \
--mode baseline \
--service_url http://localhost:19603 \
--model_name Qwen/Qwen3-VL-2B-Instruct \
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
# Disagg mode (compares outputs):
python test_scale_out_ec_e2e.py \
--mode disagg \
--render_url http://localhost:19600 \
--encode_url http://localhost:19601 \
--prefill_url http://localhost:19602 \
--model_name Qwen/Qwen3-VL-2B-Instruct \
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
"""
import argparse
import json
import sys
import time
import requests
from vllm.assets.image import ImageAsset
from vllm.multimodal.utils import encode_image_url
MAX_OUTPUT_LEN = 128
REQUEST_TIMEOUT = 300
image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720))
image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720))
image_1_url = encode_image_url(image_1)
image_2_url = encode_image_url(image_2)
# Each prompt is a chat completion request body; the same body is sent to the
# baseline instance, to /render, and to /derender (as chat_request).
SAMPLE_PROMPTS: list[dict] = [
{
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": image_1_url}},
{"type": "text", "text": "What's in this image?"},
],
}
],
"description": "single image",
},
{
"messages": [
{
"role": "user",
"content": [
{"type": "image_url", "image_url": {"url": image_2_url}},
{"type": "image_url", "image_url": {"url": image_1_url}},
{"type": "text", "text": "Describe these 2 images in detail."},
],
}
],
"description": "two images",
},
{
"messages": [{"role": "user", "content": "What is the capital of France?"}],
"description": "text only",
},
]
def chat_request(prompt: dict) -> dict:
return {
"model": None, # filled by the caller
"messages": prompt["messages"],
"max_tokens": MAX_OUTPUT_LEN,
"temperature": 0.0,
"seed": 42,
}
def post_json(url: str, payload: dict) -> requests.Response:
response = requests.post(url, json=payload, timeout=REQUEST_TIMEOUT)
if response.status_code != 200:
raise RuntimeError(
f"POST {url} failed with {response.status_code}: {response.text}"
)
return response
def run_baseline(args) -> None:
outputs = []
for prompt in SAMPLE_PROMPTS:
request = chat_request(prompt) | {"model": args.model_name}
response = post_json(f"{args.service_url}/v1/chat/completions", request).json()
outputs.append(response["choices"][0]["message"]["content"])
print(f"[baseline] {prompt['description']}: {outputs[-1][:80]!r}")
with open(args.baseline_file, "w") as f:
json.dump(outputs, f)
print(f"Baseline saved to {args.baseline_file}")
def _assert_mm_features(features: dict) -> None:
kwargs_data = features.get("kwargs_data") or {}
mm_metadata = features.get("mm_metadata")
assert any(item is not None for items in kwargs_data.values() for item in items), (
f"render response has no kwargs_data items: {features}"
)
assert mm_metadata, "render response is missing mm_metadata"
for modality, hashes in features["mm_hashes"].items():
assert modality in mm_metadata, f"mm_metadata missing modality {modality}"
assert len(mm_metadata[modality]) == len(hashes)
def run_disagg(args) -> None:
with open(args.baseline_file) as f:
baseline = json.load(f)
failures = []
for prompt, expected in zip(SAMPLE_PROMPTS, baseline, strict=True):
request = chat_request(prompt) | {"model": args.model_name}
got = run_disagg_one(args, request)
if got != expected:
failures.append(
f"{prompt['description']}:\n"
f" baseline: {expected!r}\n disagg: {got!r}"
)
else:
print(f"[disagg] {prompt['description']}: matched baseline")
if failures:
print("Output mismatch:")
for failure in failures:
print(failure)
sys.exit(1)
print("All scale-out EC outputs matched the baseline")
def run_disagg_one(args, request: dict) -> str:
# 1. Render: chat request -> token ids + multimodal features (GPU-less).
rendered = post_json(
f"{args.render_url}/v1/chat/completions/render", request
).json()
token_ids = rendered["token_ids"]
features = rendered.get("features")
request_id = rendered["request_id"]
# 2. Encode: full kwargs_data so the vision encoder can run. The
# response carries ec_transfer_params for the prefill instance.
ec_transfer_params = None
if features is not None:
_assert_mm_features(features)
encode_payload = {
"request_id": request_id,
"token_ids": token_ids,
"features": {
"mm_hashes": features["mm_hashes"],
"mm_placeholders": features["mm_placeholders"],
"kwargs_data": features["kwargs_data"],
},
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
}
encode_response = post_json(
f"{args.encode_url}/inference/v1/generate", encode_payload
).json()
ec_transfer_params = encode_response.get("ec_transfer_params")
assert ec_transfer_params, (
f"encode response has no ec_transfer_params: {encode_response}"
)
# 3. Prefill: metadata-only features. kwargs_data is dropped; the
# embeddings arrive through the EC connector.
if features is not None:
prefill_features = {
"mm_hashes": features["mm_hashes"],
"mm_placeholders": features["mm_placeholders"],
"mm_metadata": features["mm_metadata"],
}
_check_metadata_only_rejected_without_ec(args, token_ids, prefill_features)
else:
prefill_features = None
prefill_payload = {
"request_id": request_id,
"token_ids": token_ids,
"sampling_params": {"max_tokens": MAX_OUTPUT_LEN, "temperature": 0.0},
}
if prefill_features is not None:
prefill_payload["features"] = prefill_features
prefill_payload["ec_transfer_params"] = ec_transfer_params
prefill_response = post_json(
f"{args.prefill_url}/inference/v1/generate", prefill_payload
).json()
output_token_ids = prefill_response["choices"][0]["token_ids"]
assert output_token_ids, f"prefill returned no token ids: {prefill_response}"
# 4. Derender: token ids -> chat completion response (GPU-less).
derendered = post_json(
f"{args.render_url}/v1/chat/completions/derender",
{
"model": args.model_name,
"generate_response": prefill_response,
"prompt_tokens": len(token_ids),
"chat_request": request,
},
).json()
return derendered["choices"][0]["message"]["content"]
def _check_metadata_only_rejected_without_ec(
args, token_ids: list[int], prefill_features: dict
) -> None:
"""mm_metadata without kwargs_data must be rejected without EC params."""
payload = {
"token_ids": token_ids,
"features": prefill_features,
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
}
response = requests.post(
f"{args.prefill_url}/inference/v1/generate",
json=payload,
timeout=REQUEST_TIMEOUT,
)
assert response.status_code in (400, 422), (
"metadata-only features without ec_transfer_params should be rejected, "
f"got {response.status_code}: {response.text}"
)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--mode", choices=["baseline", "disagg"], required=True)
parser.add_argument("--service_url", help="baseline single-instance URL")
parser.add_argument("--render_url", help="GPU-less render server URL")
parser.add_argument("--encode_url", help="EC producer (encode) URL")
parser.add_argument("--prefill_url", help="EC consumer (prefill) URL")
parser.add_argument("--model_name", required=True)
parser.add_argument("--baseline_file", required=True)
args = parser.parse_args()
start = time.time()
if args.mode == "baseline":
assert args.service_url, "--service_url is required in baseline mode"
run_baseline(args)
else:
assert args.render_url, "--render_url is required in disagg mode"
assert args.encode_url, "--encode_url is required in disagg mode"
assert args.prefill_url, "--prefill_url is required in disagg mode"
run_disagg(args)
print(f"Mode {args.mode} finished in {time.time() - start:.1f}s")
if __name__ == "__main__":
main()
@@ -17,6 +17,7 @@ from vllm.distributed.ec_transfer.ec_connector.utils import (
collect_ec_item_metadata,
)
from vllm.logger import init_logger
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
@@ -99,7 +100,10 @@ class ECExampleConnector(ECConnectorBase):
if mm_data.mm_hash in encoder_cache:
continue
filename = self._generate_filename_debug(mm_data.mm_hash)
ec_cache = safetensors.torch.load_file(filename, device=device)["ec_cache"]
with gpu_sync_allowed():
ec_cache = safetensors.torch.load_file(filename, device=device)[
"ec_cache"
]
encoder_cache[mm_data.mm_hash] = ec_cache
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
@@ -121,7 +125,8 @@ class ECExampleConnector(ECConnectorBase):
return
filename = self._generate_filename_debug(mm_hash)
ec_cache = encoder_cache[mm_hash]
tensors = {"ec_cache": ec_cache.detach().cpu()}
with gpu_sync_allowed():
tensors = {"ec_cache": ec_cache.detach().cpu()}
safetensors.torch.save_file(tensors, filename)
logger.debug("Save cache successful for mm_hash %s", mm_hash)