[CI] Add e2e test for scale-out EC connector flow (#54973)
Signed-off-by: zhouyou9505 <zhouyou9505@gmail.com> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: zhouyou9505 <zhouyou9505@gmail.com> Co-authored-by: Cursor Grok 4.6 <noreply@cursor.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
zhouyou9505
Cursor Grok 4.6
OpenAI Codex
parent
cc4210f671
commit
9ffb8cea96
@@ -0,0 +1,16 @@
|
||||
group: Disaggregated EC
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: ":nvidia: (L4) Scale-out EC Connector E2E"
|
||||
key: scale-out-ec-e2e-2-gpus
|
||||
timeout_in_minutes: 40
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
device: l4
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/ec_transfer/
|
||||
- vllm/entrypoints/scale_out/
|
||||
- tests/entrypoints/scale_out/ec_integration/
|
||||
commands:
|
||||
- bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
|
||||
@@ -0,0 +1,43 @@
|
||||
# Scale-out EC connector e2e tests
|
||||
|
||||
End-to-end tests for the disaggregated multimodal (encoder-cache) flow over
|
||||
the scale-out endpoints, in the bash-script style of
|
||||
`tests/v1/kv_connector/nixl_integration`.
|
||||
|
||||
## Topology
|
||||
|
||||
```text
|
||||
render (GPU-less) encode (EC producer)
|
||||
/v1/chat/completions/render /inference/v1/generate (kwargs_data)
|
||||
/v1/chat/completions/derender |
|
||||
prefill (EC consumer)
|
||||
/inference/v1/generate (mm_metadata + ec_transfer_params)
|
||||
```
|
||||
|
||||
- The **render** server (`vllm launch render`) preprocesses the chat request
|
||||
into token ids and multimodal features (`kwargs_data` + `mm_metadata`).
|
||||
- The **encode** instance (`ec_role: ec_producer`, encode-only) runs the
|
||||
vision encoder on `kwargs_data` and publishes embeddings through the
|
||||
`ECExampleConnector` shared storage. Its response carries
|
||||
`ec_transfer_params`.
|
||||
- The **prefill** instance (`ec_role: ec_consumer`) receives metadata-only
|
||||
features (`mm_metadata`, no `kwargs_data`) plus `ec_transfer_params`, loads
|
||||
the embeddings through the EC connector, and generates output token ids.
|
||||
- The **derender** endpoint turns the output token ids back into a chat
|
||||
completion response.
|
||||
|
||||
## Running
|
||||
|
||||
```bash
|
||||
bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
|
||||
```
|
||||
|
||||
The script first collects baseline outputs from a single `vllm serve`
|
||||
instance, then runs the scale-out topology and compares the derendered
|
||||
outputs for exact equality. It also checks that metadata-only features are
|
||||
rejected when `ec_transfer_params` is missing.
|
||||
|
||||
Defaults: `Qwen/Qwen3-VL-2B-Instruct`, encode on GPU 0, prefill on GPU 1.
|
||||
Override via environment variables (`MODEL`, `GPU_E`, `GPU_PD`,
|
||||
`EC_SHARED_STORAGE_PATH`, `PYTHON=.venv/bin/python`, ...); see the header of
|
||||
the script for the full list.
|
||||
@@ -0,0 +1,275 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Scale-out EC connector E2E test
|
||||
#
|
||||
# Tests the disaggregated multimodal flow over the scale-out endpoints,
|
||||
# following the bash-script style of tests/v1/kv_connector/nixl_integration:
|
||||
#
|
||||
# 1. Baseline: a single vLLM instance serving /v1/chat/completions
|
||||
# 2. Scale-out:
|
||||
# render (GPU-less, `vllm launch render`)
|
||||
# encode (EC producer, encode-only)
|
||||
# prefill (EC consumer)
|
||||
#
|
||||
# The Python client renders each multimodal request once, sends the full
|
||||
# kwargs_data to the encode instance, then sends metadata-only features
|
||||
# (mm_metadata, no kwargs_data) plus ec_transfer_params to the prefill
|
||||
# instance, and derenders the output tokens. Outputs are compared against
|
||||
# the baseline for exact equality.
|
||||
#
|
||||
# Usage:
|
||||
# bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh
|
||||
#
|
||||
# Requires 2 GPUs by default (encode on GPU 0, prefill on GPU 1); the
|
||||
# baseline reuses GPU 0 after the encode instance is stopped.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Resolve the repository root from the script location instead of `.git`.
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||
|
||||
# Model and engine configuration
|
||||
MODEL="${MODEL:-Qwen/Qwen3-VL-2B-Instruct}"
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-8192}"
|
||||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}"
|
||||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
|
||||
|
||||
# GPU configuration. encode + prefill run concurrently on separate GPUs;
|
||||
# the baseline reuses GPU_E after the scale-out instances are stopped.
|
||||
GPU_E="${GPU_E:-0}"
|
||||
GPU_PD="${GPU_PD:-1}"
|
||||
GPU_SINGLE="${GPU_SINGLE:-$GPU_E}"
|
||||
|
||||
# Ports
|
||||
RENDER_PORT="${RENDER_PORT:-19600}"
|
||||
ENCODE_PORT="${ENCODE_PORT:-19601}"
|
||||
PREFILL_PORT="${PREFILL_PORT:-19602}"
|
||||
BASELINE_PORT="${BASELINE_PORT:-19603}"
|
||||
|
||||
# Storage path for the ECExampleConnector encoder cache
|
||||
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_scale_out_test}"
|
||||
|
||||
# Python interpreter (override to .venv/bin/python for local runs)
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
|
||||
# Output files and logs
|
||||
LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_scale_out_ec_baseline.txt}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}"
|
||||
BASELINE_LOG="${LOG_PATH}/scale_out_ec_baseline.log"
|
||||
RENDER_LOG="${LOG_PATH}/scale_out_ec_render.log"
|
||||
ENCODE_LOG="${LOG_PATH}/scale_out_ec_encode.log"
|
||||
PREFILL_LOG="${LOG_PATH}/scale_out_ec_prefill.log"
|
||||
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local pid=$2
|
||||
local name=$3
|
||||
local deadline=$((SECONDS + TIMEOUT_SECONDS))
|
||||
|
||||
while ((SECONDS < deadline)); do
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
echo "$name instance exited before becoming healthy"
|
||||
return 1
|
||||
fi
|
||||
if curl --max-time 2 -fsS "http://localhost:${port}/health" \
|
||||
> /dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "Timed out waiting for $name instance on port $port"
|
||||
return 1
|
||||
}
|
||||
|
||||
dump_log() {
|
||||
local name=$1
|
||||
local log=$2
|
||||
|
||||
echo "--- $name log (last 200 lines)"
|
||||
if [[ -f "$log" ]]; then
|
||||
tail -n 200 "$log" || true
|
||||
else
|
||||
echo "Log file not found: $log"
|
||||
fi
|
||||
}
|
||||
|
||||
dump_scale_out_logs() {
|
||||
dump_log "render" "$RENDER_LOG"
|
||||
dump_log "encode" "$ENCODE_LOG"
|
||||
dump_log "prefill" "$PREFILL_LOG"
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
echo "Cleaning up any running vLLM instances..."
|
||||
pkill -TERM -f "vllm serve" || true
|
||||
pkill -TERM -f "vllm launch render" || true
|
||||
sleep 3
|
||||
pkill -9 -f "vllm serve" || true
|
||||
pkill -9 -f "vllm launch render" || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
# Step 1: baseline single instance
|
||||
run_baseline() {
|
||||
echo "================================"
|
||||
echo "Running BASELINE (single instance)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
echo "Starting baseline instance on GPU $GPU_SINGLE, port $BASELINE_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \
|
||||
--port "$BASELINE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
> "$BASELINE_LOG" 2>&1 &
|
||||
local BASELINE_PID=$!
|
||||
|
||||
echo "Waiting for baseline instance to start..."
|
||||
if ! wait_for_server "$BASELINE_PORT" "$BASELINE_PID" "baseline"; then
|
||||
dump_log "baseline" "$BASELINE_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
|
||||
--mode baseline \
|
||||
--service_url "http://localhost:$BASELINE_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--baseline_file "$BASELINE_FILE"; then
|
||||
dump_log "baseline" "$BASELINE_LOG"
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Stopping baseline instance..."
|
||||
kill "$BASELINE_PID" 2>/dev/null || true
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Step 2: render + encode (EC producer) + prefill (EC consumer)
|
||||
run_scale_out_ec() {
|
||||
echo "================================"
|
||||
echo "Running SCALE-OUT EC (render + encode + prefill)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
mkdir -p "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
declare -a PIDS=()
|
||||
|
||||
# GPU-less render/derender server
|
||||
echo "Starting render server on port $RENDER_PORT"
|
||||
vllm launch render "$MODEL" \
|
||||
--port "$RENDER_PORT" \
|
||||
> "$RENDER_LOG" 2>&1 &
|
||||
PIDS+=("$!")
|
||||
|
||||
# Encode-only EC producer instance. It runs the vision encoder and
|
||||
# publishes embeddings through the ECExampleConnector shared storage.
|
||||
echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_E" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
|
||||
vllm serve "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.01 \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$ENCODE_LOG" 2>&1 &
|
||||
PIDS+=("$!")
|
||||
|
||||
# Prefill/decode EC consumer instance. It receives metadata-only
|
||||
# features plus ec_transfer_params and loads the embeddings that the
|
||||
# encode instance published.
|
||||
echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT"
|
||||
env CUDA_VISIBLE_DEVICES="$GPU_PD" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \
|
||||
vllm serve "$MODEL" \
|
||||
--port "$PREFILL_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$PREFILL_LOG" 2>&1 &
|
||||
PIDS+=("$!")
|
||||
|
||||
echo "Waiting for render instance..."
|
||||
if ! wait_for_server "$RENDER_PORT" "${PIDS[0]}" "render"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
echo "Waiting for encode instance..."
|
||||
if ! wait_for_server "$ENCODE_PORT" "${PIDS[1]}" "encode"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
echo "Waiting for prefill instance..."
|
||||
if ! wait_for_server "$PREFILL_PORT" "${PIDS[2]}" "prefill"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "All scale-out EC services are up!"
|
||||
|
||||
if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \
|
||||
--mode disagg \
|
||||
--render_url "http://localhost:$RENDER_PORT" \
|
||||
--encode_url "http://localhost:$ENCODE_PORT" \
|
||||
--prefill_url "http://localhost:$PREFILL_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--baseline_file "$BASELINE_FILE"; then
|
||||
dump_scale_out_logs
|
||||
return 1
|
||||
fi
|
||||
|
||||
echo "Stopping scale-out EC instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Main execution
|
||||
echo "================================"
|
||||
echo "Scale-out EC E2E Test Suite"
|
||||
echo "Model: $MODEL"
|
||||
echo "================================"
|
||||
|
||||
run_baseline
|
||||
run_scale_out_ec
|
||||
|
||||
rm -f "$BASELINE_FILE"
|
||||
|
||||
echo "================================"
|
||||
echo "All scale-out EC e2e tests finished!"
|
||||
echo "================================"
|
||||
@@ -0,0 +1,276 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""E2E test for the scale-out EC connector flow (render -> encode -> prefill).
|
||||
|
||||
Mimics the bash-script style of ``tests/v1/kv_connector/nixl_integration``:
|
||||
the instances are started by ``run_scale_out_ec_e2e_test.sh`` and this file is
|
||||
the HTTP client that drives the disaggregated request path and compares its
|
||||
output against a single-instance baseline.
|
||||
|
||||
Topology under test (started by the bash script):
|
||||
|
||||
render (GPU-less) -- /v1/chat/completions/render, /derender
|
||||
encode (EC producer) -- /inference/v1/generate
|
||||
prefill (EC consumer) -- /inference/v1/generate
|
||||
|
||||
The client renders each multimodal chat request once, sends the full
|
||||
``kwargs_data`` to the encode instance, then sends metadata-only features
|
||||
(``mm_metadata`` without ``kwargs_data``) plus the encode response's
|
||||
``ec_transfer_params`` to the prefill instance, and finally derenders the
|
||||
output token ids. The derendered text must match the baseline exactly.
|
||||
|
||||
Usage (from the bash script, or manually):
|
||||
# Baseline mode (saves outputs):
|
||||
python test_scale_out_ec_e2e.py \
|
||||
--mode baseline \
|
||||
--service_url http://localhost:19603 \
|
||||
--model_name Qwen/Qwen3-VL-2B-Instruct \
|
||||
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
|
||||
|
||||
# Disagg mode (compares outputs):
|
||||
python test_scale_out_ec_e2e.py \
|
||||
--mode disagg \
|
||||
--render_url http://localhost:19600 \
|
||||
--encode_url http://localhost:19601 \
|
||||
--prefill_url http://localhost:19602 \
|
||||
--model_name Qwen/Qwen3-VL-2B-Instruct \
|
||||
--baseline_file /tmp/vllm_scale_out_ec_baseline.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MAX_OUTPUT_LEN = 128
|
||||
REQUEST_TIMEOUT = 300
|
||||
|
||||
image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720))
|
||||
image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720))
|
||||
image_1_url = encode_image_url(image_1)
|
||||
image_2_url = encode_image_url(image_2)
|
||||
|
||||
# Each prompt is a chat completion request body; the same body is sent to the
|
||||
# baseline instance, to /render, and to /derender (as chat_request).
|
||||
SAMPLE_PROMPTS: list[dict] = [
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image_1_url}},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"description": "single image",
|
||||
},
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image_2_url}},
|
||||
{"type": "image_url", "image_url": {"url": image_1_url}},
|
||||
{"type": "text", "text": "Describe these 2 images in detail."},
|
||||
],
|
||||
}
|
||||
],
|
||||
"description": "two images",
|
||||
},
|
||||
{
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"description": "text only",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def chat_request(prompt: dict) -> dict:
|
||||
return {
|
||||
"model": None, # filled by the caller
|
||||
"messages": prompt["messages"],
|
||||
"max_tokens": MAX_OUTPUT_LEN,
|
||||
"temperature": 0.0,
|
||||
"seed": 42,
|
||||
}
|
||||
|
||||
|
||||
def post_json(url: str, payload: dict) -> requests.Response:
|
||||
response = requests.post(url, json=payload, timeout=REQUEST_TIMEOUT)
|
||||
if response.status_code != 200:
|
||||
raise RuntimeError(
|
||||
f"POST {url} failed with {response.status_code}: {response.text}"
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def run_baseline(args) -> None:
|
||||
outputs = []
|
||||
for prompt in SAMPLE_PROMPTS:
|
||||
request = chat_request(prompt) | {"model": args.model_name}
|
||||
response = post_json(f"{args.service_url}/v1/chat/completions", request).json()
|
||||
outputs.append(response["choices"][0]["message"]["content"])
|
||||
print(f"[baseline] {prompt['description']}: {outputs[-1][:80]!r}")
|
||||
|
||||
with open(args.baseline_file, "w") as f:
|
||||
json.dump(outputs, f)
|
||||
print(f"Baseline saved to {args.baseline_file}")
|
||||
|
||||
|
||||
def _assert_mm_features(features: dict) -> None:
|
||||
kwargs_data = features.get("kwargs_data") or {}
|
||||
mm_metadata = features.get("mm_metadata")
|
||||
assert any(item is not None for items in kwargs_data.values() for item in items), (
|
||||
f"render response has no kwargs_data items: {features}"
|
||||
)
|
||||
assert mm_metadata, "render response is missing mm_metadata"
|
||||
for modality, hashes in features["mm_hashes"].items():
|
||||
assert modality in mm_metadata, f"mm_metadata missing modality {modality}"
|
||||
assert len(mm_metadata[modality]) == len(hashes)
|
||||
|
||||
|
||||
def run_disagg(args) -> None:
|
||||
with open(args.baseline_file) as f:
|
||||
baseline = json.load(f)
|
||||
|
||||
failures = []
|
||||
for prompt, expected in zip(SAMPLE_PROMPTS, baseline, strict=True):
|
||||
request = chat_request(prompt) | {"model": args.model_name}
|
||||
got = run_disagg_one(args, request)
|
||||
if got != expected:
|
||||
failures.append(
|
||||
f"{prompt['description']}:\n"
|
||||
f" baseline: {expected!r}\n disagg: {got!r}"
|
||||
)
|
||||
else:
|
||||
print(f"[disagg] {prompt['description']}: matched baseline")
|
||||
|
||||
if failures:
|
||||
print("Output mismatch:")
|
||||
for failure in failures:
|
||||
print(failure)
|
||||
sys.exit(1)
|
||||
print("All scale-out EC outputs matched the baseline")
|
||||
|
||||
|
||||
def run_disagg_one(args, request: dict) -> str:
|
||||
# 1. Render: chat request -> token ids + multimodal features (GPU-less).
|
||||
rendered = post_json(
|
||||
f"{args.render_url}/v1/chat/completions/render", request
|
||||
).json()
|
||||
token_ids = rendered["token_ids"]
|
||||
features = rendered.get("features")
|
||||
request_id = rendered["request_id"]
|
||||
|
||||
# 2. Encode: full kwargs_data so the vision encoder can run. The
|
||||
# response carries ec_transfer_params for the prefill instance.
|
||||
ec_transfer_params = None
|
||||
if features is not None:
|
||||
_assert_mm_features(features)
|
||||
encode_payload = {
|
||||
"request_id": request_id,
|
||||
"token_ids": token_ids,
|
||||
"features": {
|
||||
"mm_hashes": features["mm_hashes"],
|
||||
"mm_placeholders": features["mm_placeholders"],
|
||||
"kwargs_data": features["kwargs_data"],
|
||||
},
|
||||
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
|
||||
}
|
||||
encode_response = post_json(
|
||||
f"{args.encode_url}/inference/v1/generate", encode_payload
|
||||
).json()
|
||||
ec_transfer_params = encode_response.get("ec_transfer_params")
|
||||
assert ec_transfer_params, (
|
||||
f"encode response has no ec_transfer_params: {encode_response}"
|
||||
)
|
||||
|
||||
# 3. Prefill: metadata-only features. kwargs_data is dropped; the
|
||||
# embeddings arrive through the EC connector.
|
||||
if features is not None:
|
||||
prefill_features = {
|
||||
"mm_hashes": features["mm_hashes"],
|
||||
"mm_placeholders": features["mm_placeholders"],
|
||||
"mm_metadata": features["mm_metadata"],
|
||||
}
|
||||
_check_metadata_only_rejected_without_ec(args, token_ids, prefill_features)
|
||||
else:
|
||||
prefill_features = None
|
||||
|
||||
prefill_payload = {
|
||||
"request_id": request_id,
|
||||
"token_ids": token_ids,
|
||||
"sampling_params": {"max_tokens": MAX_OUTPUT_LEN, "temperature": 0.0},
|
||||
}
|
||||
if prefill_features is not None:
|
||||
prefill_payload["features"] = prefill_features
|
||||
prefill_payload["ec_transfer_params"] = ec_transfer_params
|
||||
prefill_response = post_json(
|
||||
f"{args.prefill_url}/inference/v1/generate", prefill_payload
|
||||
).json()
|
||||
output_token_ids = prefill_response["choices"][0]["token_ids"]
|
||||
assert output_token_ids, f"prefill returned no token ids: {prefill_response}"
|
||||
|
||||
# 4. Derender: token ids -> chat completion response (GPU-less).
|
||||
derendered = post_json(
|
||||
f"{args.render_url}/v1/chat/completions/derender",
|
||||
{
|
||||
"model": args.model_name,
|
||||
"generate_response": prefill_response,
|
||||
"prompt_tokens": len(token_ids),
|
||||
"chat_request": request,
|
||||
},
|
||||
).json()
|
||||
return derendered["choices"][0]["message"]["content"]
|
||||
|
||||
|
||||
def _check_metadata_only_rejected_without_ec(
|
||||
args, token_ids: list[int], prefill_features: dict
|
||||
) -> None:
|
||||
"""mm_metadata without kwargs_data must be rejected without EC params."""
|
||||
payload = {
|
||||
"token_ids": token_ids,
|
||||
"features": prefill_features,
|
||||
"sampling_params": {"max_tokens": 1, "temperature": 0.0},
|
||||
}
|
||||
response = requests.post(
|
||||
f"{args.prefill_url}/inference/v1/generate",
|
||||
json=payload,
|
||||
timeout=REQUEST_TIMEOUT,
|
||||
)
|
||||
assert response.status_code in (400, 422), (
|
||||
"metadata-only features without ec_transfer_params should be rejected, "
|
||||
f"got {response.status_code}: {response.text}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--mode", choices=["baseline", "disagg"], required=True)
|
||||
parser.add_argument("--service_url", help="baseline single-instance URL")
|
||||
parser.add_argument("--render_url", help="GPU-less render server URL")
|
||||
parser.add_argument("--encode_url", help="EC producer (encode) URL")
|
||||
parser.add_argument("--prefill_url", help="EC consumer (prefill) URL")
|
||||
parser.add_argument("--model_name", required=True)
|
||||
parser.add_argument("--baseline_file", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
start = time.time()
|
||||
if args.mode == "baseline":
|
||||
assert args.service_url, "--service_url is required in baseline mode"
|
||||
run_baseline(args)
|
||||
else:
|
||||
assert args.render_url, "--render_url is required in disagg mode"
|
||||
assert args.encode_url, "--encode_url is required in disagg mode"
|
||||
assert args.prefill_url, "--prefill_url is required in disagg mode"
|
||||
run_disagg(args)
|
||||
print(f"Mode {args.mode} finished in {time.time() - start:.1f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -99,7 +100,10 @@ class ECExampleConnector(ECConnectorBase):
|
||||
if mm_data.mm_hash in encoder_cache:
|
||||
continue
|
||||
filename = self._generate_filename_debug(mm_data.mm_hash)
|
||||
ec_cache = safetensors.torch.load_file(filename, device=device)["ec_cache"]
|
||||
with gpu_sync_allowed():
|
||||
ec_cache = safetensors.torch.load_file(filename, device=device)[
|
||||
"ec_cache"
|
||||
]
|
||||
encoder_cache[mm_data.mm_hash] = ec_cache
|
||||
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
|
||||
|
||||
@@ -121,7 +125,8 @@ class ECExampleConnector(ECConnectorBase):
|
||||
return
|
||||
filename = self._generate_filename_debug(mm_hash)
|
||||
ec_cache = encoder_cache[mm_hash]
|
||||
tensors = {"ec_cache": ec_cache.detach().cpu()}
|
||||
with gpu_sync_allowed():
|
||||
tensors = {"ec_cache": ec_cache.detach().cpu()}
|
||||
safetensors.torch.save_file(tensors, filename)
|
||||
logger.debug("Save cache successful for mm_hash %s", mm_hash)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user