From 9ffb8cea96369e2164af184a10faaa618eaf6b92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 9 Sep 2026 16:38:32 +0200 Subject: [PATCH] [CI] Add e2e test for scale-out EC connector flow (#54973) Signed-off-by: zhouyou9505 Signed-off-by: NickLucche Co-authored-by: zhouyou9505 Co-authored-by: Cursor Grok 4.6 Co-authored-by: OpenAI Codex --- .buildkite/test_areas/disaggregated_ec.yaml | 16 + .../scale_out/ec_integration/README.md | 43 +++ .../run_scale_out_ec_e2e_test.sh | 275 +++++++++++++++++ .../ec_integration/test_scale_out_ec_e2e.py | 276 ++++++++++++++++++ .../ec_connector/example_connector.py | 9 +- 5 files changed, 617 insertions(+), 2 deletions(-) create mode 100644 .buildkite/test_areas/disaggregated_ec.yaml create mode 100644 tests/entrypoints/scale_out/ec_integration/README.md create mode 100755 tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh create mode 100644 tests/entrypoints/scale_out/ec_integration/test_scale_out_ec_e2e.py diff --git a/.buildkite/test_areas/disaggregated_ec.yaml b/.buildkite/test_areas/disaggregated_ec.yaml new file mode 100644 index 0000000000..9100ad0a43 --- /dev/null +++ b/.buildkite/test_areas/disaggregated_ec.yaml @@ -0,0 +1,16 @@ +group: Disaggregated EC +depends_on: + - image-build +steps: +- label: ":nvidia: (L4) Scale-out EC Connector E2E" + key: scale-out-ec-e2e-2-gpus + timeout_in_minutes: 40 + working_dir: "/vllm-workspace/tests" + device: l4 + num_devices: 2 + source_file_dependencies: + - vllm/distributed/ec_transfer/ + - vllm/entrypoints/scale_out/ + - tests/entrypoints/scale_out/ec_integration/ + commands: + - bash entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh diff --git a/tests/entrypoints/scale_out/ec_integration/README.md b/tests/entrypoints/scale_out/ec_integration/README.md new file mode 100644 index 0000000000..cc17cf2e9e --- /dev/null +++ b/tests/entrypoints/scale_out/ec_integration/README.md @@ -0,0 +1,43 @@ +# Scale-out EC connector e2e tests + +End-to-end tests for the disaggregated multimodal (encoder-cache) flow over +the scale-out endpoints, in the bash-script style of +`tests/v1/kv_connector/nixl_integration`. + +## Topology + +```text +render (GPU-less) encode (EC producer) + /v1/chat/completions/render /inference/v1/generate (kwargs_data) + /v1/chat/completions/derender | + prefill (EC consumer) + /inference/v1/generate (mm_metadata + ec_transfer_params) +``` + +- The **render** server (`vllm launch render`) preprocesses the chat request + into token ids and multimodal features (`kwargs_data` + `mm_metadata`). +- The **encode** instance (`ec_role: ec_producer`, encode-only) runs the + vision encoder on `kwargs_data` and publishes embeddings through the + `ECExampleConnector` shared storage. Its response carries + `ec_transfer_params`. +- The **prefill** instance (`ec_role: ec_consumer`) receives metadata-only + features (`mm_metadata`, no `kwargs_data`) plus `ec_transfer_params`, loads + the embeddings through the EC connector, and generates output token ids. +- The **derender** endpoint turns the output token ids back into a chat + completion response. + +## Running + +```bash +bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh +``` + +The script first collects baseline outputs from a single `vllm serve` +instance, then runs the scale-out topology and compares the derendered +outputs for exact equality. It also checks that metadata-only features are +rejected when `ec_transfer_params` is missing. + +Defaults: `Qwen/Qwen3-VL-2B-Instruct`, encode on GPU 0, prefill on GPU 1. +Override via environment variables (`MODEL`, `GPU_E`, `GPU_PD`, +`EC_SHARED_STORAGE_PATH`, `PYTHON=.venv/bin/python`, ...); see the header of +the script for the full list. diff --git a/tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh b/tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh new file mode 100755 index 0000000000..b99b1c3ca5 --- /dev/null +++ b/tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh @@ -0,0 +1,275 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Scale-out EC connector E2E test +# +# Tests the disaggregated multimodal flow over the scale-out endpoints, +# following the bash-script style of tests/v1/kv_connector/nixl_integration: +# +# 1. Baseline: a single vLLM instance serving /v1/chat/completions +# 2. Scale-out: +# render (GPU-less, `vllm launch render`) +# encode (EC producer, encode-only) +# prefill (EC consumer) +# +# The Python client renders each multimodal request once, sends the full +# kwargs_data to the encode instance, then sends metadata-only features +# (mm_metadata, no kwargs_data) plus ec_transfer_params to the prefill +# instance, and derenders the output tokens. Outputs are compared against +# the baseline for exact equality. +# +# Usage: +# bash tests/entrypoints/scale_out/ec_integration/run_scale_out_ec_e2e_test.sh +# +# Requires 2 GPUs by default (encode on GPU 0, prefill on GPU 1); the +# baseline reuses GPU 0 after the encode instance is stopped. + +set -euo pipefail + +# Resolve the repository root from the script location instead of `.git`. +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" + +# Model and engine configuration +MODEL="${MODEL:-Qwen/Qwen3-VL-2B-Instruct}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-8192}" +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}" + +# GPU configuration. encode + prefill run concurrently on separate GPUs; +# the baseline reuses GPU_E after the scale-out instances are stopped. +GPU_E="${GPU_E:-0}" +GPU_PD="${GPU_PD:-1}" +GPU_SINGLE="${GPU_SINGLE:-$GPU_E}" + +# Ports +RENDER_PORT="${RENDER_PORT:-19600}" +ENCODE_PORT="${ENCODE_PORT:-19601}" +PREFILL_PORT="${PREFILL_PORT:-19602}" +BASELINE_PORT="${BASELINE_PORT:-19603}" + +# Storage path for the ECExampleConnector encoder cache +EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_scale_out_test}" + +# Python interpreter (override to .venv/bin/python for local runs) +PYTHON="${PYTHON:-python3}" + +# Output files and logs +LOG_PATH="${LOG_PATH:-/tmp}" +BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_scale_out_ec_baseline.txt}" +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}" +BASELINE_LOG="${LOG_PATH}/scale_out_ec_baseline.log" +RENDER_LOG="${LOG_PATH}/scale_out_ec_render.log" +ENCODE_LOG="${LOG_PATH}/scale_out_ec_encode.log" +PREFILL_LOG="${LOG_PATH}/scale_out_ec_prefill.log" + +mkdir -p "$LOG_PATH" + +# Trap the SIGINT signal (triggered by Ctrl+C) +trap 'kill $(jobs -pr) 2>/dev/null || true' SIGINT SIGTERM EXIT + +wait_for_server() { + local port=$1 + local pid=$2 + local name=$3 + local deadline=$((SECONDS + TIMEOUT_SECONDS)) + + while ((SECONDS < deadline)); do + if ! kill -0 "$pid" 2>/dev/null; then + echo "$name instance exited before becoming healthy" + return 1 + fi + if curl --max-time 2 -fsS "http://localhost:${port}/health" \ + > /dev/null 2>&1; then + return 0 + fi + sleep 1 + done + + echo "Timed out waiting for $name instance on port $port" + return 1 +} + +dump_log() { + local name=$1 + local log=$2 + + echo "--- $name log (last 200 lines)" + if [[ -f "$log" ]]; then + tail -n 200 "$log" || true + else + echo "Log file not found: $log" + fi +} + +dump_scale_out_logs() { + dump_log "render" "$RENDER_LOG" + dump_log "encode" "$ENCODE_LOG" + dump_log "prefill" "$PREFILL_LOG" +} + +cleanup_instances() { + echo "Cleaning up any running vLLM instances..." + pkill -TERM -f "vllm serve" || true + pkill -TERM -f "vllm launch render" || true + sleep 3 + pkill -9 -f "vllm serve" || true + pkill -9 -f "vllm launch render" || true + sleep 2 +} + +# Step 1: baseline single instance +run_baseline() { + echo "================================" + echo "Running BASELINE (single instance)" + echo "================================" + + cleanup_instances + rm -rf "$EC_SHARED_STORAGE_PATH" + + echo "Starting baseline instance on GPU $GPU_SINGLE, port $BASELINE_PORT" + env CUDA_VISIBLE_DEVICES="$GPU_SINGLE" vllm serve "$MODEL" \ + --port "$BASELINE_PORT" \ + --max-model-len "$MAX_MODEL_LEN" \ + --enforce-eager \ + --gpu-memory-utilization 0.9 \ + --max-num-seqs "$MAX_NUM_SEQS" \ + > "$BASELINE_LOG" 2>&1 & + local BASELINE_PID=$! + + echo "Waiting for baseline instance to start..." + if ! wait_for_server "$BASELINE_PORT" "$BASELINE_PID" "baseline"; then + dump_log "baseline" "$BASELINE_LOG" + return 1 + fi + + if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \ + --mode baseline \ + --service_url "http://localhost:$BASELINE_PORT" \ + --model_name "$MODEL" \ + --baseline_file "$BASELINE_FILE"; then + dump_log "baseline" "$BASELINE_LOG" + return 1 + fi + + echo "Stopping baseline instance..." + kill "$BASELINE_PID" 2>/dev/null || true + sleep 2 + cleanup_instances +} + +# Step 2: render + encode (EC producer) + prefill (EC consumer) +run_scale_out_ec() { + echo "================================" + echo "Running SCALE-OUT EC (render + encode + prefill)" + echo "================================" + + cleanup_instances + rm -rf "$EC_SHARED_STORAGE_PATH" + mkdir -p "$EC_SHARED_STORAGE_PATH" + + declare -a PIDS=() + + # GPU-less render/derender server + echo "Starting render server on port $RENDER_PORT" + vllm launch render "$MODEL" \ + --port "$RENDER_PORT" \ + > "$RENDER_LOG" 2>&1 & + PIDS+=("$!") + + # Encode-only EC producer instance. It runs the vision encoder and + # publishes embeddings through the ECExampleConnector shared storage. + echo "Starting encode instance on GPU $GPU_E, port $ENCODE_PORT" + env CUDA_VISIBLE_DEVICES="$GPU_E" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \ + vllm serve "$MODEL" \ + --port "$ENCODE_PORT" \ + --max-model-len "$MAX_MODEL_LEN" \ + --enforce-eager \ + --gpu-memory-utilization 0.01 \ + --no-enable-prefix-caching \ + --max-num-batched-tokens 114688 \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --ec-transfer-config '{ + "ec_connector": "ECExampleConnector", + "ec_role": "ec_producer", + "ec_connector_extra_config": { + "shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'" + } + }' \ + > "$ENCODE_LOG" 2>&1 & + PIDS+=("$!") + + # Prefill/decode EC consumer instance. It receives metadata-only + # features plus ec_transfer_params and loads the embeddings that the + # encode instance published. + echo "Starting prefill instance on GPU $GPU_PD, port $PREFILL_PORT" + env CUDA_VISIBLE_DEVICES="$GPU_PD" VLLM_ENABLE_SCALE_OUT_ENDPOINTS=1 \ + vllm serve "$MODEL" \ + --port "$PREFILL_PORT" \ + --max-model-len "$MAX_MODEL_LEN" \ + --enforce-eager \ + --enable-mm-embeds \ + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --ec-transfer-config '{ + "ec_connector": "ECExampleConnector", + "ec_role": "ec_consumer", + "ec_connector_extra_config": { + "shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'" + } + }' \ + > "$PREFILL_LOG" 2>&1 & + PIDS+=("$!") + + echo "Waiting for render instance..." + if ! wait_for_server "$RENDER_PORT" "${PIDS[0]}" "render"; then + dump_scale_out_logs + return 1 + fi + echo "Waiting for encode instance..." + if ! wait_for_server "$ENCODE_PORT" "${PIDS[1]}" "encode"; then + dump_scale_out_logs + return 1 + fi + echo "Waiting for prefill instance..." + if ! wait_for_server "$PREFILL_PORT" "${PIDS[2]}" "prefill"; then + dump_scale_out_logs + return 1 + fi + + echo "All scale-out EC services are up!" + + if ! "$PYTHON" "${SCRIPT_DIR}/test_scale_out_ec_e2e.py" \ + --mode disagg \ + --render_url "http://localhost:$RENDER_PORT" \ + --encode_url "http://localhost:$ENCODE_PORT" \ + --prefill_url "http://localhost:$PREFILL_PORT" \ + --model_name "$MODEL" \ + --baseline_file "$BASELINE_FILE"; then + dump_scale_out_logs + return 1 + fi + + echo "Stopping scale-out EC instances..." + for pid in "${PIDS[@]}"; do + kill "$pid" 2>/dev/null || true + done + sleep 2 + cleanup_instances +} + +# Main execution +echo "================================" +echo "Scale-out EC E2E Test Suite" +echo "Model: $MODEL" +echo "================================" + +run_baseline +run_scale_out_ec + +rm -f "$BASELINE_FILE" + +echo "================================" +echo "All scale-out EC e2e tests finished!" +echo "================================" diff --git a/tests/entrypoints/scale_out/ec_integration/test_scale_out_ec_e2e.py b/tests/entrypoints/scale_out/ec_integration/test_scale_out_ec_e2e.py new file mode 100644 index 0000000000..1f7dfab2f2 --- /dev/null +++ b/tests/entrypoints/scale_out/ec_integration/test_scale_out_ec_e2e.py @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""E2E test for the scale-out EC connector flow (render -> encode -> prefill). + +Mimics the bash-script style of ``tests/v1/kv_connector/nixl_integration``: +the instances are started by ``run_scale_out_ec_e2e_test.sh`` and this file is +the HTTP client that drives the disaggregated request path and compares its +output against a single-instance baseline. + +Topology under test (started by the bash script): + + render (GPU-less) -- /v1/chat/completions/render, /derender + encode (EC producer) -- /inference/v1/generate + prefill (EC consumer) -- /inference/v1/generate + +The client renders each multimodal chat request once, sends the full +``kwargs_data`` to the encode instance, then sends metadata-only features +(``mm_metadata`` without ``kwargs_data``) plus the encode response's +``ec_transfer_params`` to the prefill instance, and finally derenders the +output token ids. The derendered text must match the baseline exactly. + +Usage (from the bash script, or manually): + # Baseline mode (saves outputs): + python test_scale_out_ec_e2e.py \ + --mode baseline \ + --service_url http://localhost:19603 \ + --model_name Qwen/Qwen3-VL-2B-Instruct \ + --baseline_file /tmp/vllm_scale_out_ec_baseline.txt + + # Disagg mode (compares outputs): + python test_scale_out_ec_e2e.py \ + --mode disagg \ + --render_url http://localhost:19600 \ + --encode_url http://localhost:19601 \ + --prefill_url http://localhost:19602 \ + --model_name Qwen/Qwen3-VL-2B-Instruct \ + --baseline_file /tmp/vllm_scale_out_ec_baseline.txt +""" + +import argparse +import json +import sys +import time + +import requests + +from vllm.assets.image import ImageAsset +from vllm.multimodal.utils import encode_image_url + +MAX_OUTPUT_LEN = 128 +REQUEST_TIMEOUT = 300 + +image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720)) +image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720)) +image_1_url = encode_image_url(image_1) +image_2_url = encode_image_url(image_2) + +# Each prompt is a chat completion request body; the same body is sent to the +# baseline instance, to /render, and to /derender (as chat_request). +SAMPLE_PROMPTS: list[dict] = [ + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image_1_url}}, + {"type": "text", "text": "What's in this image?"}, + ], + } + ], + "description": "single image", + }, + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": {"url": image_2_url}}, + {"type": "image_url", "image_url": {"url": image_1_url}}, + {"type": "text", "text": "Describe these 2 images in detail."}, + ], + } + ], + "description": "two images", + }, + { + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "description": "text only", + }, +] + + +def chat_request(prompt: dict) -> dict: + return { + "model": None, # filled by the caller + "messages": prompt["messages"], + "max_tokens": MAX_OUTPUT_LEN, + "temperature": 0.0, + "seed": 42, + } + + +def post_json(url: str, payload: dict) -> requests.Response: + response = requests.post(url, json=payload, timeout=REQUEST_TIMEOUT) + if response.status_code != 200: + raise RuntimeError( + f"POST {url} failed with {response.status_code}: {response.text}" + ) + return response + + +def run_baseline(args) -> None: + outputs = [] + for prompt in SAMPLE_PROMPTS: + request = chat_request(prompt) | {"model": args.model_name} + response = post_json(f"{args.service_url}/v1/chat/completions", request).json() + outputs.append(response["choices"][0]["message"]["content"]) + print(f"[baseline] {prompt['description']}: {outputs[-1][:80]!r}") + + with open(args.baseline_file, "w") as f: + json.dump(outputs, f) + print(f"Baseline saved to {args.baseline_file}") + + +def _assert_mm_features(features: dict) -> None: + kwargs_data = features.get("kwargs_data") or {} + mm_metadata = features.get("mm_metadata") + assert any(item is not None for items in kwargs_data.values() for item in items), ( + f"render response has no kwargs_data items: {features}" + ) + assert mm_metadata, "render response is missing mm_metadata" + for modality, hashes in features["mm_hashes"].items(): + assert modality in mm_metadata, f"mm_metadata missing modality {modality}" + assert len(mm_metadata[modality]) == len(hashes) + + +def run_disagg(args) -> None: + with open(args.baseline_file) as f: + baseline = json.load(f) + + failures = [] + for prompt, expected in zip(SAMPLE_PROMPTS, baseline, strict=True): + request = chat_request(prompt) | {"model": args.model_name} + got = run_disagg_one(args, request) + if got != expected: + failures.append( + f"{prompt['description']}:\n" + f" baseline: {expected!r}\n disagg: {got!r}" + ) + else: + print(f"[disagg] {prompt['description']}: matched baseline") + + if failures: + print("Output mismatch:") + for failure in failures: + print(failure) + sys.exit(1) + print("All scale-out EC outputs matched the baseline") + + +def run_disagg_one(args, request: dict) -> str: + # 1. Render: chat request -> token ids + multimodal features (GPU-less). + rendered = post_json( + f"{args.render_url}/v1/chat/completions/render", request + ).json() + token_ids = rendered["token_ids"] + features = rendered.get("features") + request_id = rendered["request_id"] + + # 2. Encode: full kwargs_data so the vision encoder can run. The + # response carries ec_transfer_params for the prefill instance. + ec_transfer_params = None + if features is not None: + _assert_mm_features(features) + encode_payload = { + "request_id": request_id, + "token_ids": token_ids, + "features": { + "mm_hashes": features["mm_hashes"], + "mm_placeholders": features["mm_placeholders"], + "kwargs_data": features["kwargs_data"], + }, + "sampling_params": {"max_tokens": 1, "temperature": 0.0}, + } + encode_response = post_json( + f"{args.encode_url}/inference/v1/generate", encode_payload + ).json() + ec_transfer_params = encode_response.get("ec_transfer_params") + assert ec_transfer_params, ( + f"encode response has no ec_transfer_params: {encode_response}" + ) + + # 3. Prefill: metadata-only features. kwargs_data is dropped; the + # embeddings arrive through the EC connector. + if features is not None: + prefill_features = { + "mm_hashes": features["mm_hashes"], + "mm_placeholders": features["mm_placeholders"], + "mm_metadata": features["mm_metadata"], + } + _check_metadata_only_rejected_without_ec(args, token_ids, prefill_features) + else: + prefill_features = None + + prefill_payload = { + "request_id": request_id, + "token_ids": token_ids, + "sampling_params": {"max_tokens": MAX_OUTPUT_LEN, "temperature": 0.0}, + } + if prefill_features is not None: + prefill_payload["features"] = prefill_features + prefill_payload["ec_transfer_params"] = ec_transfer_params + prefill_response = post_json( + f"{args.prefill_url}/inference/v1/generate", prefill_payload + ).json() + output_token_ids = prefill_response["choices"][0]["token_ids"] + assert output_token_ids, f"prefill returned no token ids: {prefill_response}" + + # 4. Derender: token ids -> chat completion response (GPU-less). + derendered = post_json( + f"{args.render_url}/v1/chat/completions/derender", + { + "model": args.model_name, + "generate_response": prefill_response, + "prompt_tokens": len(token_ids), + "chat_request": request, + }, + ).json() + return derendered["choices"][0]["message"]["content"] + + +def _check_metadata_only_rejected_without_ec( + args, token_ids: list[int], prefill_features: dict +) -> None: + """mm_metadata without kwargs_data must be rejected without EC params.""" + payload = { + "token_ids": token_ids, + "features": prefill_features, + "sampling_params": {"max_tokens": 1, "temperature": 0.0}, + } + response = requests.post( + f"{args.prefill_url}/inference/v1/generate", + json=payload, + timeout=REQUEST_TIMEOUT, + ) + assert response.status_code in (400, 422), ( + "metadata-only features without ec_transfer_params should be rejected, " + f"got {response.status_code}: {response.text}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["baseline", "disagg"], required=True) + parser.add_argument("--service_url", help="baseline single-instance URL") + parser.add_argument("--render_url", help="GPU-less render server URL") + parser.add_argument("--encode_url", help="EC producer (encode) URL") + parser.add_argument("--prefill_url", help="EC consumer (prefill) URL") + parser.add_argument("--model_name", required=True) + parser.add_argument("--baseline_file", required=True) + args = parser.parse_args() + + start = time.time() + if args.mode == "baseline": + assert args.service_url, "--service_url is required in baseline mode" + run_baseline(args) + else: + assert args.render_url, "--render_url is required in disagg mode" + assert args.encode_url, "--encode_url is required in disagg mode" + assert args.prefill_url, "--prefill_url is required in disagg mode" + run_disagg(args) + print(f"Mode {args.mode} finished in {time.time() - start:.1f}s") + + +if __name__ == "__main__": + main() diff --git a/vllm/distributed/ec_transfer/ec_connector/example_connector.py b/vllm/distributed/ec_transfer/ec_connector/example_connector.py index 98e6f1f109..2db12f903c 100644 --- a/vllm/distributed/ec_transfer/ec_connector/example_connector.py +++ b/vllm/distributed/ec_transfer/ec_connector/example_connector.py @@ -17,6 +17,7 @@ from vllm.distributed.ec_transfer.ec_connector.utils import ( collect_ec_item_metadata, ) from vllm.logger import init_logger +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.v1.core.sched.output import SchedulerOutput if TYPE_CHECKING: @@ -99,7 +100,10 @@ class ECExampleConnector(ECConnectorBase): if mm_data.mm_hash in encoder_cache: continue filename = self._generate_filename_debug(mm_data.mm_hash) - ec_cache = safetensors.torch.load_file(filename, device=device)["ec_cache"] + with gpu_sync_allowed(): + ec_cache = safetensors.torch.load_file(filename, device=device)[ + "ec_cache" + ] encoder_cache[mm_data.mm_hash] = ec_cache logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash) @@ -121,7 +125,8 @@ class ECExampleConnector(ECConnectorBase): return filename = self._generate_filename_debug(mm_hash) ec_cache = encoder_cache[mm_hash] - tensors = {"ec_cache": ec_cache.detach().cpu()} + with gpu_sync_allowed(): + tensors = {"ec_cache": ec_cache.detach().cpu()} safetensors.torch.save_file(tensors, filename) logger.debug("Save cache successful for mm_hash %s", mm_hash)