[EPD] Add ECMooncakeConnector for encoder cache over Mooncake TransferEngine (#41567)

Signed-off-by: Teng Ma <sima.mt@alibaba-inc.com>
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Signed-off-by: Zhou ziheng <jiaranran2@gmail.com>
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: Zhou ziheng <jiaranran2@gmail.com>
Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Teng Ma
2026-09-07 06:21:43 +00:00
committed by GitHub
co-authored by Cursor Tianyu Guo Zhou ziheng jiangkuaixue123 OpenAI Codex
parent f7f060d253
commit 6fbb00b188
30 changed files with 5774 additions and 301 deletions
@@ -167,6 +167,38 @@ def test_get_freed_mm_hashes_clears_freed_list():
assert manager.get_freed_mm_hashes() == []
def test_referencing_a_freeable_entry_protects_it_from_eviction():
"""A waiting request can hold an entry it is not scheduled for yet.
Entries with no referent are evictable. A request that is deferred (e.g.
by an EC connector waiting on another item) still needs the items it
already has, and a connector that hands out one transfer per request
cannot always produce the same item a second time.
"""
manager = EncoderCacheManager(cache_size=10)
owner = MockRequest("owner", ["a"], [5])
waiter = MockRequest("waiter", ["a"], [5])
newcomer = MockRequest("newcomer", ["b"], [6])
manager.allocate(owner, 0)
manager.free_encoder_input(owner, 0)
assert "a" in manager.freeable
# The waiting request references it; it is no longer reclaimable.
assert manager.check_and_update_cache(waiter, 0)
assert "a" not in manager.freeable
# 'b' no longer fits, and 'a' must not be taken away to make room.
assert not manager.can_allocate(newcomer, 0, int(1e9), 0)
assert "a" in manager.cached
assert manager.get_freed_mm_hashes() == []
# Once the waiter is done with it, the entry is reclaimable again.
manager.free_encoder_input(waiter, 0)
assert manager.can_allocate(newcomer, 0, int(1e9), 0)
assert manager.get_freed_mm_hashes() == ["a"]
def test_reallocated_hash_is_not_reported_as_freed():
manager = EncoderCacheManager(cache_size=8)
req_a = MockRequest("reqA", ["a"], [4])
+89 -3
View File
@@ -5643,6 +5643,68 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
assert manager.get_cached_input_ids(request) == set()
def test_free_encoder_inputs_notifies_the_ec_connector():
"""The connector learns the item is consumed, not just that the request ended.
Connectors hold per-item transfer state (remote buffers, reservations);
waiting for `request_finished` pins it for the whole generation.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
mm_start_pos, mm_length = 50, 100
request = create_requests(
num_requests=1,
num_tokens=mm_start_pos + mm_length + 10,
mm_positions=[[PlaceholderRange(offset=mm_start_pos, length=mm_length)]],
)[0]
scheduler.encoder_cache_manager.allocate(request, 0)
scheduler.ec_connector = Mock()
request.num_computed_tokens = mm_start_pos + mm_length - 1
scheduler._free_encoder_inputs(request)
scheduler.ec_connector.update_state_after_free.assert_not_called()
request.num_computed_tokens = mm_start_pos + mm_length
scheduler._free_encoder_inputs(request)
scheduler.ec_connector.update_state_after_free.assert_called_once_with(request, 0)
def test_unavailable_encoder_input_fails_the_request_as_retryable():
"""An encoder input the connector gave up on must end the request.
`FinishReason.ERROR` is the retryable channel the KV connector already uses
for load failures, so the caller can re-issue; deferring instead parked the
request until the client timed out.
"""
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
request = create_requests(
num_requests=1,
num_tokens=160,
mm_positions=[[PlaceholderRange(offset=50, length=100)]],
)[0]
scheduler.add_request(request)
scheduler.ec_connector = Mock()
scheduler.ec_connector.take_unavailable_requests.return_value = {request.request_id}
# `request_finished` reports (delay_free, params) for the request teardown.
scheduler.ec_connector.request_finished.return_value = (False, None)
scheduler_output = scheduler.schedule()
outputs = scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
assert request.status == RequestStatus.FINISHED_ERROR
engine_outputs = outputs[0].outputs
assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR]
def test_free_encoder_inputs_defers_for_eagle_lookahead():
"""With EAGLE speculative decoding, the encoder input is retained one extra
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
@@ -5840,9 +5902,8 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
# ensure_cache_available must have been called with (request, num_computed_tokens=0)
# for a brand-new request that has no cached tokens yet.
scheduler.ec_connector.ensure_cache_available.assert_called_once_with(
request_deferred, 0
)
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
assert ensure_call.args == (request_deferred, 0)
# Deferred request must NOT be scheduled
assert request_deferred.request_id not in output.num_scheduled_tokens
_assert_right_encoder_cache_allocated(scheduler, expected_total_allocated=0)
@@ -5874,6 +5935,31 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
_assert_right_encoder_inputs(output, expected_total_reqs=0)
def test_ec_connector_defers_running_request_for_async_reload():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=32,
use_ec_connector=True,
ec_role="ec_consumer",
)
request = create_requests(
num_requests=1,
num_tokens=128,
mm_positions=[[PlaceholderRange(offset=48, length=32)]],
req_ids=["request"],
)[0]
scheduler.ec_connector.ensure_cache_available = Mock(side_effect=[True, False])
scheduler.add_request(request)
first_output = scheduler.schedule()
assert first_output.num_scheduled_tokens[request.request_id] == 32
second_output = scheduler.schedule()
assert request.request_id not in second_output.num_scheduled_tokens
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
assert ensure_call.args[:2] == (request, 32)
def test_ec_connector_pending_prefetch_only_checks_future_mm_features():
"""Test that future mm feature filtering only yields features beyond
the computed token frontier.
+212 -170
View File
@@ -1,171 +1,213 @@
# EPD Correctness Test
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
## What It Tests
- **Baseline**: Single vLLM instance serving a multimodal model
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
# EPD Correctness Test
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
## What It Tests
- **Baseline**: Single vLLM instance serving a multimodal model
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features.
## Files
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
- `test_epd_correctness.py` - Python test script (compares outputs)
## Usage
### Multimodal Prompts (Default)
```bash
cd vllm
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs the test with actual multimodal (image) prompts.
### Text-Only Prompts
```bash
cd vllm
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs a quick test with text-only prompts to verify the setup works.
### Custom Configuration
```bash
# Use specific GPUs
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific ports
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific model
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific storage path
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
## How It Works
### Step 1: Baseline
1. Start single vLLM instance on GPU
2. Run test prompts (multimodal or text-only)
3. Save outputs to `.vllm_epd_baseline.txt`
4. Shutdown instance
### Step 2: EPD (1E + 1PD)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
### Step 3: EPD (1E + 1P + 1D)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
## Test Scenarios
### Multimodal Prompts (--use_mm_prompts)
Tests encoder cache transfer:
- Single image query
- Multiple images in one request
- Mixed image and text
- Image with detailed questions
### Text-Only Prompts (default)
Quick sanity check:
- Simple text queries
- Text-only explanations
- Verifies proxy routing works
## Expected Behavior
### ✅ Test Passes When
- All disagg outputs match baseline outputs exactly
- No errors during instance startup
- Encoder cache is properly saved and loaded
- Proxy correctly routes requests
### ❌ Test Fails When
- Outputs differ between baseline and disagg
- Server startup fails
- Encoder cache not found (should fall back to local execution)
- Proxy routing errors
## Notes
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
- Encoder cache should enable exact output reproduction
- Test cleans up all instances and cache files after completion
- Safe to run multiple times (idempotent)
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
## Requirements
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
- Internet access (for accessing vllm test images)
## Debugging
### Check Logs
Logs and baseline output are saved in `/tmp/` by default.
Can be customized by changing the environment variables.
### Check Encoder Cache
```bash
# Verify cache files are created
ls -la $EC_SHARED_STORAGE_PATH/
# Should see directories with mm_hash names
# Each containing encoder_cache.safetensors
```
### Manual Testing
Run individual components:
```bash
# Baseline only
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode baseline \
--baseline_file test_output.txt \
--use_mm_prompts
# Disagg only (requires baseline output file!)
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode disagg \
--baseline_file test_output.txt \
--use_mm_prompts
```
## Files
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
- `test_epd_correctness.py` - Python test script (compares outputs)
## Usage
### Multimodal Prompts (Default)
```bash
cd vllm
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs the test with actual multimodal (image) prompts.
### Text-Only Prompts
```bash
cd vllm
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
This runs a quick test with text-only prompts to verify the setup works.
### Custom Configuration
```bash
# Use specific GPUs
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific ports
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific model
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
# Use specific storage path
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
```
## How It Works
### Mooncake EC (1E + 1PD)
```bash
PYTHON_BIN="$PWD/.venv/bin/python" \
bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
```
Requires two GPUs and Mooncake TransferEngine. TCP is the default transport;
no RDMA-capable network hardware is required.
The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA.
The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1.
`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport
environment variables should be set by the caller.
Three black-box cases check fixed short answers and compare with the baseline:
- One image: read the STOP sign.
- Two different images (including a local file): identify flowers and birds.
- The same image twice: read both STOP signs.
By default, all three requests run concurrently for two rounds, exercising
shared hashes across requests and reuse after completion. Every response is
compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override.
Prefix caching is disabled and CUDA graphs remain enabled. This is a small
correctness suite, not a performance benchmark or failure-injection suite.
`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output.
`SKIP_BASELINE=1` reuses a reference generated with the same model and test
configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer.
Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs.
The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and
selected for changes to EC, multimodal processing, scheduler/model-runner
integration, the proxy, or these tests (subject to normal PR CI approval).
It reuses the CI image's Python packages through a system-site-packages venv
and installs the CUDA-compatible Mooncake wheel. It uses loopback networking
and TCP only, without RDMA devices or peer-memory setup.
The job fails on startup errors, request errors, or answer mismatches. On
failure, the script prints the last 100 lines of each server/proxy log to
the CI job log; full files remain under `LOG_PATH` while the container exists.
### Step 1: Baseline
1. Start single vLLM instance on GPU
2. Run test prompts (multimodal or text-only)
3. Save outputs to `.vllm_epd_baseline.txt`
4. Shutdown instance
### Step 2: EPD (1E + 1PD)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
### Step 3: EPD (1E + 1P + 1D)
1. Clear encoder cache storage
2. Start instances and proxy
3. Run same test prompts
4. Assert outputs match baseline exactly
5. Shutdown instances
## Test Scenarios
### Multimodal Prompts (--use_mm_prompts)
Tests encoder cache transfer:
- Single image query
- Multiple images in one request
- Mixed image and text
- Image with detailed questions
### Text-Only Prompts (default)
Quick sanity check:
- Simple text queries
- Text-only explanations
- Verifies proxy routing works
## Expected Behavior
### ✅ Test Passes When
- All disagg outputs match baseline outputs exactly
- No errors during instance startup
- Encoder cache is properly saved and loaded
- Proxy correctly routes requests
### ❌ Test Fails When
- Outputs differ between baseline and disagg
- Server startup fails
- Encoder cache not found (should fall back to local execution)
- Proxy routing errors
## Notes
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
- Encoder cache should enable exact output reproduction
- Test cleans up all instances and cache files after completion
- Safe to run multiple times (idempotent)
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
## Requirements
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
- Internet access (for accessing vllm test images)
## Debugging
### Check Logs
Logs and baseline output are saved in `/tmp/` by default.
Can be customized by changing the environment variables.
### Check Encoder Cache
```bash
# Verify cache files are created
ls -la $EC_SHARED_STORAGE_PATH/
# Should see directories with mm_hash names
# Each containing encoder_cache.safetensors
```
### Manual Testing
Run individual components:
```bash
# Baseline only
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode baseline \
--baseline_file test_output.txt \
--use_mm_prompts
# Disagg only (requires baseline output file!)
python test_epd_correctness.py \
--service_url http://localhost:8000 \
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
--mode disagg \
--baseline_file test_output.txt \
--use_mm_prompts
```
@@ -0,0 +1,289 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine):
# 1) Single-GPU baseline (multimodal) -> saves baseline JSON
# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline
#
# Usage (from repo root):
# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
#
# Env:
# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct)
# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1)
# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT
# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1)
# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp)
# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity
# TIMEOUT_SECONDS wait_for_server timeout (default 1200)
# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE
# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)
# MAX_MODEL_LEN context length (default 16384)
set -euo pipefail
GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
cd "$GIT_ROOT" || exit 1
export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}"
PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}"
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}")
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)
fi
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
GPU_SINGLE="${GPU_SINGLE:-0}"
GPU_E="${GPU_E:-0}"
GPU_PD="${GPU_PD:-1}"
ENCODE_PORT="${ENCODE_PORT:-19534}"
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
ENDPOINT_PORT="${ENDPOINT_PORT:-10002}"
BASELINE_PORT="${BASELINE_PORT:-10003}"
EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}"
EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}"
MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}"
export EC_MOONCAKE_RESERVATION_HOST
export EC_MOONCAKE_RESERVATION_PORT
export MOONCAKE_EC_PROTOCOL
if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then
# TransferEngine may otherwise auto-select RDMA on hosts with an HCA.
export MC_FORCE_TCP=1
else
unset MC_FORCE_TCP
fi
LOG_PATH="${LOG_PATH:-/tmp}"
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}"
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}"
PIDS=()
mkdir -p "$LOG_PATH"
VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve)
ENC_EC_JSON=$("$PYTHON_BIN" <<PY
import json, os
print(json.dumps({
"ec_connector": "ECMooncakeConnector",
"ec_role": "ec_producer",
"ec_connector_extra_config": {
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
},
}, separators=(",", ":")))
PY
)
PD_EC_JSON=$("$PYTHON_BIN" <<PY
import json, os
print(json.dumps({
"ec_connector": "ECMooncakeConnector",
"ec_role": "ec_consumer",
"ec_ip": os.environ["EC_MOONCAKE_RESERVATION_HOST"],
"ec_port": int(os.environ.get("EC_MOONCAKE_RESERVATION_PORT", "19019")),
"ec_connector_extra_config": {
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
},
}, separators=(",", ":")))
PY
)
EC_MOONCAKE_RESERVATION_ADDR=$("$PYTHON_BIN" <<'PY'
import os
from vllm.utils.network_utils import make_zmq_path
print(make_zmq_path(
"tcp", os.environ["EC_MOONCAKE_RESERVATION_HOST"],
int(os.environ["EC_MOONCAKE_RESERVATION_PORT"]),
))
PY
)
wait_for_server() {
local port=$1
local pid=$2
local deadline=$((SECONDS + TIMEOUT_SECONDS))
while ((SECONDS < deadline)); do
kill -0 "$pid" 2>/dev/null || return 1
if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then
return 0
fi
sleep 2
done
return 1
}
cleanup_instances() {
if ((${#PIDS[@]} == 0)); then
return
fi
echo "Cleaning up tracked vLLM / proxy processes..."
for pid in "${PIDS[@]}"; do
kill "$pid" 2>/dev/null || true
done
for _ in {1..10}; do
local alive=0
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
alive=1
fi
done
((alive == 0)) && break
sleep 1
done
for pid in "${PIDS[@]}"; do
if kill -0 "$pid" 2>/dev/null; then
kill -KILL "$pid" 2>/dev/null || true
fi
wait "$pid" 2>/dev/null || true
done
PIDS=()
}
finish() {
local status=$?
cleanup_instances
if ((status != 0)); then
for log in "${LOG_PATH}"/mooncake_epd_*.log; do
[[ -f "$log" ]] || continue
echo "=== $log (last 100 lines) ==="
tail -100 "$log"
done
fi
exit "$status"
}
trap finish EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
run_baseline() {
echo "================================"
echo "BASELINE (single vLLM, MM if enabled)"
echo "================================"
cleanup_instances
local PORT=$BASELINE_PORT
echo "Starting baseline on GPU $GPU_SINGLE port $PORT"
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$PORT" \
--gpu-memory-utilization 0.75 \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--no-enable-prefix-caching \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
>"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 &
local BASELINE_PID=$!
PIDS+=("$BASELINE_PID")
echo "Waiting for baseline..."
wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; }
curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true
echo ""
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$PORT" \
--model_name "$MODEL" \
--mode baseline \
--baseline_file "$BASELINE_FILE" \
"${TEST_ARGS[@]}"
cleanup_instances
}
run_epd_mooncake() {
echo "================================"
echo "EPD 1E + 1PD with ECMooncakeConnector"
echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT"
echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL"
echo "================================"
cleanup_instances
echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"
CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$ENCODE_PORT" \
--gpu-memory-utilization 0.35 \
--mm-tensor-ipc torch_shm \
--enable-request-id-headers \
--no-enable-prefix-caching \
--max-num-batched-tokens "$MAX_MODEL_LEN" \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
--ec-transfer-config "$ENC_EC_JSON" \
>"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &
local ENCODER_PID=$!
PIDS+=("$ENCODER_PID")
echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT"
CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \
--port "$PREFILL_DECODE_PORT" \
--gpu-memory-utilization 0.75 \
--mm-tensor-ipc torch_shm \
--enable-mm-embeds \
--enable-request-id-headers \
--max-num-seqs 32 \
--max-model-len "$MAX_MODEL_LEN" \
--no-enable-prefix-caching \
--limit-mm-per-prompt '{"image":2,"video":0}' \
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
--ec-transfer-config "$PD_EC_JSON" \
>"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 &
local PD_PID=$!
PIDS+=("$PD_PID")
echo "Waiting for encoder..."
wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; }
echo "Waiting for PD..."
wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; }
echo "Starting EPD proxy on $ENDPOINT_PORT"
"$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
--host "0.0.0.0" \
--port "$ENDPOINT_PORT" \
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
--prefill-servers-urls "disable" \
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
--ec-consumer-zmq-addrs \
"$EC_MOONCAKE_RESERVATION_ADDR" \
>"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 &
local PROXY_PID=$!
PIDS+=("$PROXY_PID")
echo "Waiting for proxy..."
wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; }
curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true
echo ""
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
--service_url "http://localhost:$ENDPOINT_PORT" \
--model_name "$MODEL" \
--mode disagg \
--baseline_file "$BASELINE_FILE" \
"${TEST_ARGS[@]}"
cleanup_instances
}
echo "================================"
echo "1E + 1PD ECMooncake end-to-end correctness"
echo "MODEL=$MODEL"
echo "================================"
if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then
run_baseline
else
echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE"
[[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; }
fi
run_epd_mooncake
echo "================================"
echo "PASS: Mooncake EC EPD matches baseline"
echo "Logs: ${LOG_PATH}/mooncake_epd_*.log"
echo "================================"
@@ -26,6 +26,7 @@ import argparse
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
import openai
import requests
@@ -138,17 +139,20 @@ def run_chat_completion(
Returns:
Generated text content
"""
client = openai.OpenAI(api_key="EMPTY", base_url=base_url)
with openai.OpenAI(
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
) as client:
completion = client.chat.completions.create(
model=model_name,
messages=messages,
max_tokens=max_tokens,
temperature=0.0,
seed=42,
)
completion = client.chat.completions.create(
model=model_name,
messages=messages,
max_tokens=max_tokens,
temperature=0.0,
seed=42,
)
return completion.choices[0].message.content
content = completion.choices[0].message.content
assert content, "Expected a nonempty completion"
return content
def main():
@@ -198,7 +202,19 @@ def main():
help="Skip the two-image multimodal prompt",
)
parser.add_argument(
"--mm_smoke_test",
action="store_true",
help="Use three short, fixed-answer image cases, including duplicate images",
)
parser.add_argument("--concurrency", type=int, default=1)
parser.add_argument("--repeat", type=int, default=1)
args = parser.parse_args()
if args.concurrency < 1 or args.repeat < 1:
parser.error("--concurrency and --repeat must be positive")
if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt):
parser.error("--mm_smoke_test requires all multimodal prompts")
print(f"Service URL: {args.service_url}")
print(f"Model: {args.model_name}")
@@ -235,27 +251,76 @@ def main():
test_prompts = SAMPLE_PROMPTS_TEXT
print("Using text-only prompts for quick testing")
if args.mm_smoke_test:
image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0]
image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2]
cases = [
(
"Single image",
[image],
"What word is on the red road sign? Reply with only that word "
"in uppercase.",
"STOP",
),
(
"Two different images",
image_pair,
"Do these pictures contain both flowers and birds? "
"Reply with only YES or NO.",
"YES",
),
(
"Same image twice",
[image, image],
"Read the red road sign in each image, in image order. Reply "
"with only the two uppercase words separated by a comma and a space.",
"STOP, STOP",
),
]
test_prompts = [
{
"description": description,
"expected": expected,
"messages": [
{
"role": "user",
"content": [*images, {"type": "text", "text": question}],
}
],
}
for description, images, question, expected in cases
]
# Run completions
service_url = f"{args.service_url}/v1"
output_strs = {}
for i, prompt_data in enumerate(test_prompts):
print(
f"\nRunning prompt {i + 1}/{len(test_prompts)}: "
f"{prompt_data['description']}"
)
output_str = run_chat_completion(
def complete(prompt_data):
output = run_chat_completion(
base_url=service_url,
model_name=args.model_name,
messages=prompt_data["messages"],
max_tokens=MAX_OUTPUT_LEN,
max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN,
)
if args.mm_smoke_test:
output = output.strip()
assert output == prompt_data["expected"], (
f"{prompt_data['description']}: expected {prompt_data['expected']!r}, "
f"got {output!r}"
)
return output
# Use description as key for comparison
key = prompt_data["description"]
output_strs[key] = output_str
print(f"Output: {output_str}")
# Each round includes concurrent requests sharing image hashes; later rounds
# exercise reuse after previous requests have finished.
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
for repeat in range(args.repeat):
outputs = executor.map(complete, test_prompts)
for prompt_data, output_str in zip(test_prompts, outputs):
key = prompt_data["description"]
if args.repeat > 1:
key = f"{key} (round {repeat + 1})"
output_strs[key] = output_str
print(f"{key}: {output_str}")
if args.mode in ("baseline", "baseline_pd"):
# Baseline mode: Save outputs
@@ -0,0 +1,172 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Body handling across the EPD proxy's decode retries.
Exercises the REAL helpers loaded from the ``examples/`` proxy, so a future
change to them is what these tests catch.
"""
import asyncio
import importlib.util
from pathlib import Path
import pytest
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
@pytest.fixture(scope="module")
def proxy():
path = Path(__file__).parents[4] / PROXY_REL
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class _Response:
def __init__(self, params):
self._params = params
async def json(self):
return {"kv_transfer_params": self._params}
def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch):
"""A decode retry re-enters this function with the body it was given.
Mutating that body in place let one attempt's `remote_block_ids` survive
into the next, so a retry whose prefill returns nothing sent decode blocks
the prefiller may already have freed.
"""
served = [{"remote_block_ids": [1, 2]}, {}]
async def _stage(req_data, p_url, req_id):
assert "kv_transfer_params" not in req_data
return _Response(served.pop(0))
monkeypatch.setattr(proxy, "process_prefill_stage", _stage)
body = {"messages": [], "stream": False}
first = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
assert first["kv_transfer_params"] == {"remote_block_ids": [1, 2]}
assert "kv_transfer_params" not in body
second = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
assert "kv_transfer_params" not in second
class _EncoderResponse:
def __init__(self, params):
self.status = 200
self._params = params
async def json(self):
return {"ec_transfer_params": self._params}
async def text(self):
return ""
class _EncoderSession:
"""Serve one canned encoder reply per attempt."""
def __init__(self, replies):
self._replies = list(replies)
async def post(self, url, json=None, headers=None):
return _EncoderResponse(self._replies.pop(0))
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
"""The retry loop re-enters `prepare_for_decode` with the same body.
Recording the encoder's connector handles on that body in place let
attempt 1's handle survive into attempt 2, so a second encode that
reported nothing still sent decode a handle on an embedding the encoder
no longer publishes -- the exact state the retry exists to leave behind.
"""
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}, "peer_port": 1234}
monkeypatch.setattr(
proxy,
"encode_session",
_EncoderSession([{"encoder-side-hash": handle}, {}]),
)
async def _no_prefill(req_data, p_url, req_id):
return req_data
monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill)
body = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
],
"stream": False,
}
args = ("r1", ["http://encoder"], "http://prefill", None)
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
reported = first["ec_transfer_params"]
assert [handle] == [value for key, value in reported.items() if key != "ec_items"]
assert "ec_transfer_params" not in body
# Attempt 2's encode reports nothing: decode must be told nothing.
second, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
assert "ec_transfer_params" not in second
def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}}
monkeypatch.setattr(proxy, "NO_REWRITE", True)
monkeypatch.setattr(
proxy, "encode_session", _EncoderSession([{"encoded-hash": handle}])
)
body = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
]
}
prepared, _, _ = asyncio.run(
proxy.prepare_for_decode(
body, "request", ["http://encoder"], "", "tcp://consumer:1"
)
)
assert prepared["messages"] == body["messages"]
assert "ec_transfer_params" not in body
params = prepared["ec_transfer_params"]
assert params["encoded-hash"] == handle
assert params["ec_items"][0]["mm_hash"] == "encoded-hash"
assert params["ec_items"][0]["transfer_id"]
@pytest.mark.parametrize("server_keep_alive", ["0.1", "1", "5", "2", "30"])
def test_pooled_connections_are_retired_before_the_server_closes_them(
proxy, monkeypatch, server_keep_alive
):
"""The proxy must not hand a request a connection the server has dropped.
vLLM closes idle keep-alive connections at `VLLM_HTTP_TIMEOUT_KEEP_ALIVE`
seconds while aiohttp pools them for 15, so a slow hop leaves a dead
connection in the pool. The next request fails with
ServerDisconnectedError and the server logs nothing, because it closed the
socket before the request arrived.
"""
monkeypatch.setenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", server_keep_alive)
monkeypatch.setattr(proxy.app.state, "p_urls", [], raising=False)
asyncio.run(proxy.on_startup())
try:
# No public accessor for the pool's idle timeout.
pooled_for = proxy.encode_session.connector._keepalive_timeout
assert pooled_for < float(server_keep_alive)
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
finally:
asyncio.run(proxy.on_shutdown())
@@ -32,8 +32,13 @@ def _load_proxy_module():
@pytest.fixture(scope="module")
def assign():
return _load_proxy_module().encoder_rr_assignment
def proxy():
return _load_proxy_module()
@pytest.fixture(scope="module")
def assign(proxy):
return proxy.encoder_rr_assignment
def _drive(assign, e_urls, counts):
@@ -80,3 +85,44 @@ def test_single_encoder_always_resolves_to_it(assign):
urls, next_cursor = assign(["E0"], 0, 4)
assert urls == ["E0"] * 4
assert next_cursor == 0
def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy):
proxy.validate_ec_consumer_routing(["http://prefill"], [])
def test_mooncake_e_pd_routing_is_supported(proxy):
proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"])
def test_mooncake_independent_prefill_routing_fails_fast(proxy):
with pytest.raises(ValueError, match=r"supports E\+PD only"):
proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"])
def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
request = {
"messages": [
{
"role": "user",
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
}
]
}
rewritten = proxy.rewrite_for_decode(
request,
{
0: {
"mm_hash": "proxy-uuid",
"ec_mm_hash": "engine-derived-hash",
"transfer_id": "transfer",
"image_grid_thw": [1, 2, 3],
}
},
)
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
assert rewritten["ec_transfer_params"]["ec_items"] == [
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
]
@@ -12,6 +12,9 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
from vllm.v1.worker.ec_connector_model_runner_mixin import (
ECConnectorModelRunnerMixin,
)
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
pytestmark = pytest.mark.cpu_test
@@ -52,6 +55,9 @@ def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
assert saved == (["mm_new"] if is_producer else [])
assert fake.start_save_caches.called == is_producer
if is_producer:
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
assert fake.start_load_caches.called == is_consumer
@@ -66,6 +72,26 @@ def test_worker_meta_is_reported_on_context_exit():
assert fake.clear_connector_metadata.called
def test_v1_producer_receives_existing_encoder_cache():
encoder_cache = {"mm_cached": None}
fake = MagicMock(spec=ECConnectorBase)
fake.is_producer = True
fake.is_consumer = False
fake.get_finished.return_value = (None, None)
module = "vllm.v1.worker.ec_connector_model_runner_mixin"
with (
patch(f"{module}.has_ec_transfer", return_value=True),
patch(f"{module}.get_ec_transfer", return_value=fake),
ECConnectorModelRunnerMixin.maybe_get_ec_connector_output(
_scheduler_output(), encoder_cache
),
):
pass
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
def test_no_forward_reports_without_running_the_model():
connector, _ = _connector()