[EPD] Add ECMooncakeConnector for encoder cache over Mooncake TransferEngine (#41567)
Signed-off-by: Teng Ma <sima.mt@alibaba-inc.com> Signed-off-by: Tianyu Guo <guoty@inferact.ai> Signed-off-by: Zhou ziheng <jiaranran2@gmail.com> Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: Zhou ziheng <jiaranran2@gmail.com> Co-authored-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Cursor
Tianyu Guo
Zhou ziheng
jiangkuaixue123
OpenAI Codex
parent
f7f060d253
commit
6fbb00b188
@@ -167,6 +167,38 @@ def test_get_freed_mm_hashes_clears_freed_list():
|
||||
assert manager.get_freed_mm_hashes() == []
|
||||
|
||||
|
||||
def test_referencing_a_freeable_entry_protects_it_from_eviction():
|
||||
"""A waiting request can hold an entry it is not scheduled for yet.
|
||||
|
||||
Entries with no referent are evictable. A request that is deferred (e.g.
|
||||
by an EC connector waiting on another item) still needs the items it
|
||||
already has, and a connector that hands out one transfer per request
|
||||
cannot always produce the same item a second time.
|
||||
"""
|
||||
manager = EncoderCacheManager(cache_size=10)
|
||||
owner = MockRequest("owner", ["a"], [5])
|
||||
waiter = MockRequest("waiter", ["a"], [5])
|
||||
newcomer = MockRequest("newcomer", ["b"], [6])
|
||||
|
||||
manager.allocate(owner, 0)
|
||||
manager.free_encoder_input(owner, 0)
|
||||
assert "a" in manager.freeable
|
||||
|
||||
# The waiting request references it; it is no longer reclaimable.
|
||||
assert manager.check_and_update_cache(waiter, 0)
|
||||
assert "a" not in manager.freeable
|
||||
|
||||
# 'b' no longer fits, and 'a' must not be taken away to make room.
|
||||
assert not manager.can_allocate(newcomer, 0, int(1e9), 0)
|
||||
assert "a" in manager.cached
|
||||
assert manager.get_freed_mm_hashes() == []
|
||||
|
||||
# Once the waiter is done with it, the entry is reclaimable again.
|
||||
manager.free_encoder_input(waiter, 0)
|
||||
assert manager.can_allocate(newcomer, 0, int(1e9), 0)
|
||||
assert manager.get_freed_mm_hashes() == ["a"]
|
||||
|
||||
|
||||
def test_reallocated_hash_is_not_reported_as_freed():
|
||||
manager = EncoderCacheManager(cache_size=8)
|
||||
req_a = MockRequest("reqA", ["a"], [4])
|
||||
|
||||
@@ -5643,6 +5643,68 @@ def test_free_encoder_inputs_respects_unconfirmed_placeholders():
|
||||
assert manager.get_cached_input_ids(request) == set()
|
||||
|
||||
|
||||
def test_free_encoder_inputs_notifies_the_ec_connector():
|
||||
"""The connector learns the item is consumed, not just that the request ended.
|
||||
|
||||
Connectors hold per-item transfer state (remote buffers, reservations);
|
||||
waiting for `request_finished` pins it for the whole generation.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
mm_start_pos, mm_length = 50, 100
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=mm_start_pos + mm_length + 10,
|
||||
mm_positions=[[PlaceholderRange(offset=mm_start_pos, length=mm_length)]],
|
||||
)[0]
|
||||
scheduler.encoder_cache_manager.allocate(request, 0)
|
||||
scheduler.ec_connector = Mock()
|
||||
|
||||
request.num_computed_tokens = mm_start_pos + mm_length - 1
|
||||
scheduler._free_encoder_inputs(request)
|
||||
scheduler.ec_connector.update_state_after_free.assert_not_called()
|
||||
|
||||
request.num_computed_tokens = mm_start_pos + mm_length
|
||||
scheduler._free_encoder_inputs(request)
|
||||
scheduler.ec_connector.update_state_after_free.assert_called_once_with(request, 0)
|
||||
|
||||
|
||||
def test_unavailable_encoder_input_fails_the_request_as_retryable():
|
||||
"""An encoder input the connector gave up on must end the request.
|
||||
|
||||
`FinishReason.ERROR` is the retryable channel the KV connector already uses
|
||||
for load failures, so the caller can re-issue; deferring instead parked the
|
||||
request until the client timed out.
|
||||
"""
|
||||
scheduler = create_scheduler(model="llava-hf/llava-1.5-7b-hf")
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=160,
|
||||
mm_positions=[[PlaceholderRange(offset=50, length=100)]],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
scheduler.ec_connector = Mock()
|
||||
scheduler.ec_connector.take_unavailable_requests.return_value = {request.request_id}
|
||||
# `request_finished` reports (delay_free, params) for the request teardown.
|
||||
scheduler.ec_connector.request_finished.return_value = (False, None)
|
||||
|
||||
scheduler_output = scheduler.schedule()
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
assert request.status == RequestStatus.FINISHED_ERROR
|
||||
engine_outputs = outputs[0].outputs
|
||||
assert [o.finish_reason for o in engine_outputs] == [FinishReason.ERROR]
|
||||
|
||||
|
||||
def test_free_encoder_inputs_defers_for_eagle_lookahead():
|
||||
"""With EAGLE speculative decoding, the encoder input is retained one extra
|
||||
position so the drafter's +1 look-ahead mm-embedding gather (which reads one
|
||||
@@ -5840,9 +5902,8 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
|
||||
|
||||
# ensure_cache_available must have been called with (request, num_computed_tokens=0)
|
||||
# for a brand-new request that has no cached tokens yet.
|
||||
scheduler.ec_connector.ensure_cache_available.assert_called_once_with(
|
||||
request_deferred, 0
|
||||
)
|
||||
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
|
||||
assert ensure_call.args == (request_deferred, 0)
|
||||
# Deferred request must NOT be scheduled
|
||||
assert request_deferred.request_id not in output.num_scheduled_tokens
|
||||
_assert_right_encoder_cache_allocated(scheduler, expected_total_allocated=0)
|
||||
@@ -5874,6 +5935,31 @@ def test_ec_connector_ensure_cache_available_defers_request(use_kv_connector):
|
||||
_assert_right_encoder_inputs(output, expected_total_reqs=0)
|
||||
|
||||
|
||||
def test_ec_connector_defers_running_request_for_async_reload():
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=32,
|
||||
use_ec_connector=True,
|
||||
ec_role="ec_consumer",
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=128,
|
||||
mm_positions=[[PlaceholderRange(offset=48, length=32)]],
|
||||
req_ids=["request"],
|
||||
)[0]
|
||||
scheduler.ec_connector.ensure_cache_available = Mock(side_effect=[True, False])
|
||||
|
||||
scheduler.add_request(request)
|
||||
first_output = scheduler.schedule()
|
||||
assert first_output.num_scheduled_tokens[request.request_id] == 32
|
||||
|
||||
second_output = scheduler.schedule()
|
||||
assert request.request_id not in second_output.num_scheduled_tokens
|
||||
ensure_call = scheduler.ec_connector.ensure_cache_available.call_args
|
||||
assert ensure_call.args[:2] == (request, 32)
|
||||
|
||||
|
||||
def test_ec_connector_pending_prefetch_only_checks_future_mm_features():
|
||||
"""Test that future mm feature filtering only yields features beyond
|
||||
the computed token frontier.
|
||||
|
||||
@@ -1,171 +1,213 @@
|
||||
# EPD Correctness Test
|
||||
|
||||
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
|
||||
|
||||
## What It Tests
|
||||
|
||||
- **Baseline**: Single vLLM instance serving a multimodal model
|
||||
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
|
||||
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
|
||||
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
|
||||
|
||||
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
|
||||
|
||||
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
|
||||
|
||||
# EPD Correctness Test
|
||||
|
||||
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
|
||||
|
||||
## What It Tests
|
||||
|
||||
- **Baseline**: Single vLLM instance serving a multimodal model
|
||||
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
|
||||
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
|
||||
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
|
||||
|
||||
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
|
||||
|
||||
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
|
||||
|
||||
Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features.
|
||||
|
||||
## Files
|
||||
|
||||
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
|
||||
- `test_epd_correctness.py` - Python test script (compares outputs)
|
||||
|
||||
## Usage
|
||||
|
||||
### Multimodal Prompts (Default)
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs the test with actual multimodal (image) prompts.
|
||||
|
||||
### Text-Only Prompts
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs a quick test with text-only prompts to verify the setup works.
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
```bash
|
||||
# Use specific GPUs
|
||||
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific ports
|
||||
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific model
|
||||
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific storage path
|
||||
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### Step 1: Baseline
|
||||
|
||||
1. Start single vLLM instance on GPU
|
||||
2. Run test prompts (multimodal or text-only)
|
||||
3. Save outputs to `.vllm_epd_baseline.txt`
|
||||
4. Shutdown instance
|
||||
|
||||
### Step 2: EPD (1E + 1PD)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
### Step 3: EPD (1E + 1P + 1D)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Multimodal Prompts (--use_mm_prompts)
|
||||
|
||||
Tests encoder cache transfer:
|
||||
|
||||
- Single image query
|
||||
- Multiple images in one request
|
||||
- Mixed image and text
|
||||
- Image with detailed questions
|
||||
|
||||
### Text-Only Prompts (default)
|
||||
|
||||
Quick sanity check:
|
||||
|
||||
- Simple text queries
|
||||
- Text-only explanations
|
||||
- Verifies proxy routing works
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### ✅ Test Passes When
|
||||
|
||||
- All disagg outputs match baseline outputs exactly
|
||||
- No errors during instance startup
|
||||
- Encoder cache is properly saved and loaded
|
||||
- Proxy correctly routes requests
|
||||
|
||||
### ❌ Test Fails When
|
||||
|
||||
- Outputs differ between baseline and disagg
|
||||
- Server startup fails
|
||||
- Encoder cache not found (should fall back to local execution)
|
||||
- Proxy routing errors
|
||||
|
||||
## Notes
|
||||
|
||||
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
|
||||
- Encoder cache should enable exact output reproduction
|
||||
- Test cleans up all instances and cache files after completion
|
||||
- Safe to run multiple times (idempotent)
|
||||
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
|
||||
|
||||
## Requirements
|
||||
|
||||
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
|
||||
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
|
||||
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
|
||||
- Internet access (for accessing vllm test images)
|
||||
|
||||
## Debugging
|
||||
|
||||
### Check Logs
|
||||
|
||||
Logs and baseline output are saved in `/tmp/` by default.
|
||||
Can be customized by changing the environment variables.
|
||||
|
||||
### Check Encoder Cache
|
||||
|
||||
```bash
|
||||
# Verify cache files are created
|
||||
ls -la $EC_SHARED_STORAGE_PATH/
|
||||
|
||||
# Should see directories with mm_hash names
|
||||
# Each containing encoder_cache.safetensors
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
Run individual components:
|
||||
|
||||
```bash
|
||||
# Baseline only
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode baseline \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
|
||||
# Disagg only (requires baseline output file!)
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode disagg \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
```
|
||||
|
||||
## Files
|
||||
|
||||
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
|
||||
- `test_epd_correctness.py` - Python test script (compares outputs)
|
||||
|
||||
## Usage
|
||||
|
||||
### Multimodal Prompts (Default)
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs the test with actual multimodal (image) prompts.
|
||||
|
||||
### Text-Only Prompts
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs a quick test with text-only prompts to verify the setup works.
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
```bash
|
||||
# Use specific GPUs
|
||||
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific ports
|
||||
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific model
|
||||
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific storage path
|
||||
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### Mooncake EC (1E + 1PD)
|
||||
|
||||
```bash
|
||||
PYTHON_BIN="$PWD/.venv/bin/python" \
|
||||
bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
```
|
||||
|
||||
Requires two GPUs and Mooncake TransferEngine. TCP is the default transport;
|
||||
no RDMA-capable network hardware is required.
|
||||
The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA.
|
||||
The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1.
|
||||
`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport
|
||||
environment variables should be set by the caller.
|
||||
|
||||
Three black-box cases check fixed short answers and compare with the baseline:
|
||||
|
||||
- One image: read the STOP sign.
|
||||
- Two different images (including a local file): identify flowers and birds.
|
||||
- The same image twice: read both STOP signs.
|
||||
|
||||
By default, all three requests run concurrently for two rounds, exercising
|
||||
shared hashes across requests and reuse after completion. Every response is
|
||||
compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override.
|
||||
Prefix caching is disabled and CUDA graphs remain enabled. This is a small
|
||||
correctness suite, not a performance benchmark or failure-injection suite.
|
||||
|
||||
`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output.
|
||||
`SKIP_BASELINE=1` reuses a reference generated with the same model and test
|
||||
configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer.
|
||||
|
||||
Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs.
|
||||
The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and
|
||||
selected for changes to EC, multimodal processing, scheduler/model-runner
|
||||
integration, the proxy, or these tests (subject to normal PR CI approval).
|
||||
It reuses the CI image's Python packages through a system-site-packages venv
|
||||
and installs the CUDA-compatible Mooncake wheel. It uses loopback networking
|
||||
and TCP only, without RDMA devices or peer-memory setup.
|
||||
|
||||
The job fails on startup errors, request errors, or answer mismatches. On
|
||||
failure, the script prints the last 100 lines of each server/proxy log to
|
||||
the CI job log; full files remain under `LOG_PATH` while the container exists.
|
||||
|
||||
### Step 1: Baseline
|
||||
|
||||
1. Start single vLLM instance on GPU
|
||||
2. Run test prompts (multimodal or text-only)
|
||||
3. Save outputs to `.vllm_epd_baseline.txt`
|
||||
4. Shutdown instance
|
||||
|
||||
### Step 2: EPD (1E + 1PD)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
### Step 3: EPD (1E + 1P + 1D)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Multimodal Prompts (--use_mm_prompts)
|
||||
|
||||
Tests encoder cache transfer:
|
||||
|
||||
- Single image query
|
||||
- Multiple images in one request
|
||||
- Mixed image and text
|
||||
- Image with detailed questions
|
||||
|
||||
### Text-Only Prompts (default)
|
||||
|
||||
Quick sanity check:
|
||||
|
||||
- Simple text queries
|
||||
- Text-only explanations
|
||||
- Verifies proxy routing works
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### ✅ Test Passes When
|
||||
|
||||
- All disagg outputs match baseline outputs exactly
|
||||
- No errors during instance startup
|
||||
- Encoder cache is properly saved and loaded
|
||||
- Proxy correctly routes requests
|
||||
|
||||
### ❌ Test Fails When
|
||||
|
||||
- Outputs differ between baseline and disagg
|
||||
- Server startup fails
|
||||
- Encoder cache not found (should fall back to local execution)
|
||||
- Proxy routing errors
|
||||
|
||||
## Notes
|
||||
|
||||
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
|
||||
- Encoder cache should enable exact output reproduction
|
||||
- Test cleans up all instances and cache files after completion
|
||||
- Safe to run multiple times (idempotent)
|
||||
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
|
||||
|
||||
## Requirements
|
||||
|
||||
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
|
||||
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
|
||||
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
|
||||
- Internet access (for accessing vllm test images)
|
||||
|
||||
## Debugging
|
||||
|
||||
### Check Logs
|
||||
|
||||
Logs and baseline output are saved in `/tmp/` by default.
|
||||
Can be customized by changing the environment variables.
|
||||
|
||||
### Check Encoder Cache
|
||||
|
||||
```bash
|
||||
# Verify cache files are created
|
||||
ls -la $EC_SHARED_STORAGE_PATH/
|
||||
|
||||
# Should see directories with mm_hash names
|
||||
# Each containing encoder_cache.safetensors
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
Run individual components:
|
||||
|
||||
```bash
|
||||
# Baseline only
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode baseline \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
|
||||
# Disagg only (requires baseline output file!)
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode disagg \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
```
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine):
|
||||
# 1) Single-GPU baseline (multimodal) -> saves baseline JSON
|
||||
# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
#
|
||||
# Env:
|
||||
# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct)
|
||||
# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1)
|
||||
# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT
|
||||
# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1)
|
||||
# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp)
|
||||
# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity
|
||||
# TIMEOUT_SECONDS wait_for_server timeout (default 1200)
|
||||
# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE
|
||||
# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)
|
||||
# MAX_MODEL_LEN context length (default 16384)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
|
||||
cd "$GIT_ROOT" || exit 1
|
||||
export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}"
|
||||
|
||||
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
|
||||
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
|
||||
TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}")
|
||||
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
|
||||
TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)
|
||||
fi
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
|
||||
|
||||
GPU_SINGLE="${GPU_SINGLE:-0}"
|
||||
GPU_E="${GPU_E:-0}"
|
||||
GPU_PD="${GPU_PD:-1}"
|
||||
|
||||
ENCODE_PORT="${ENCODE_PORT:-19534}"
|
||||
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
|
||||
ENDPOINT_PORT="${ENDPOINT_PORT:-10002}"
|
||||
BASELINE_PORT="${BASELINE_PORT:-10003}"
|
||||
|
||||
EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}"
|
||||
EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}"
|
||||
MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}"
|
||||
export EC_MOONCAKE_RESERVATION_HOST
|
||||
export EC_MOONCAKE_RESERVATION_PORT
|
||||
export MOONCAKE_EC_PROTOCOL
|
||||
if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then
|
||||
# TransferEngine may otherwise auto-select RDMA on hosts with an HCA.
|
||||
export MC_FORCE_TCP=1
|
||||
else
|
||||
unset MC_FORCE_TCP
|
||||
fi
|
||||
|
||||
LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}"
|
||||
PIDS=()
|
||||
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve)
|
||||
|
||||
ENC_EC_JSON=$("$PYTHON_BIN" <<PY
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
|
||||
},
|
||||
}, separators=(",", ":")))
|
||||
PY
|
||||
)
|
||||
|
||||
PD_EC_JSON=$("$PYTHON_BIN" <<PY
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_ip": os.environ["EC_MOONCAKE_RESERVATION_HOST"],
|
||||
"ec_port": int(os.environ.get("EC_MOONCAKE_RESERVATION_PORT", "19019")),
|
||||
"ec_connector_extra_config": {
|
||||
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
|
||||
},
|
||||
}, separators=(",", ":")))
|
||||
PY
|
||||
)
|
||||
|
||||
EC_MOONCAKE_RESERVATION_ADDR=$("$PYTHON_BIN" <<'PY'
|
||||
import os
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
print(make_zmq_path(
|
||||
"tcp", os.environ["EC_MOONCAKE_RESERVATION_HOST"],
|
||||
int(os.environ["EC_MOONCAKE_RESERVATION_PORT"]),
|
||||
))
|
||||
PY
|
||||
)
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local pid=$2
|
||||
local deadline=$((SECONDS + TIMEOUT_SECONDS))
|
||||
while ((SECONDS < deadline)); do
|
||||
kill -0 "$pid" 2>/dev/null || return 1
|
||||
if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
if ((${#PIDS[@]} == 0)); then
|
||||
return
|
||||
fi
|
||||
echo "Cleaning up tracked vLLM / proxy processes..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
for _ in {1..10}; do
|
||||
local alive=0
|
||||
for pid in "${PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
alive=1
|
||||
fi
|
||||
done
|
||||
((alive == 0)) && break
|
||||
sleep 1
|
||||
done
|
||||
for pid in "${PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
wait "$pid" 2>/dev/null || true
|
||||
done
|
||||
PIDS=()
|
||||
}
|
||||
|
||||
finish() {
|
||||
local status=$?
|
||||
cleanup_instances
|
||||
if ((status != 0)); then
|
||||
for log in "${LOG_PATH}"/mooncake_epd_*.log; do
|
||||
[[ -f "$log" ]] || continue
|
||||
echo "=== $log (last 100 lines) ==="
|
||||
tail -100 "$log"
|
||||
done
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
run_baseline() {
|
||||
echo "================================"
|
||||
echo "BASELINE (single vLLM, MM if enabled)"
|
||||
echo "================================"
|
||||
cleanup_instances
|
||||
local PORT=$BASELINE_PORT
|
||||
echo "Starting baseline on GPU $GPU_SINGLE port $PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--no-enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
>"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 &
|
||||
local BASELINE_PID=$!
|
||||
PIDS+=("$BASELINE_PID")
|
||||
echo "Waiting for baseline..."
|
||||
wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; }
|
||||
curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true
|
||||
echo ""
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode baseline \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_ARGS[@]}"
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
run_epd_mooncake() {
|
||||
echo "================================"
|
||||
echo "EPD 1E + 1PD with ECMooncakeConnector"
|
||||
echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT"
|
||||
echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL"
|
||||
echo "================================"
|
||||
cleanup_instances
|
||||
|
||||
echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--gpu-memory-utilization 0.35 \
|
||||
--mm-tensor-ipc torch_shm \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens "$MAX_MODEL_LEN" \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
--ec-transfer-config "$ENC_EC_JSON" \
|
||||
>"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &
|
||||
local ENCODER_PID=$!
|
||||
PIDS+=("$ENCODER_PID")
|
||||
|
||||
echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$PREFILL_DECODE_PORT" \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--mm-tensor-ipc torch_shm \
|
||||
--enable-mm-embeds \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--no-enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
--ec-transfer-config "$PD_EC_JSON" \
|
||||
>"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 &
|
||||
local PD_PID=$!
|
||||
PIDS+=("$PD_PID")
|
||||
|
||||
echo "Waiting for encoder..."
|
||||
wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; }
|
||||
echo "Waiting for PD..."
|
||||
wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; }
|
||||
|
||||
echo "Starting EPD proxy on $ENDPOINT_PORT"
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port "$ENDPOINT_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "disable" \
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
|
||||
--ec-consumer-zmq-addrs \
|
||||
"$EC_MOONCAKE_RESERVATION_ADDR" \
|
||||
>"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 &
|
||||
local PROXY_PID=$!
|
||||
PIDS+=("$PROXY_PID")
|
||||
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; }
|
||||
curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true
|
||||
echo ""
|
||||
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$ENDPOINT_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode disagg \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_ARGS[@]}"
|
||||
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
echo "================================"
|
||||
echo "1E + 1PD ECMooncake end-to-end correctness"
|
||||
echo "MODEL=$MODEL"
|
||||
echo "================================"
|
||||
|
||||
if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then
|
||||
run_baseline
|
||||
else
|
||||
echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE"
|
||||
[[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; }
|
||||
fi
|
||||
|
||||
run_epd_mooncake
|
||||
|
||||
echo "================================"
|
||||
echo "PASS: Mooncake EC EPD matches baseline"
|
||||
echo "Logs: ${LOG_PATH}/mooncake_epd_*.log"
|
||||
echo "================================"
|
||||
@@ -26,6 +26,7 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import openai
|
||||
import requests
|
||||
@@ -138,17 +139,20 @@ def run_chat_completion(
|
||||
Returns:
|
||||
Generated text content
|
||||
"""
|
||||
client = openai.OpenAI(api_key="EMPTY", base_url=base_url)
|
||||
with openai.OpenAI(
|
||||
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
|
||||
) as client:
|
||||
completion = client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
completion = client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
return completion.choices[0].message.content
|
||||
content = completion.choices[0].message.content
|
||||
assert content, "Expected a nonempty completion"
|
||||
return content
|
||||
|
||||
|
||||
def main():
|
||||
@@ -198,7 +202,19 @@ def main():
|
||||
help="Skip the two-image multimodal prompt",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--mm_smoke_test",
|
||||
action="store_true",
|
||||
help="Use three short, fixed-answer image cases, including duplicate images",
|
||||
)
|
||||
parser.add_argument("--concurrency", type=int, default=1)
|
||||
parser.add_argument("--repeat", type=int, default=1)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.concurrency < 1 or args.repeat < 1:
|
||||
parser.error("--concurrency and --repeat must be positive")
|
||||
if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt):
|
||||
parser.error("--mm_smoke_test requires all multimodal prompts")
|
||||
|
||||
print(f"Service URL: {args.service_url}")
|
||||
print(f"Model: {args.model_name}")
|
||||
@@ -235,27 +251,76 @@ def main():
|
||||
test_prompts = SAMPLE_PROMPTS_TEXT
|
||||
print("Using text-only prompts for quick testing")
|
||||
|
||||
if args.mm_smoke_test:
|
||||
image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0]
|
||||
image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2]
|
||||
cases = [
|
||||
(
|
||||
"Single image",
|
||||
[image],
|
||||
"What word is on the red road sign? Reply with only that word "
|
||||
"in uppercase.",
|
||||
"STOP",
|
||||
),
|
||||
(
|
||||
"Two different images",
|
||||
image_pair,
|
||||
"Do these pictures contain both flowers and birds? "
|
||||
"Reply with only YES or NO.",
|
||||
"YES",
|
||||
),
|
||||
(
|
||||
"Same image twice",
|
||||
[image, image],
|
||||
"Read the red road sign in each image, in image order. Reply "
|
||||
"with only the two uppercase words separated by a comma and a space.",
|
||||
"STOP, STOP",
|
||||
),
|
||||
]
|
||||
test_prompts = [
|
||||
{
|
||||
"description": description,
|
||||
"expected": expected,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [*images, {"type": "text", "text": question}],
|
||||
}
|
||||
],
|
||||
}
|
||||
for description, images, question, expected in cases
|
||||
]
|
||||
|
||||
# Run completions
|
||||
service_url = f"{args.service_url}/v1"
|
||||
output_strs = {}
|
||||
|
||||
for i, prompt_data in enumerate(test_prompts):
|
||||
print(
|
||||
f"\nRunning prompt {i + 1}/{len(test_prompts)}: "
|
||||
f"{prompt_data['description']}"
|
||||
)
|
||||
|
||||
output_str = run_chat_completion(
|
||||
def complete(prompt_data):
|
||||
output = run_chat_completion(
|
||||
base_url=service_url,
|
||||
model_name=args.model_name,
|
||||
messages=prompt_data["messages"],
|
||||
max_tokens=MAX_OUTPUT_LEN,
|
||||
max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN,
|
||||
)
|
||||
if args.mm_smoke_test:
|
||||
output = output.strip()
|
||||
assert output == prompt_data["expected"], (
|
||||
f"{prompt_data['description']}: expected {prompt_data['expected']!r}, "
|
||||
f"got {output!r}"
|
||||
)
|
||||
return output
|
||||
|
||||
# Use description as key for comparison
|
||||
key = prompt_data["description"]
|
||||
output_strs[key] = output_str
|
||||
print(f"Output: {output_str}")
|
||||
# Each round includes concurrent requests sharing image hashes; later rounds
|
||||
# exercise reuse after previous requests have finished.
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
for repeat in range(args.repeat):
|
||||
outputs = executor.map(complete, test_prompts)
|
||||
for prompt_data, output_str in zip(test_prompts, outputs):
|
||||
key = prompt_data["description"]
|
||||
if args.repeat > 1:
|
||||
key = f"{key} (round {repeat + 1})"
|
||||
output_strs[key] = output_str
|
||||
print(f"{key}: {output_str}")
|
||||
|
||||
if args.mode in ("baseline", "baseline_pd"):
|
||||
# Baseline mode: Save outputs
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Body handling across the EPD proxy's decode retries.
|
||||
|
||||
Exercises the REAL helpers loaded from the ``examples/`` proxy, so a future
|
||||
change to them is what these tests catch.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy():
|
||||
path = Path(__file__).parents[4] / PROXY_REL
|
||||
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, params):
|
||||
self._params = params
|
||||
|
||||
async def json(self):
|
||||
return {"kv_transfer_params": self._params}
|
||||
|
||||
|
||||
def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch):
|
||||
"""A decode retry re-enters this function with the body it was given.
|
||||
|
||||
Mutating that body in place let one attempt's `remote_block_ids` survive
|
||||
into the next, so a retry whose prefill returns nothing sent decode blocks
|
||||
the prefiller may already have freed.
|
||||
"""
|
||||
served = [{"remote_block_ids": [1, 2]}, {}]
|
||||
|
||||
async def _stage(req_data, p_url, req_id):
|
||||
assert "kv_transfer_params" not in req_data
|
||||
return _Response(served.pop(0))
|
||||
|
||||
monkeypatch.setattr(proxy, "process_prefill_stage", _stage)
|
||||
|
||||
body = {"messages": [], "stream": False}
|
||||
first = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
|
||||
assert first["kv_transfer_params"] == {"remote_block_ids": [1, 2]}
|
||||
assert "kv_transfer_params" not in body
|
||||
|
||||
second = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
|
||||
assert "kv_transfer_params" not in second
|
||||
|
||||
|
||||
class _EncoderResponse:
|
||||
def __init__(self, params):
|
||||
self.status = 200
|
||||
self._params = params
|
||||
|
||||
async def json(self):
|
||||
return {"ec_transfer_params": self._params}
|
||||
|
||||
async def text(self):
|
||||
return ""
|
||||
|
||||
|
||||
class _EncoderSession:
|
||||
"""Serve one canned encoder reply per attempt."""
|
||||
|
||||
def __init__(self, replies):
|
||||
self._replies = list(replies)
|
||||
|
||||
async def post(self, url, json=None, headers=None):
|
||||
return _EncoderResponse(self._replies.pop(0))
|
||||
|
||||
|
||||
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
|
||||
"""The retry loop re-enters `prepare_for_decode` with the same body.
|
||||
|
||||
Recording the encoder's connector handles on that body in place let
|
||||
attempt 1's handle survive into attempt 2, so a second encode that
|
||||
reported nothing still sent decode a handle on an embedding the encoder
|
||||
no longer publishes -- the exact state the retry exists to leave behind.
|
||||
"""
|
||||
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}, "peer_port": 1234}
|
||||
monkeypatch.setattr(
|
||||
proxy,
|
||||
"encode_session",
|
||||
_EncoderSession([{"encoder-side-hash": handle}, {}]),
|
||||
)
|
||||
|
||||
async def _no_prefill(req_data, p_url, req_id):
|
||||
return req_data
|
||||
|
||||
monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill)
|
||||
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
],
|
||||
"stream": False,
|
||||
}
|
||||
args = ("r1", ["http://encoder"], "http://prefill", None)
|
||||
|
||||
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
reported = first["ec_transfer_params"]
|
||||
assert [handle] == [value for key, value in reported.items() if key != "ec_items"]
|
||||
assert "ec_transfer_params" not in body
|
||||
|
||||
# Attempt 2's encode reports nothing: decode must be told nothing.
|
||||
second, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
assert "ec_transfer_params" not in second
|
||||
|
||||
|
||||
def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
|
||||
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}}
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", True)
|
||||
monkeypatch.setattr(
|
||||
proxy, "encode_session", _EncoderSession([{"encoded-hash": handle}])
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
]
|
||||
}
|
||||
prepared, _, _ = asyncio.run(
|
||||
proxy.prepare_for_decode(
|
||||
body, "request", ["http://encoder"], "", "tcp://consumer:1"
|
||||
)
|
||||
)
|
||||
assert prepared["messages"] == body["messages"]
|
||||
assert "ec_transfer_params" not in body
|
||||
params = prepared["ec_transfer_params"]
|
||||
assert params["encoded-hash"] == handle
|
||||
assert params["ec_items"][0]["mm_hash"] == "encoded-hash"
|
||||
assert params["ec_items"][0]["transfer_id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("server_keep_alive", ["0.1", "1", "5", "2", "30"])
|
||||
def test_pooled_connections_are_retired_before_the_server_closes_them(
|
||||
proxy, monkeypatch, server_keep_alive
|
||||
):
|
||||
"""The proxy must not hand a request a connection the server has dropped.
|
||||
|
||||
vLLM closes idle keep-alive connections at `VLLM_HTTP_TIMEOUT_KEEP_ALIVE`
|
||||
seconds while aiohttp pools them for 15, so a slow hop leaves a dead
|
||||
connection in the pool. The next request fails with
|
||||
ServerDisconnectedError and the server logs nothing, because it closed the
|
||||
socket before the request arrived.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", server_keep_alive)
|
||||
monkeypatch.setattr(proxy.app.state, "p_urls", [], raising=False)
|
||||
|
||||
asyncio.run(proxy.on_startup())
|
||||
try:
|
||||
# No public accessor for the pool's idle timeout.
|
||||
pooled_for = proxy.encode_session.connector._keepalive_timeout
|
||||
assert pooled_for < float(server_keep_alive)
|
||||
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
|
||||
finally:
|
||||
asyncio.run(proxy.on_shutdown())
|
||||
@@ -32,8 +32,13 @@ def _load_proxy_module():
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def assign():
|
||||
return _load_proxy_module().encoder_rr_assignment
|
||||
def proxy():
|
||||
return _load_proxy_module()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def assign(proxy):
|
||||
return proxy.encoder_rr_assignment
|
||||
|
||||
|
||||
def _drive(assign, e_urls, counts):
|
||||
@@ -80,3 +85,44 @@ def test_single_encoder_always_resolves_to_it(assign):
|
||||
urls, next_cursor = assign(["E0"], 0, 4)
|
||||
assert urls == ["E0"] * 4
|
||||
assert next_cursor == 0
|
||||
|
||||
|
||||
def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy):
|
||||
proxy.validate_ec_consumer_routing(["http://prefill"], [])
|
||||
|
||||
|
||||
def test_mooncake_e_pd_routing_is_supported(proxy):
|
||||
proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"])
|
||||
|
||||
|
||||
def test_mooncake_independent_prefill_routing_fails_fast(proxy):
|
||||
with pytest.raises(ValueError, match=r"supports E\+PD only"):
|
||||
proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"])
|
||||
|
||||
|
||||
def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
|
||||
request = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
rewritten = proxy.rewrite_for_decode(
|
||||
request,
|
||||
{
|
||||
0: {
|
||||
"mm_hash": "proxy-uuid",
|
||||
"ec_mm_hash": "engine-derived-hash",
|
||||
"transfer_id": "transfer",
|
||||
"image_grid_thw": [1, 2, 3],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
|
||||
assert rewritten["ec_transfer_params"]["ec_items"] == [
|
||||
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
|
||||
]
|
||||
|
||||
@@ -12,6 +12,9 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import (
|
||||
ECConnectorModelRunnerMixin,
|
||||
)
|
||||
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -52,6 +55,9 @@ def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
|
||||
|
||||
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
|
||||
assert saved == (["mm_new"] if is_producer else [])
|
||||
assert fake.start_save_caches.called == is_producer
|
||||
if is_producer:
|
||||
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
|
||||
assert fake.start_load_caches.called == is_consumer
|
||||
|
||||
|
||||
@@ -66,6 +72,26 @@ def test_worker_meta_is_reported_on_context_exit():
|
||||
assert fake.clear_connector_metadata.called
|
||||
|
||||
|
||||
def test_v1_producer_receives_existing_encoder_cache():
|
||||
encoder_cache = {"mm_cached": None}
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
fake.is_producer = True
|
||||
fake.is_consumer = False
|
||||
fake.get_finished.return_value = (None, None)
|
||||
|
||||
module = "vllm.v1.worker.ec_connector_model_runner_mixin"
|
||||
with (
|
||||
patch(f"{module}.has_ec_transfer", return_value=True),
|
||||
patch(f"{module}.get_ec_transfer", return_value=fake),
|
||||
ECConnectorModelRunnerMixin.maybe_get_ec_connector_output(
|
||||
_scheduler_output(), encoder_cache
|
||||
),
|
||||
):
|
||||
pass
|
||||
|
||||
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
|
||||
|
||||
|
||||
def test_no_forward_reports_without_running_the_model():
|
||||
connector, _ = _connector()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user