[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders
- Delete the snapshot package, CLI command, and its launch tests - Delete CPU workers, cpu_communicator, and the hw_agnostic layer package - Delete the EC CPU connector, keeping the EC base glue intact - Delete dead composite/triton_flashinfer backends and third_party/flashmla - Delete dead video decoders, video-bound tests, and orphan utils - Remove the deep_gemm_mega_moe option and dead allreduce fusion pair Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1,111 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.model_executor.warmup import jit_warmup_cutedsl_helper
|
||||
from vllm.model_executor.warmup.jit_warmup import kernel_launcher
|
||||
from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import (
|
||||
CuTeDSLLaunchSpec,
|
||||
VllmCuTeDSLJitKernel,
|
||||
)
|
||||
|
||||
|
||||
class _TestCuTeDSLKernel(VllmCuTeDSLJitKernel["_TestCuTeDSLKernel.CompileKey"]):
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
variant: int
|
||||
|
||||
@staticmethod
|
||||
def kernel(compile_key: CompileKey) -> tuple[str, int]:
|
||||
return "entry", compile_key.variant
|
||||
|
||||
def dispatch(self, *, variant: int) -> CompileKey:
|
||||
return self.CompileKey(variant=variant)
|
||||
|
||||
def get_warmup_keys(self) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(variant=(1, 2))
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]:
|
||||
return (f"fake-{compile_key.variant}",)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
payload: str,
|
||||
*,
|
||||
variant: int,
|
||||
) -> CuTeDSLLaunchSpec[CompileKey]:
|
||||
compile_key = self.dispatch(variant=variant)
|
||||
return compile_key, (payload,)
|
||||
|
||||
|
||||
def test_cutedsl_launcher_reuses_compiled_executor(monkeypatch) -> None:
|
||||
compile_calls: list[tuple[Any, tuple[Any, ...]]] = []
|
||||
launch_calls: list[tuple[int, tuple[Any, ...]]] = []
|
||||
|
||||
def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any:
|
||||
compile_calls.append((entry, args))
|
||||
variant = entry[1]
|
||||
|
||||
def executor(*runtime_args: Any) -> tuple[int, tuple[Any, ...]]:
|
||||
launch_calls.append((variant, runtime_args))
|
||||
return variant, runtime_args
|
||||
|
||||
return executor
|
||||
|
||||
monkeypatch.setattr(
|
||||
jit_warmup_cutedsl_helper,
|
||||
"compile_cutedsl",
|
||||
compile_cutedsl,
|
||||
)
|
||||
kernel = _TestCuTeDSLKernel()
|
||||
compile_key = kernel.CompileKey(variant=1)
|
||||
|
||||
kernel.compile(compile_key)
|
||||
kernel.compile(compile_key)
|
||||
assert compile_calls == [(("entry", 1), ("fake-1",))]
|
||||
|
||||
assert kernel("runtime", variant=1) == (1, ("runtime",))
|
||||
assert kernel("other", variant=2) == (2, ("other",))
|
||||
assert compile_calls == [
|
||||
(("entry", 1), ("fake-1",)),
|
||||
(("entry", 2), ("fake-2",)),
|
||||
]
|
||||
assert launch_calls == [(1, ("runtime",)), (2, ("other",))]
|
||||
|
||||
|
||||
def test_cutedsl_launcher_runs_post_launch_finalizer(monkeypatch) -> None:
|
||||
events: list[str] = []
|
||||
|
||||
def compile_cutedsl(entry: tuple[str, int], *args: Any) -> Any:
|
||||
def executor(*runtime_args: Any) -> None:
|
||||
events.append("launch")
|
||||
|
||||
return executor
|
||||
|
||||
class FinalizedKernel(_TestCuTeDSLKernel):
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
payload: str,
|
||||
*,
|
||||
variant: int,
|
||||
) -> CuTeDSLLaunchSpec[_TestCuTeDSLKernel.CompileKey]:
|
||||
compile_key = self.dispatch(variant=variant)
|
||||
|
||||
def finalize() -> str:
|
||||
events.append("finalize")
|
||||
return "output"
|
||||
|
||||
return compile_key, (payload,), "unused", finalize
|
||||
|
||||
monkeypatch.setattr(
|
||||
jit_warmup_cutedsl_helper,
|
||||
"compile_cutedsl",
|
||||
compile_cutedsl,
|
||||
)
|
||||
|
||||
assert FinalizedKernel()("runtime", variant=1) == "output"
|
||||
assert events == ["launch", "finalize"]
|
||||
@@ -1,113 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end check of the Transformers backend's hw-agnostic layers.
|
||||
|
||||
Serves a tiny Llama through `vllm_runner` with and without
|
||||
`VLLM_USE_HW_AGNOSTIC` and checks the layer providers and logprobs. Needs a
|
||||
real engine, so it runs in the GPU job. The CPU-only unit tests for the
|
||||
resolution logic (`layers._resolve`) are in `test_layer_registry.py`.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def tiny_llama_path(tmp_path_factory):
|
||||
"""A randomly-initialized microscopic Llama saved to disk (with an ungated
|
||||
tokenizer) so vLLM can load it like any local checkpoint."""
|
||||
from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
|
||||
config = LlamaConfig(
|
||||
vocab_size=tokenizer.vocab_size,
|
||||
hidden_size=128,
|
||||
intermediate_size=256,
|
||||
num_hidden_layers=2,
|
||||
num_attention_heads=4,
|
||||
rms_norm_eps=1e-6,
|
||||
hidden_act="silu",
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
model = LlamaForCausalLM(config)
|
||||
|
||||
path = tmp_path_factory.mktemp("tiny_llama")
|
||||
model.save_pretrained(path)
|
||||
tokenizer.save_pretrained(path)
|
||||
return str(path)
|
||||
|
||||
|
||||
# Registered names of the layers the backend can
|
||||
# currently route to hw-agnostic implementations.
|
||||
_COVERED_LAYERS = ("rms_norm", "silu_and_mul")
|
||||
|
||||
|
||||
def _layer_providers(model) -> dict[str, str]:
|
||||
"""Map each covered layer type present in the model to the provider its
|
||||
implementation came from (``hw_agnostic`` or ``vllm``).
|
||||
"""
|
||||
|
||||
def provider_of(module) -> str | None:
|
||||
for cls in type(module).__mro__:
|
||||
if "hw_agnostic.layers" in cls.__module__:
|
||||
return "hw_agnostic"
|
||||
if ".model_executor.layers." in cls.__module__:
|
||||
return "vllm"
|
||||
return None
|
||||
|
||||
providers: dict[str, str] = {}
|
||||
for module in model.modules():
|
||||
name = getattr(module, "name", None)
|
||||
if (
|
||||
name in _COVERED_LAYERS
|
||||
and name not in providers
|
||||
and (prov := provider_of(module)) is not None
|
||||
):
|
||||
providers[name] = prov
|
||||
return providers
|
||||
|
||||
|
||||
def _serve(vllm_runner, model_path, prompts):
|
||||
"""Serve the model through the backend; return (layer_providers, logprobs)."""
|
||||
with vllm_runner(
|
||||
model_path,
|
||||
model_impl="transformers",
|
||||
max_model_len=64,
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.3,
|
||||
) as runner:
|
||||
assert runner.llm.llm_engine.model_config.using_transformers_backend()
|
||||
providers = runner.apply_model(_layer_providers)[0]
|
||||
outputs = runner.generate_greedy_logprobs(
|
||||
prompts, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
return providers, outputs
|
||||
|
||||
|
||||
def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path):
|
||||
"""Serving the tiny model with hw-agnostic layers matches the vLLM baseline."""
|
||||
# spawn: `layers` resolves its symbols at import time, so the worker must
|
||||
# re-import it with VLLM_USE_HW_AGNOSTIC already set (fork would inherit
|
||||
# the parent's already-resolved module).
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
# apply_model pickles the introspection function.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
prompts = ["The capital of France is", "vLLM is"]
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0")
|
||||
vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
|
||||
# Both replaceable layers present in a Llama block must be vLLM's here.
|
||||
assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"}
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1")
|
||||
hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts)
|
||||
assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"}
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=vllm_outputs,
|
||||
outputs_1_lst=hw_outputs,
|
||||
name_0="vllm",
|
||||
name_1="hw_agnostic",
|
||||
)
|
||||
@@ -1,233 +0,0 @@
|
||||
# EPD Correctness Test
|
||||
|
||||
This test verifies that EPD (Encoder-Prefill-Decode) disaggregation produces identical outputs to a baseline single instance.
|
||||
|
||||
## What It Tests
|
||||
|
||||
- **Baseline**: Single vLLM instance serving a multimodal model
|
||||
- **EPD (1E+1PD)**: 1 Encoder + 1 Prefill-Decode instance
|
||||
- **Baseline (1P+1D)**: 1 Prefill + 1 Decode instance
|
||||
- **EPD (1E+1P+1D)**: 1 Encoder + 1 Prefill + 1 Decode instance
|
||||
|
||||
The test ensures that disaggregated encoding produces **identical** outputs to the baseline.
|
||||
|
||||
Note that currently PD disaggregation set up may give slightly different results from a single instance. Therefore, we need the result from 1P+1D as the baseline for 1E+1P+1D
|
||||
|
||||
Please refer to [Disaggregated Encoder Feature](../../../../docs/features/disagg_encoder.md) for the detailed explanation for the EPD features.
|
||||
|
||||
## Files
|
||||
|
||||
- `run_epd_correctness_test.sh` - Main test script (starts all instances and runs tests)
|
||||
- `test_epd_correctness.py` - Python test script (compares outputs)
|
||||
|
||||
## Usage
|
||||
|
||||
### Multimodal Prompts (Default)
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs the test with actual multimodal (image) prompts.
|
||||
|
||||
### Text-Only Prompts
|
||||
|
||||
```bash
|
||||
cd vllm
|
||||
USE_MM_PROMPTS=0 ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
This runs a quick test with text-only prompts to verify the setup works.
|
||||
|
||||
### Custom Configuration
|
||||
|
||||
```bash
|
||||
# Use specific GPUs
|
||||
GPU_E=0 GPU_PD=1 GPU_P=1 GPU_D=2 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific ports
|
||||
ENDPOINT_PORT=10001 bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific model
|
||||
MODEL="Qwen/Qwen2.5-VL-3B-Instruct" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
|
||||
# Use specific storage path
|
||||
EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integration/run_epd_correctness_test.sh
|
||||
```
|
||||
|
||||
## How It Works
|
||||
|
||||
### NIXL EC failure isolation (1E + 1PD)
|
||||
|
||||
```bash
|
||||
PATH="$PWD/.venv/bin:$PATH" CUDA_VISIBLE_DEVICES=0,1 \
|
||||
.venv/bin/python -m pytest \
|
||||
tests/v1/ec_connector/integration/test_nixl_failure.py -v -s
|
||||
```
|
||||
|
||||
Requires two CUDA GPUs and NIXL. `MODEL` overrides the default
|
||||
`Qwen/Qwen2.5-VL-3B-Instruct`. The test starts real E/PD servers with CUDA
|
||||
graphs enabled and uses the proxy's rewrite helper. Only the failing request's
|
||||
control endpoint is replaced: a ZMQ peer returns `NACK_MISSING` and verifies
|
||||
that the consumer actually requested the encoding.
|
||||
|
||||
Metadata-only requests must return a request-level error; requests retaining
|
||||
the image must succeed through local fallback. After each case, a fresh
|
||||
metadata-only image request must succeed through the real NIXL transfer path.
|
||||
No server restart, proxy retry, cache hit, or timeout race can mask the result.
|
||||
This is a correctness test, not a throughput benchmark.
|
||||
|
||||
### Mooncake EC (1E + 1PD)
|
||||
|
||||
```bash
|
||||
PYTHON_BIN="$PWD/.venv/bin/python" \
|
||||
bash tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
```
|
||||
|
||||
Requires two GPUs and Mooncake TransferEngine. TCP is the default transport;
|
||||
no RDMA-capable network hardware is required.
|
||||
The script sets `MC_FORCE_TCP=1` in TCP mode so Mooncake cannot auto-select RDMA.
|
||||
The baseline runs first on GPU 0, followed by E on GPU 0 and PD on GPU 1.
|
||||
`MOONCAKE_EC_PROTOCOL=rdma` selects RDMA instead; host-specific transport
|
||||
environment variables should be set by the caller.
|
||||
|
||||
Three black-box cases check fixed short answers and compare with the baseline:
|
||||
|
||||
- One image: read the STOP sign.
|
||||
- Two different images (including a local file): identify flowers and birds.
|
||||
- The same image twice: read both STOP signs.
|
||||
|
||||
By default, all three requests run concurrently for two rounds, exercising
|
||||
shared hashes across requests and reuse after completion. Every response is
|
||||
compared, not just the final round. Set `CONCURRENCY` and `REPEAT` to override.
|
||||
Prefix caching is disabled and CUDA graphs remain enabled. This is a small
|
||||
correctness suite, not a performance benchmark or failure-injection suite.
|
||||
|
||||
`LOG_PATH` and `BASELINE_FILE` select the log directory and reference output.
|
||||
`SKIP_BASELINE=1` reuses a reference generated with the same model and test
|
||||
configuration. `USE_MM_PROMPTS=0` only checks text routing, not EC transfer.
|
||||
|
||||
Buildkite runs this script as `mooncake-ec-tcp-e2e-2-gpus` on two L4 GPUs.
|
||||
The job is defined in `.buildkite/test_areas/disaggregated_mooncake.yaml` and
|
||||
selected for changes to EC, multimodal processing, scheduler/model-runner
|
||||
integration, the proxy, or these tests (subject to normal PR CI approval).
|
||||
It reuses the CI image's Python packages through a system-site-packages venv
|
||||
and installs the CUDA-compatible Mooncake wheel. It uses loopback networking
|
||||
and TCP only, without RDMA devices or peer-memory setup.
|
||||
|
||||
The job fails on startup errors, request errors, or answer mismatches. On
|
||||
failure, the script prints the last 100 lines of each server/proxy log to
|
||||
the CI job log; full files remain under `LOG_PATH` while the container exists.
|
||||
|
||||
### Step 1: Baseline
|
||||
|
||||
1. Start single vLLM instance on GPU
|
||||
2. Run test prompts (multimodal or text-only)
|
||||
3. Save outputs to `.vllm_epd_baseline.txt`
|
||||
4. Shutdown instance
|
||||
|
||||
### Step 2: EPD (1E + 1PD)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
### Step 3: EPD (1E + 1P + 1D)
|
||||
|
||||
1. Clear encoder cache storage
|
||||
2. Start instances and proxy
|
||||
3. Run same test prompts
|
||||
4. Assert outputs match baseline exactly
|
||||
5. Shutdown instances
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### Multimodal Prompts (--use_mm_prompts)
|
||||
|
||||
Tests encoder cache transfer:
|
||||
|
||||
- Single image query
|
||||
- Multiple images in one request
|
||||
- Mixed image and text
|
||||
- Image with detailed questions
|
||||
|
||||
### Text-Only Prompts (default)
|
||||
|
||||
Quick sanity check:
|
||||
|
||||
- Simple text queries
|
||||
- Text-only explanations
|
||||
- Verifies proxy routing works
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### ✅ Test Passes When
|
||||
|
||||
- All disagg outputs match baseline outputs exactly
|
||||
- No errors during instance startup
|
||||
- Encoder cache is properly saved and loaded
|
||||
- Proxy correctly routes requests
|
||||
|
||||
### ❌ Test Fails When
|
||||
|
||||
- Outputs differ between baseline and disagg
|
||||
- Server startup fails
|
||||
- Encoder cache not found (should fall back to local execution)
|
||||
- Proxy routing errors
|
||||
|
||||
## Notes
|
||||
|
||||
- The test uses deterministic generation (`temperature=0.0`, `seed=42`)
|
||||
- Encoder cache should enable exact output reproduction
|
||||
- Test cleans up all instances and cache files after completion
|
||||
- Safe to run multiple times (idempotent)
|
||||
- We setup the PD disagg part with NixlConnector. Please read details about EPD in `examples/disaggregated/disaggregated_encoder/README.md`
|
||||
|
||||
## Requirements
|
||||
|
||||
- Multiple GPUs (3 for 1E+1P+1D, 2 for 1E+1PD, 1 for baseline)
|
||||
- 1E+1P+1D is runnable with 2 GPU by assign E and P on the same GPU now.
|
||||
- Multimodal model (e.g., Qwen2.5-VL-3B-Instruct)
|
||||
- Internet access (for accessing vllm test images)
|
||||
|
||||
## Debugging
|
||||
|
||||
### Check Logs
|
||||
|
||||
Logs and baseline output are saved in `/tmp/` by default.
|
||||
Can be customized by changing the environment variables.
|
||||
|
||||
### Check Encoder Cache
|
||||
|
||||
```bash
|
||||
# Verify cache files are created
|
||||
ls -la $EC_SHARED_STORAGE_PATH/
|
||||
|
||||
# Should see directories with mm_hash names
|
||||
# Each containing encoder_cache.safetensors
|
||||
```
|
||||
|
||||
### Manual Testing
|
||||
|
||||
Run individual components:
|
||||
|
||||
```bash
|
||||
# Baseline only
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode baseline \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
|
||||
# Disagg only (requires baseline output file!)
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode disagg \
|
||||
--baseline_file test_output.txt \
|
||||
--use_mm_prompts
|
||||
```
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 821 KiB |
@@ -1,505 +0,0 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# EPD (Encoder-Prefill-Decode) Correctness Test
|
||||
#
|
||||
# This script tests that EPD disaggregation produces the same outputs as baseline.
|
||||
# It runs:
|
||||
# 1. Baseline: Single vLLM instance
|
||||
# 2. EPD: 1E + 1PD setup
|
||||
# 3. Baseline for (E + P + D): 1P + 1D vLLM instances disagg
|
||||
# 4. EPD: 1E + 1P + 1D setup
|
||||
|
||||
# For GPU usage
|
||||
|
||||
# set -xe
|
||||
|
||||
# Resolve the repository root from the script location instead of `.git`.
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
|
||||
|
||||
# Model to test
|
||||
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-10240}"
|
||||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.7}"
|
||||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
|
||||
|
||||
# Set 1 to use multimodal prompts; else to use text-only
|
||||
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
|
||||
USE_TWO_IMAGE_PROMPT="${USE_TWO_IMAGE_PROMPT:-1}"
|
||||
TEST_FLAGS=()
|
||||
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
|
||||
TEST_FLAGS+=(--use_mm_prompts)
|
||||
fi
|
||||
if [[ "$USE_TWO_IMAGE_PROMPT" != "1" ]]; then
|
||||
TEST_FLAGS+=(--skip_two_image_prompt)
|
||||
fi
|
||||
|
||||
# GPU configuration
|
||||
GPU_E="${GPU_E:-0}"
|
||||
GPU_P="${GPU_P:-1}"
|
||||
GPU_D="${GPU_D:-2}"
|
||||
GPU_SINGLE="${GPU_SINGLE:-$GPU_P}"
|
||||
GPU_PD="${GPU_PD:-$GPU_P}"
|
||||
|
||||
# Device platform and affinity environment variable
|
||||
DEVICE_PLATFORM="${DEVICE_PLATFORM:-cuda}"
|
||||
if [[ -z "${DEVICE_AFFINITY_ENV:-}" ]]; then
|
||||
if [[ "${DEVICE_PLATFORM,,}" == "xpu" ]]; then
|
||||
DEVICE_AFFINITY_ENV="ZE_AFFINITY_MASK"
|
||||
else
|
||||
DEVICE_AFFINITY_ENV="CUDA_VISIBLE_DEVICES"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Port
|
||||
ENCODE_PORT="${ENCODE_PORT:-19534}"
|
||||
PREFILL_PORT="${PREFILL_PORT:-19535}"
|
||||
DECODE_PORT="${DECODE_PORT:-19536}"
|
||||
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
|
||||
ENDPOINT_PORT="${ENDPOINT_PORT:-10001}"
|
||||
|
||||
# Storage path for encoder cache
|
||||
EC_SHARED_STORAGE_PATH="${EC_SHARED_STORAGE_PATH:-/tmp/ec_cache_test}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-600}"
|
||||
|
||||
# Output file for baseline comparison and logs
|
||||
LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_baseline.txt}"
|
||||
BASELINE_PD_FILE="${BASELINE_PD_FILE:-/tmp/vllm_epd_baseline.txt}"
|
||||
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
# Trap the SIGINT signal (triggered by Ctrl+C)
|
||||
trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
|
||||
|
||||
# Wait for server to be ready
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
timeout "$TIMEOUT_SECONDS" bash -c "
|
||||
until curl -s localhost:${port}/v1/chat/completions > /dev/null; do
|
||||
sleep 1
|
||||
done" && return 0 || return 1
|
||||
}
|
||||
|
||||
# Cleanup function
|
||||
cleanup_instances() {
|
||||
echo "Cleaning up any running vLLM instances..."
|
||||
pkill -f "vllm serve" || true
|
||||
pkill -f "disagg_epd_proxy.py" || true
|
||||
sleep 2
|
||||
}
|
||||
|
||||
# Function to run baseline (single instance)
|
||||
run_baseline() {
|
||||
echo "================================"
|
||||
echo "Running BASELINE (single instance)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
local PORT=$ENDPOINT_PORT
|
||||
|
||||
# Start baseline instance
|
||||
echo "Starting baseline instance on GPU $GPU_SINGLE, port $PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_SINGLE" vllm serve "$MODEL" \
|
||||
--port "$PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
> "$LOG_PATH"/baseline.log 2>&1 &
|
||||
|
||||
local BASELINE_PID=$!
|
||||
|
||||
# Wait for baseline to start
|
||||
echo "Waiting for baseline instance to start..."
|
||||
wait_for_server "$PORT"
|
||||
|
||||
curl http://127.0.0.1:"$PORT"/v1/models
|
||||
echo ""
|
||||
|
||||
# Run test in baseline mode
|
||||
echo "Running baseline..."
|
||||
|
||||
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode baseline \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_FLAGS[@]}"
|
||||
|
||||
# Cleanup baseline
|
||||
echo "Stopping baseline instance..."
|
||||
kill $BASELINE_PID 2>/dev/null || true
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Function to run EPD with 1E + 1PD
|
||||
run_epd_1e_1pd() {
|
||||
echo "================================"
|
||||
echo "Running EPD (1E + 1PD)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
mkdir -p "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
local ENCODE_PORT=$ENCODE_PORT
|
||||
local PREFILL_DECODE_PORT=$PREFILL_DECODE_PORT
|
||||
local PROXY_PORT=$ENDPOINT_PORT
|
||||
|
||||
declare -a PIDS=()
|
||||
|
||||
# Start encoder instance
|
||||
echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.01 \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$LOG_PATH"/1e1pd_encoder.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start prefill+decode instance
|
||||
echo "Starting PD instance on GPU $GPU_PD, port $PREFILL_DECODE_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_PD" vllm serve "$MODEL" \
|
||||
--port "$PREFILL_DECODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$LOG_PATH"/1e1pd_pd.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for instances to start
|
||||
echo "Waiting for encoder instance..."
|
||||
wait_for_server "$ENCODE_PORT"
|
||||
echo "Waiting for PD instance..."
|
||||
wait_for_server "$PREFILL_DECODE_PORT"
|
||||
|
||||
# Start proxy
|
||||
echo "Starting EPD proxy on port $PROXY_PORT"
|
||||
python "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port "$PROXY_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "disable" \
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
|
||||
> "$LOG_PATH"/1e1pd_proxy.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for proxy
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server "$PROXY_PORT"
|
||||
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/v1/models
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/health
|
||||
echo ""
|
||||
|
||||
echo "All EPD (1E+1PD) services are up!"
|
||||
|
||||
# Run test in disagg mode
|
||||
echo "Running EPD (1E+1PD) correctness test..."
|
||||
|
||||
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PROXY_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode disagg \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_FLAGS[@]}"
|
||||
|
||||
# Cleanup
|
||||
echo "✓✓ 1E+1PD Correctness Test finished"
|
||||
echo "Stopping EPD (1E+1PD) instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Function to run baseline for 1E + 1P + 1D (PD disagg)
|
||||
run_baseline_1p_1d() {
|
||||
echo "================================"
|
||||
echo "Running PD BASELINE (1P + 1D)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
mkdir -p "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
local PREFILL_PORT=$PREFILL_PORT
|
||||
local DECODE_PORT=$DECODE_PORT
|
||||
local PROXY_PORT=$ENDPOINT_PORT
|
||||
|
||||
declare -a PIDS=()
|
||||
|
||||
# Start prefill instance
|
||||
echo "Starting prefill instance on GPU $GPU_P, port $PREFILL_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_P" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
|
||||
vllm serve "$MODEL" \
|
||||
--port "$PREFILL_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_producer"
|
||||
}' \
|
||||
> "$LOG_PATH"/1p1d_prefill.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start decode instance
|
||||
echo "Starting decode instance on GPU $GPU_D, port $DECODE_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_D" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
|
||||
vllm serve "$MODEL" \
|
||||
--port "$DECODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_consumer"
|
||||
}' \
|
||||
> "$LOG_PATH"/1p1d_decode.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for instances to start
|
||||
echo "Waiting for prefill instance..."
|
||||
wait_for_server "$PREFILL_PORT"
|
||||
echo "Waiting for decode instance..."
|
||||
wait_for_server "$DECODE_PORT"
|
||||
|
||||
# Start proxy
|
||||
echo "Starting EPD proxy on port $PROXY_PORT"
|
||||
python "${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port "$PROXY_PORT" \
|
||||
--prefiller-ports "$PREFILL_PORT" \
|
||||
--decoder-ports "$DECODE_PORT" \
|
||||
> "$LOG_PATH"/1p1d_proxy.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for proxy
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server "$PROXY_PORT"
|
||||
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/healthcheck
|
||||
echo ""
|
||||
|
||||
echo "All PD (1P+1D) services are up!"
|
||||
|
||||
# Run test in baseline mode
|
||||
echo "Running PD disagg baseline..."
|
||||
|
||||
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PROXY_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode baseline_pd \
|
||||
--baseline_file "$BASELINE_PD_FILE" \
|
||||
"${TEST_FLAGS[@]}"
|
||||
|
||||
# Cleanup
|
||||
echo "Stopping PD (1P+1D) instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Function to run EPD with 1E + 1P + 1D
|
||||
run_epd_1e_1p_1d() {
|
||||
echo "================================"
|
||||
echo "Running EPD (1E + 1P + 1D)"
|
||||
echo "================================"
|
||||
|
||||
cleanup_instances
|
||||
rm -rf "$EC_SHARED_STORAGE_PATH"
|
||||
mkdir -p "$EC_SHARED_STORAGE_PATH"
|
||||
|
||||
local ENCODE_PORT=$ENCODE_PORT
|
||||
local PREFILL_PORT=$PREFILL_PORT
|
||||
local DECODE_PORT=$DECODE_PORT
|
||||
local PROXY_PORT=$ENDPOINT_PORT
|
||||
|
||||
declare -a PIDS=()
|
||||
|
||||
# Start encoder instance
|
||||
echo "Starting encoder instance on GPU $GPU_E, port $ENCODE_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_E" vllm serve "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.01 \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens 114688 \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
> "$LOG_PATH"/1e1p1d_encoder.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start prefill instance
|
||||
echo "Starting prefill instance on GPU $GPU_P, port $PREFILL_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_P" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=5559 \
|
||||
vllm serve "$MODEL" \
|
||||
--port "$PREFILL_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--ec-transfer-config '{
|
||||
"ec_connector": "ECExampleConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_connector_extra_config": {
|
||||
"shared_storage_path": "'"$EC_SHARED_STORAGE_PATH"'"
|
||||
}
|
||||
}' \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_producer"
|
||||
}' \
|
||||
> "$LOG_PATH"/1e1p1d_prefill.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Start decode instance
|
||||
echo "Starting decode instance on GPU $GPU_D, port $DECODE_PORT"
|
||||
env "$DEVICE_AFFINITY_ENV=$GPU_D" \
|
||||
VLLM_NIXL_SIDE_CHANNEL_PORT=6000 \
|
||||
vllm serve "$MODEL" \
|
||||
--port "$DECODE_PORT" \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--enforce-eager \
|
||||
--enable-mm-embeds \
|
||||
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs "$MAX_NUM_SEQS" \
|
||||
--allowed-local-media-path "${GIT_ROOT}"/tests/v1/ec_connector/integration \
|
||||
--kv-transfer-config '{
|
||||
"kv_connector": "NixlConnector",
|
||||
"kv_role": "kv_consumer"
|
||||
}' \
|
||||
> "$LOG_PATH"/1e1p1d_decode.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for instances to start
|
||||
echo "Waiting for encoder instance..."
|
||||
wait_for_server "$ENCODE_PORT"
|
||||
echo "Waiting for prefill instance..."
|
||||
wait_for_server "$PREFILL_PORT"
|
||||
echo "Waiting for decode instance..."
|
||||
wait_for_server "$DECODE_PORT"
|
||||
|
||||
# Start proxy
|
||||
echo "Starting EPD proxy on port $PROXY_PORT"
|
||||
python "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port "$PROXY_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "http://localhost:$PREFILL_PORT" \
|
||||
--decode-servers-urls "http://localhost:$DECODE_PORT" \
|
||||
> "$LOG_PATH"/1e1p1d_proxy.log 2>&1 &
|
||||
PIDS+=($!)
|
||||
|
||||
# Wait for proxy
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server "$PROXY_PORT"
|
||||
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/v1/models
|
||||
curl http://127.0.0.1:"$PROXY_PORT"/health
|
||||
echo ""
|
||||
|
||||
echo "All EPD (1E+1P+1D) services are up!"
|
||||
|
||||
# Run test in disagg mode
|
||||
echo "Running EPD (1E+1P+1D) correctness test..."
|
||||
|
||||
python "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PROXY_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode disagg \
|
||||
--baseline_file "$BASELINE_PD_FILE" \
|
||||
"${TEST_FLAGS[@]}"
|
||||
|
||||
# Cleanup
|
||||
echo "✓✓ 1E+1P+1D Correctness Test finished"
|
||||
echo "Stopping EPD (1E+1P+1D) instances..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
sleep 2
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
# Main execution
|
||||
echo "================================"
|
||||
echo "EPD Correctness Test Suite"
|
||||
echo "Model: $MODEL"
|
||||
echo "================================"
|
||||
|
||||
# Step 1: Run baseline
|
||||
run_baseline
|
||||
|
||||
# Step 2: Test 1E + 1PD
|
||||
run_epd_1e_1pd
|
||||
|
||||
# Step 3: Test baseline 1P + 1D
|
||||
run_baseline_1p_1d
|
||||
|
||||
# # Step 4: Test 1E + 1P + 1D
|
||||
run_epd_1e_1p_1d
|
||||
|
||||
# Cleanup output file
|
||||
rm -f "$BASELINE_FILE"
|
||||
rm -f "$BASELINE_PD_FILE"
|
||||
|
||||
echo "================================"
|
||||
echo "✓✓ All EPD correctness tests finished!"
|
||||
echo "================================"
|
||||
@@ -1,323 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Full-stack EPD validation with ECMooncakeConnector (Mooncake TransferEngine):
|
||||
# 1) Single-GPU baseline (multimodal) -> saves baseline JSON
|
||||
# 2) 1 Encoder + 1 PD with Mooncake EC -> compare outputs to baseline
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# ./tests/v1/ec_connector/integration/run_epd_mooncake_ec_full_pipeline.sh
|
||||
#
|
||||
# Env:
|
||||
# MODEL HF model id (default: Qwen/Qwen2.5-VL-3B-Instruct)
|
||||
# GPU_SINGLE / GPU_E / GPU_PD GPU ids (defaults 0 / 0 / 1)
|
||||
# ENDPOINT_PORT, ENCODE_PORT, PREFILL_DECODE_PORT
|
||||
# EC_MOONCAKE_RESERVATION_HOST consumer control host (default 127.0.0.1)
|
||||
# MOONCAKE_EC_PROTOCOL tcp | rdma (default tcp)
|
||||
# USE_MM_PROMPTS 1 (default) or 0 for text-only quick sanity
|
||||
# TIMEOUT_SECONDS wait_for_server timeout (default 1200)
|
||||
# SKIP_BASELINE set to 1 to reuse existing BASELINE_FILE
|
||||
# CONCURRENCY / REPEAT concurrent requests / rounds (defaults 3 / 2)
|
||||
# MAX_MODEL_LEN context length (default 16384)
|
||||
# E_CUDAGRAPH_MM_ENCODER enable encoder CUDA graphs (default 0)
|
||||
# E_MM_ENCODER_ATTN_BACKEND optional encoder attention backend override
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GIT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)
|
||||
cd "$GIT_ROOT" || exit 1
|
||||
export PYTHONPATH="${GIT_ROOT}:${PYTHONPATH:-}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-${GIT_ROOT}/.venv/bin/python}"
|
||||
|
||||
MODEL="${MODEL:-Qwen/Qwen2.5-VL-3B-Instruct}"
|
||||
USE_MM_PROMPTS="${USE_MM_PROMPTS:-1}"
|
||||
TEST_ARGS=(--concurrency "${CONCURRENCY:-3}" --repeat "${REPEAT:-2}")
|
||||
if [[ "$USE_MM_PROMPTS" == "1" ]]; then
|
||||
TEST_ARGS+=(--use_mm_prompts --mm_smoke_test)
|
||||
fi
|
||||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-16384}"
|
||||
E_ATTN_ARGS=()
|
||||
if [[ -n "${E_MM_ENCODER_ATTN_BACKEND:-}" ]]; then
|
||||
E_ATTN_ARGS+=(--mm-encoder-attn-backend "$E_MM_ENCODER_ATTN_BACKEND")
|
||||
fi
|
||||
case "${E_CUDAGRAPH_MM_ENCODER:-0}" in
|
||||
0) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":false}' ;;
|
||||
1) E_COMPILATION_CONFIG='{"cudagraph_mm_encoder":true}' ;;
|
||||
*) echo "E_CUDAGRAPH_MM_ENCODER must be 0 or 1"; exit 1 ;;
|
||||
esac
|
||||
|
||||
GPU_SINGLE="${GPU_SINGLE:-0}"
|
||||
GPU_E="${GPU_E:-0}"
|
||||
GPU_PD="${GPU_PD:-1}"
|
||||
|
||||
ENCODE_PORT="${ENCODE_PORT:-19534}"
|
||||
PREFILL_DECODE_PORT="${PREFILL_DECODE_PORT:-19537}"
|
||||
ENDPOINT_PORT="${ENDPOINT_PORT:-10002}"
|
||||
BASELINE_PORT="${BASELINE_PORT:-10003}"
|
||||
|
||||
EC_MOONCAKE_RESERVATION_HOST="${EC_MOONCAKE_RESERVATION_HOST:-127.0.0.1}"
|
||||
EC_MOONCAKE_RESERVATION_PORT="${EC_MOONCAKE_RESERVATION_PORT:-19019}"
|
||||
MOONCAKE_EC_PROTOCOL="${MOONCAKE_EC_PROTOCOL:-tcp}"
|
||||
export EC_MOONCAKE_RESERVATION_HOST
|
||||
export EC_MOONCAKE_RESERVATION_PORT
|
||||
export MOONCAKE_EC_PROTOCOL
|
||||
if [[ "$MOONCAKE_EC_PROTOCOL" == "tcp" ]]; then
|
||||
# TransferEngine may otherwise auto-select RDMA on hosts with an HCA.
|
||||
export MC_FORCE_TCP=1
|
||||
else
|
||||
unset MC_FORCE_TCP
|
||||
fi
|
||||
|
||||
LOG_PATH="${LOG_PATH:-/tmp}"
|
||||
BASELINE_FILE="${BASELINE_FILE:-/tmp/vllm_epd_mooncake_baseline.txt}"
|
||||
TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-1200}"
|
||||
PIDS=()
|
||||
|
||||
mkdir -p "$LOG_PATH"
|
||||
|
||||
VLLM_SERVE=("$PYTHON_BIN" -m vllm.entrypoints.cli.main serve)
|
||||
|
||||
ENC_EC_JSON=$("$PYTHON_BIN" <<PY
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
"ec_role": "ec_producer",
|
||||
"ec_connector_extra_config": {
|
||||
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
|
||||
},
|
||||
}, separators=(",", ":")))
|
||||
PY
|
||||
)
|
||||
|
||||
PD_EC_JSON=$("$PYTHON_BIN" <<PY
|
||||
import json, os
|
||||
print(json.dumps({
|
||||
"ec_connector": "ECMooncakeConnector",
|
||||
"ec_role": "ec_consumer",
|
||||
"ec_ip": os.environ["EC_MOONCAKE_RESERVATION_HOST"],
|
||||
"ec_port": int(os.environ.get("EC_MOONCAKE_RESERVATION_PORT", "19019")),
|
||||
"ec_connector_extra_config": {
|
||||
"mooncake_protocol": os.environ.get("MOONCAKE_EC_PROTOCOL", "tcp"),
|
||||
},
|
||||
}, separators=(",", ":")))
|
||||
PY
|
||||
)
|
||||
|
||||
EC_MOONCAKE_RESERVATION_ADDR=$("$PYTHON_BIN" <<'PY'
|
||||
import os
|
||||
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
|
||||
print(make_zmq_path(
|
||||
"tcp", os.environ["EC_MOONCAKE_RESERVATION_HOST"],
|
||||
int(os.environ["EC_MOONCAKE_RESERVATION_PORT"]),
|
||||
))
|
||||
PY
|
||||
)
|
||||
|
||||
wait_for_server() {
|
||||
local port=$1
|
||||
local pid=$2
|
||||
local deadline=$((SECONDS + TIMEOUT_SECONDS))
|
||||
while ((SECONDS < deadline)); do
|
||||
kill -0 "$pid" 2>/dev/null || return 1
|
||||
if curl --max-time 2 -fsS "http://localhost:${port}/health" >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
cleanup_instances() {
|
||||
if ((${#PIDS[@]} == 0)); then
|
||||
return
|
||||
fi
|
||||
echo "Cleaning up tracked vLLM / proxy processes..."
|
||||
for pid in "${PIDS[@]}"; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
for _ in {1..10}; do
|
||||
local alive=0
|
||||
for pid in "${PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
alive=1
|
||||
fi
|
||||
done
|
||||
((alive == 0)) && break
|
||||
sleep 1
|
||||
done
|
||||
for pid in "${PIDS[@]}"; do
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -KILL "$pid" 2>/dev/null || true
|
||||
fi
|
||||
wait "$pid" 2>/dev/null || true
|
||||
done
|
||||
PIDS=()
|
||||
}
|
||||
|
||||
finish() {
|
||||
local status=$?
|
||||
cleanup_instances
|
||||
if ((status != 0)); then
|
||||
for log in "${LOG_PATH}"/mooncake_epd_*.log; do
|
||||
[[ -f "$log" ]] || continue
|
||||
echo "=== $log (last 100 lines) ==="
|
||||
tail -100 "$log"
|
||||
done
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
|
||||
trap finish EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
run_baseline() {
|
||||
echo "================================"
|
||||
echo "BASELINE (single vLLM, MM if enabled)"
|
||||
echo "================================"
|
||||
cleanup_instances
|
||||
local PORT=$BASELINE_PORT
|
||||
echo "Starting baseline on GPU $GPU_SINGLE port $PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_SINGLE" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--no-enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
>"${LOG_PATH}/mooncake_epd_baseline.log" 2>&1 &
|
||||
local BASELINE_PID=$!
|
||||
PIDS+=("$BASELINE_PID")
|
||||
echo "Waiting for baseline..."
|
||||
wait_for_server "$PORT" "$BASELINE_PID" || { echo "Baseline failed to start"; return 1; }
|
||||
curl -s "http://127.0.0.1:${PORT}/v1/models" | head -c 200 || true
|
||||
echo ""
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode baseline \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_ARGS[@]}"
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
run_epd_mooncake() {
|
||||
echo "================================"
|
||||
echo "EPD 1E + 1PD with ECMooncakeConnector"
|
||||
echo "Reservation port on consumer: $EC_MOONCAKE_RESERVATION_PORT"
|
||||
echo "Mooncake protocol: $MOONCAKE_EC_PROTOCOL"
|
||||
echo "================================"
|
||||
cleanup_instances
|
||||
|
||||
echo "Starting ENCODER on GPU $GPU_E port $ENCODE_PORT"
|
||||
VLLM_SERVER_DEV_MODE=1 CUDA_VISIBLE_DEVICES="$GPU_E" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$ENCODE_PORT" \
|
||||
--gpu-memory-utilization 0.35 \
|
||||
--mm-tensor-ipc torch_shm \
|
||||
--enable-request-id-headers \
|
||||
--no-enable-prefix-caching \
|
||||
--max-num-batched-tokens "$MAX_MODEL_LEN" \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
--ec-transfer-config "$ENC_EC_JSON" \
|
||||
--compilation-config "$E_COMPILATION_CONFIG" \
|
||||
"${E_ATTN_ARGS[@]}" \
|
||||
--worker-extension-cls tests.v1.ec_connector.integration.test_epd_correctness.EncoderGraphTestWorkerExtension \
|
||||
>"${LOG_PATH}/mooncake_epd_encoder.log" 2>&1 &
|
||||
local ENCODER_PID=$!
|
||||
PIDS+=("$ENCODER_PID")
|
||||
|
||||
echo "Starting PD on GPU $GPU_PD port $PREFILL_DECODE_PORT"
|
||||
CUDA_VISIBLE_DEVICES="$GPU_PD" "${VLLM_SERVE[@]}" "$MODEL" \
|
||||
--port "$PREFILL_DECODE_PORT" \
|
||||
--gpu-memory-utilization 0.75 \
|
||||
--mm-tensor-ipc torch_shm \
|
||||
--enable-mm-embeds \
|
||||
--enable-request-id-headers \
|
||||
--max-num-seqs 32 \
|
||||
--max-model-len "$MAX_MODEL_LEN" \
|
||||
--no-enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":2,"video":0}' \
|
||||
--allowed-local-media-path "${GIT_ROOT}/tests/v1/ec_connector/integration" \
|
||||
--ec-transfer-config "$PD_EC_JSON" \
|
||||
>"${LOG_PATH}/mooncake_epd_pd.log" 2>&1 &
|
||||
local PD_PID=$!
|
||||
PIDS+=("$PD_PID")
|
||||
|
||||
echo "Waiting for encoder..."
|
||||
wait_for_server "$ENCODE_PORT" "$ENCODER_PID" || { echo "Encoder failed to start"; return 1; }
|
||||
echo "Waiting for PD..."
|
||||
wait_for_server "$PREFILL_DECODE_PORT" "$PD_PID" || { echo "PD failed to start"; return 1; }
|
||||
|
||||
echo "Starting EPD proxy on $ENDPOINT_PORT"
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py" \
|
||||
--host "0.0.0.0" \
|
||||
--port "$ENDPOINT_PORT" \
|
||||
--encode-servers-urls "http://localhost:$ENCODE_PORT" \
|
||||
--prefill-servers-urls "disable" \
|
||||
--decode-servers-urls "http://localhost:$PREFILL_DECODE_PORT" \
|
||||
--ec-consumer-zmq-addrs \
|
||||
"$EC_MOONCAKE_RESERVATION_ADDR" \
|
||||
>"${LOG_PATH}/mooncake_epd_proxy.log" 2>&1 &
|
||||
local PROXY_PID=$!
|
||||
PIDS+=("$PROXY_PID")
|
||||
|
||||
echo "Waiting for proxy..."
|
||||
wait_for_server "$ENDPOINT_PORT" "$PROXY_PID" || { echo "Proxy failed to start"; return 1; }
|
||||
curl -s "http://127.0.0.1:${ENDPOINT_PORT}/health" || true
|
||||
echo ""
|
||||
|
||||
"$PYTHON_BIN" "${GIT_ROOT}/tests/v1/ec_connector/integration/test_epd_correctness.py" \
|
||||
--service_url "http://localhost:$ENDPOINT_PORT" \
|
||||
--model_name "$MODEL" \
|
||||
--mode disagg \
|
||||
--baseline_file "$BASELINE_FILE" \
|
||||
"${TEST_ARGS[@]}"
|
||||
|
||||
curl -fsS "http://localhost:${ENCODE_PORT}/collective_rpc" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"method":"encoder_graph_stats"}' \
|
||||
>"${LOG_PATH}/encoder_graph_stats.json"
|
||||
"$PYTHON_BIN" - "${LOG_PATH}/encoder_graph_stats.json" "${E_CUDAGRAPH_MM_ENCODER:-0}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1]) as f:
|
||||
results = json.load(f)["results"]
|
||||
assert results, "No encoder workers returned graph statistics"
|
||||
for stats in results:
|
||||
if sys.argv[2] == "1":
|
||||
assert stats["is_captured"], stats
|
||||
assert stats["graph_hits"] > 0, stats
|
||||
else:
|
||||
assert stats["graph_hits"] == 0, stats
|
||||
print("Encoder graph statistics:", results)
|
||||
PY
|
||||
|
||||
cleanup_instances
|
||||
}
|
||||
|
||||
echo "================================"
|
||||
echo "1E + 1PD ECMooncake end-to-end correctness"
|
||||
echo "MODEL=$MODEL"
|
||||
echo "================================"
|
||||
|
||||
if [[ "${SKIP_BASELINE:-0}" != "1" ]]; then
|
||||
run_baseline
|
||||
else
|
||||
echo "SKIP_BASELINE=1 -> using existing $BASELINE_FILE"
|
||||
[[ -f "$BASELINE_FILE" ]] || { echo "Missing baseline file"; exit 1; }
|
||||
fi
|
||||
|
||||
run_epd_mooncake
|
||||
|
||||
echo "================================"
|
||||
echo "PASS: Mooncake EC EPD matches baseline"
|
||||
echo "Logs: ${LOG_PATH}/mooncake_epd_*.log"
|
||||
echo "================================"
|
||||
@@ -1,395 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EPD Correctness Test.
|
||||
|
||||
Tests that EPD (Encoder-Prefill-Decode) disaggregation produces the same
|
||||
outputs as a baseline single instance.
|
||||
|
||||
Usage:
|
||||
# Baseline mode (saves outputs):
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode baseline \
|
||||
--baseline_file .vllm_epd_baseline.txt
|
||||
|
||||
# Disagg mode (compares outputs):
|
||||
python test_epd_correctness.py \
|
||||
--service_url http://localhost:8000 \
|
||||
--model_name Qwen/Qwen2.5-VL-3B-Instruct \
|
||||
--mode disagg \
|
||||
--baseline_file .vllm_epd_baseline.txt
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import openai
|
||||
import requests
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
|
||||
|
||||
class EncoderGraphTestWorkerExtension:
|
||||
"""Expose encoder replay evidence through the existing test RPC endpoint."""
|
||||
|
||||
model_runner: "MMEncoderModelRunner"
|
||||
|
||||
def encoder_graph_stats(self):
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
|
||||
runner = self.model_runner
|
||||
assert isinstance(runner, MMEncoderModelRunner)
|
||||
manager = runner.model_state.encoder_runner.cudagraph_manager
|
||||
if manager is None:
|
||||
return {"is_captured": False, "graph_hits": 0, "graph_misses": 0}
|
||||
return {"is_captured": manager.is_captured(), **manager.get_cumulative_stats()}
|
||||
|
||||
|
||||
MAX_OUTPUT_LEN = 256
|
||||
|
||||
# Sample prompts with multimodal content
|
||||
image_1 = ImageAsset("stop_sign").pil_image.resize((1280, 720))
|
||||
image_2 = ImageAsset("cherry_blossom").pil_image.resize((1280, 720))
|
||||
|
||||
image_local_path = f"{os.path.dirname(os.path.abspath(__file__))}/hato.jpg"
|
||||
|
||||
SAMPLE_PROMPTS_MM: list[dict] = [
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": encode_image_url(image_1)},
|
||||
},
|
||||
{"type": "text", "text": "What's in this image?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
"description": "Single image query",
|
||||
},
|
||||
{
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": encode_image_url(image_2)},
|
||||
},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"file://{image_local_path}"},
|
||||
},
|
||||
{"type": "text", "text": "Describe these 2 images in detail."},
|
||||
],
|
||||
}
|
||||
],
|
||||
"description": "2 images with detailed query",
|
||||
},
|
||||
]
|
||||
|
||||
# Text-only prompts for mixed testing
|
||||
SAMPLE_PROMPTS_TEXT: list[dict] = [
|
||||
{
|
||||
"messages": [{"role": "user", "content": "What is the capital of France?"}],
|
||||
"description": "Simple text-only query",
|
||||
},
|
||||
{
|
||||
"messages": [
|
||||
{"role": "user", "content": "Explain quantum computing in simple terms."}
|
||||
],
|
||||
"description": "Text-only explanation request",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def check_vllm_server(url: str, timeout=5, retries=10) -> bool:
|
||||
"""Check if the vLLM server is ready.
|
||||
|
||||
Args:
|
||||
url: The URL to check (usually /health or /healthcheck endpoint)
|
||||
timeout: Timeout in seconds for each request
|
||||
retries: Number of retries if the server is not ready
|
||||
|
||||
Returns:
|
||||
True if the server is ready, False otherwise
|
||||
|
||||
"""
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
response = requests.get(url, timeout=timeout)
|
||||
if response.status_code == 200:
|
||||
print(f"Server is ready at {url}")
|
||||
return True
|
||||
else:
|
||||
print(
|
||||
f"Attempt {attempt + 1}/{retries}: Server returned "
|
||||
f"status code {response.status_code}"
|
||||
)
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"Attempt {attempt + 1}/{retries}: Error connecting: {e}")
|
||||
time.sleep(2) # Wait before retrying
|
||||
return False
|
||||
|
||||
|
||||
def run_chat_completion(
|
||||
base_url: str,
|
||||
model_name: str,
|
||||
messages: list,
|
||||
max_tokens: int = MAX_OUTPUT_LEN,
|
||||
) -> str:
|
||||
"""Run a chat completion request.
|
||||
|
||||
Args:
|
||||
base_url: Base URL of the vLLM server
|
||||
model_name: Name of the model
|
||||
messages: Messages for chat completion
|
||||
max_tokens: Maximum tokens to generate
|
||||
|
||||
Returns:
|
||||
Generated text content
|
||||
|
||||
"""
|
||||
with openai.OpenAI(
|
||||
api_key="EMPTY", base_url=base_url, timeout=120, max_retries=0
|
||||
) as client:
|
||||
completion = client.chat.completions.create(
|
||||
model=model_name,
|
||||
messages=messages,
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.0,
|
||||
seed=42,
|
||||
)
|
||||
|
||||
content = completion.choices[0].message.content
|
||||
assert content, "Expected a nonempty completion"
|
||||
return content
|
||||
|
||||
|
||||
def main():
|
||||
"""Main test function."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="EPD correctness test - compare disagg vs baseline"
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--service_url",
|
||||
type=str,
|
||||
required=True,
|
||||
help="The vLLM service URL (e.g., http://localhost:8000)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--model_name",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Model name",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
type=str,
|
||||
default="baseline",
|
||||
choices=["baseline", "baseline_pd", "disagg"],
|
||||
help="Mode: baseline/baseline_pd (saves outputs) or disagg (compares outputs)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--baseline_file",
|
||||
type=str,
|
||||
default=".vllm_epd_baseline.txt",
|
||||
help="File to save/load baseline outputs",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--use_mm_prompts",
|
||||
action="store_true",
|
||||
help="Use multimodal prompts (default: use text-only for quick testing)",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--skip_two_image_prompt",
|
||||
action="store_true",
|
||||
help="Skip the two-image multimodal prompt",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--mm_smoke_test",
|
||||
action="store_true",
|
||||
help="Use three short, fixed-answer image cases, including duplicate images",
|
||||
)
|
||||
parser.add_argument("--concurrency", type=int, default=1)
|
||||
parser.add_argument("--repeat", type=int, default=1)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.concurrency < 1 or args.repeat < 1:
|
||||
parser.error("--concurrency and --repeat must be positive")
|
||||
if args.mm_smoke_test and (not args.use_mm_prompts or args.skip_two_image_prompt):
|
||||
parser.error("--mm_smoke_test requires all multimodal prompts")
|
||||
|
||||
print(f"Service URL: {args.service_url}")
|
||||
print(f"Model: {args.model_name}")
|
||||
print(f"Mode: {args.mode}")
|
||||
print(f"Output file: {args.baseline_file}")
|
||||
print(f"Use MM prompts: {args.use_mm_prompts}")
|
||||
|
||||
# Determine health check endpoint
|
||||
if args.mode == "baseline":
|
||||
health_check_url = f"{args.service_url}/health"
|
||||
elif args.mode == "baseline_pd":
|
||||
# Nixl toy proxy use /healthcheck
|
||||
health_check_url = f"{args.service_url}/healthcheck"
|
||||
else:
|
||||
# Disagg EPD proxy uses /health
|
||||
health_check_url = f"{args.service_url}/health"
|
||||
if not os.path.exists(args.baseline_file):
|
||||
raise ValueError(
|
||||
f"In disagg mode, the output file {args.baseline_file} from "
|
||||
"baseline does not exist. Run baseline mode first."
|
||||
)
|
||||
|
||||
# Check if server is ready
|
||||
if not check_vllm_server(health_check_url):
|
||||
raise RuntimeError(f"vLLM server at {args.service_url} is not ready!")
|
||||
|
||||
# Select prompts to use
|
||||
if args.use_mm_prompts:
|
||||
test_prompts = (
|
||||
SAMPLE_PROMPTS_MM[:1] if args.skip_two_image_prompt else SAMPLE_PROMPTS_MM
|
||||
)
|
||||
print("Using multimodal prompts")
|
||||
else:
|
||||
test_prompts = SAMPLE_PROMPTS_TEXT
|
||||
print("Using text-only prompts for quick testing")
|
||||
|
||||
if args.mm_smoke_test:
|
||||
image = SAMPLE_PROMPTS_MM[0]["messages"][0]["content"][0]
|
||||
image_pair = SAMPLE_PROMPTS_MM[1]["messages"][0]["content"][:2]
|
||||
cases = [
|
||||
(
|
||||
"Single image",
|
||||
[image],
|
||||
"What word is on the red road sign? Reply with only that word "
|
||||
"in uppercase.",
|
||||
"STOP",
|
||||
),
|
||||
(
|
||||
"Two different images",
|
||||
image_pair,
|
||||
"Do these pictures contain both flowers and birds? "
|
||||
"Reply with only YES or NO.",
|
||||
"YES",
|
||||
),
|
||||
(
|
||||
"Same image twice",
|
||||
[image, image],
|
||||
"Read the red road sign in each image, in image order. Reply "
|
||||
"with only the two uppercase words separated by a comma and a space.",
|
||||
"STOP, STOP",
|
||||
),
|
||||
]
|
||||
test_prompts = [
|
||||
{
|
||||
"description": description,
|
||||
"expected": expected,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [*images, {"type": "text", "text": question}],
|
||||
}
|
||||
],
|
||||
}
|
||||
for description, images, question, expected in cases
|
||||
]
|
||||
|
||||
# Run completions
|
||||
service_url = f"{args.service_url}/v1"
|
||||
output_strs = {}
|
||||
|
||||
def complete(prompt_data):
|
||||
output = run_chat_completion(
|
||||
base_url=service_url,
|
||||
model_name=args.model_name,
|
||||
messages=prompt_data["messages"],
|
||||
max_tokens=16 if args.mm_smoke_test else MAX_OUTPUT_LEN,
|
||||
)
|
||||
if args.mm_smoke_test:
|
||||
output = output.strip()
|
||||
assert output == prompt_data["expected"], (
|
||||
f"{prompt_data['description']}: expected {prompt_data['expected']!r}, "
|
||||
f"got {output!r}"
|
||||
)
|
||||
return output
|
||||
|
||||
# Each round includes concurrent requests sharing image hashes; later rounds
|
||||
# exercise reuse after previous requests have finished.
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
|
||||
for repeat in range(args.repeat):
|
||||
outputs = executor.map(complete, test_prompts)
|
||||
for prompt_data, output_str in zip(test_prompts, outputs):
|
||||
key = prompt_data["description"]
|
||||
if args.repeat > 1:
|
||||
key = f"{key} (round {repeat + 1})"
|
||||
output_strs[key] = output_str
|
||||
print(f"{key}: {output_str}")
|
||||
|
||||
if args.mode in ("baseline", "baseline_pd"):
|
||||
# Baseline mode: Save outputs
|
||||
print(f"\nSaving baseline outputs to {args.baseline_file}")
|
||||
try:
|
||||
with open(args.baseline_file, "w") as json_file:
|
||||
json.dump(output_strs, json_file, indent=4)
|
||||
print("✅ Baseline outputs saved successfully")
|
||||
except OSError as e:
|
||||
print(f"Error writing to file: {e}")
|
||||
raise
|
||||
else:
|
||||
# Disagg mode: Load and compare outputs
|
||||
print(f"\nLoading baseline outputs from {args.baseline_file}")
|
||||
baseline_outputs = None
|
||||
try:
|
||||
with open(args.baseline_file) as json_file:
|
||||
baseline_outputs = json.load(json_file)
|
||||
except OSError as e:
|
||||
print(f"Error reading from file: {e}")
|
||||
raise
|
||||
|
||||
# Verify outputs match
|
||||
print("\nComparing disagg outputs with baseline...")
|
||||
assert isinstance(baseline_outputs, dict), "Baseline outputs should be a dict"
|
||||
assert len(baseline_outputs) == len(output_strs), (
|
||||
f"Length mismatch: baseline has {len(baseline_outputs)}, "
|
||||
f"disagg has {len(output_strs)}"
|
||||
)
|
||||
|
||||
all_match = True
|
||||
for key, baseline_output in baseline_outputs.items():
|
||||
assert key in output_strs, f"{key} not in disagg outputs"
|
||||
|
||||
disagg_output = output_strs[key]
|
||||
if baseline_output == disagg_output:
|
||||
print(f"✅ {key}: MATCH")
|
||||
else:
|
||||
print(f"❌ {key}: MISMATCH")
|
||||
print(f" Baseline: {baseline_output}")
|
||||
print(f" Disagg: {disagg_output}")
|
||||
all_match = False
|
||||
|
||||
assert all_match, "❌❌Disagg outputs do not match baseline!❌❌"
|
||||
if all_match:
|
||||
print("\n✅ All outputs match! Test PASSED")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,389 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for EmbeddingCache.
|
||||
|
||||
Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
|
||||
|
||||
def _cache(num_blocks: int = 8) -> EmbeddingCache:
|
||||
return EmbeddingCache(num_blocks)
|
||||
|
||||
|
||||
# ── alloc ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_alloc_returns_unique_ascending_block_ids():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 3)
|
||||
assert entry is not None
|
||||
assert len(entry.block_ids) == 3
|
||||
assert len(set(entry.block_ids)) == 3
|
||||
assert all(0 <= i < 8 for i in entry.block_ids)
|
||||
# Ascending order lets consumers detect which blocks are adjacent.
|
||||
assert list(entry.block_ids) == sorted(entry.block_ids)
|
||||
|
||||
|
||||
def test_alloc_returns_tuple():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 2)
|
||||
assert isinstance(entry.block_ids, tuple)
|
||||
|
||||
|
||||
def test_alloc_returns_none_when_full():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 4)
|
||||
assert cache.alloc("b", 1) is None
|
||||
|
||||
|
||||
def test_alloc_asserts_duplicate_key():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
with pytest.raises(AssertionError, match="duplicate alloc"):
|
||||
cache.alloc("a", 2)
|
||||
|
||||
|
||||
def test_alloc_asserts_n_blocks_exceeds_capacity():
|
||||
cache = _cache(num_blocks=4)
|
||||
with pytest.raises(AssertionError, match="capacity"):
|
||||
cache.alloc("a", 5)
|
||||
|
||||
|
||||
def test_alloc_zero_blocks():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 0)
|
||||
assert entry is not None
|
||||
assert entry.block_ids == ()
|
||||
|
||||
|
||||
def test_alloc_starts_not_ready():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 2)
|
||||
assert not entry.ready
|
||||
assert not entry.evictable
|
||||
|
||||
|
||||
# ── get ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_missing_returns_none():
|
||||
cache = _cache()
|
||||
assert cache.get("a") is None
|
||||
|
||||
|
||||
def test_get_returns_entry():
|
||||
cache = _cache()
|
||||
alloc_entry = cache.alloc("a", 2)
|
||||
got = cache.get("a")
|
||||
assert got is alloc_entry
|
||||
|
||||
|
||||
def test_get_returns_not_ready_entry():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
entry = cache.get("a")
|
||||
assert entry is not None
|
||||
assert not entry.ready
|
||||
|
||||
|
||||
# ── mark_ready ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_mark_ready_transitions():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
entry = cache.get("a")
|
||||
assert not entry.ready
|
||||
cache.mark_ready("a")
|
||||
assert entry.ready
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_mark_ready_asserts_already_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
with pytest.raises(AssertionError, match="already-ready"):
|
||||
cache.mark_ready("a")
|
||||
|
||||
|
||||
def test_mark_ready_asserts_missing_key():
|
||||
cache = _cache()
|
||||
with pytest.raises(KeyError):
|
||||
cache.mark_ready("nonexistent")
|
||||
|
||||
|
||||
# ── pin / unpin ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_asserts_not_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
with pytest.raises(AssertionError, match="not-ready"):
|
||||
cache.pin("a")
|
||||
|
||||
|
||||
def test_pin_unpin_basic():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
entry = cache.get("a")
|
||||
assert not entry.evictable
|
||||
cache.unpin("a")
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_nested_pin():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.pin("a")
|
||||
cache.unpin("a")
|
||||
entry = cache.get("a")
|
||||
assert not entry.evictable # still pinned (count=1)
|
||||
cache.unpin("a")
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_unpin_asserts_on_unpinned():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
with pytest.raises(AssertionError, match="unpinned"):
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
# ── has_held_entries ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_has_held_entries_empty_cache():
|
||||
cache = _cache()
|
||||
assert cache.has_held_entries() is False
|
||||
|
||||
|
||||
def test_has_held_entries_true_while_not_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2) # not ready → held
|
||||
assert cache.has_held_entries() is True
|
||||
cache.mark_ready("a") # ready + unpinned → settled
|
||||
assert cache.has_held_entries() is False
|
||||
|
||||
|
||||
def test_has_held_entries_true_while_pinned():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
assert cache.has_held_entries() is False
|
||||
cache.pin("a") # pinned → held
|
||||
assert cache.has_held_entries() is True
|
||||
cache.unpin("a")
|
||||
assert cache.has_held_entries() is False
|
||||
|
||||
|
||||
# ── eviction ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_eviction_frees_ready_unpinned_fifo():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# Pool full. Alloc should evict "a" (oldest).
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is None # evicted
|
||||
assert cache.get("b") is not None # still there
|
||||
|
||||
|
||||
def test_eviction_skips_pinned():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# "a" is pinned, so "b" should be evicted instead.
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is not None # pinned, survived
|
||||
assert cache.get("b") is None # evicted
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
def test_eviction_skips_not_ready():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2) # not ready
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# "a" is not ready (non-evictable), "b" is evictable.
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is not None # not ready, survived
|
||||
assert cache.get("b") is None # evicted
|
||||
cache.mark_ready("a") # clean up
|
||||
|
||||
|
||||
def test_eviction_returns_none_if_all_non_evictable():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.alloc("b", 2) # not ready
|
||||
# Both entries are non-evictable (a=pinned, b=not-ready).
|
||||
assert cache.alloc("c", 1) is None
|
||||
# Fail-fast: nothing was evicted.
|
||||
assert cache.get("a") is not None
|
||||
assert cache.get("b") is not None
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
def test_fail_fast_no_wasted_eviction():
|
||||
cache = _cache(num_blocks=6)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a") # evictable: 2 blocks
|
||||
cache.alloc("b", 2) # not ready
|
||||
cache.alloc("c", 2) # not ready
|
||||
# free=0, evictable=2, need 3 -> cannot succeed
|
||||
assert cache.alloc("d", 3) is None
|
||||
# "a" was NOT evicted despite being evictable
|
||||
assert cache.get("a") is not None
|
||||
|
||||
|
||||
def test_eviction_multiple_entries():
|
||||
cache = _cache(num_blocks=6)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
cache.alloc("c", 2)
|
||||
cache.mark_ready("c")
|
||||
# Need 4 blocks: must evict both "a" and "b".
|
||||
entry_d = cache.alloc("d", 4)
|
||||
assert entry_d is not None
|
||||
assert cache.get("a") is None
|
||||
assert cache.get("b") is None
|
||||
assert cache.get("c") is not None
|
||||
|
||||
|
||||
# ── thread safety ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_concurrent_pin_unpin():
|
||||
cache = _cache(num_blocks=8)
|
||||
cache.alloc("a", 4)
|
||||
cache.mark_ready("a")
|
||||
errors: list[Exception] = []
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
try:
|
||||
for _ in range(500):
|
||||
cache.pin("a")
|
||||
cache.unpin("a")
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(8)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
entry = cache.get("a")
|
||||
assert entry.evictable # all pins balanced
|
||||
|
||||
|
||||
def test_concurrent_alloc_evict():
|
||||
cache = _cache(num_blocks=4)
|
||||
errors: list[Exception] = []
|
||||
barrier = threading.Barrier(4)
|
||||
counter = {"i": 0}
|
||||
counter_lock = threading.Lock()
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
try:
|
||||
for _ in range(200):
|
||||
with counter_lock:
|
||||
key = f"k{counter['i']}"
|
||||
counter["i"] += 1
|
||||
entry = cache.alloc(key, 1)
|
||||
if entry is not None:
|
||||
cache.mark_ready(key)
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(4)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
|
||||
|
||||
# ── discard ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_discard_not_ready_returns_blocks():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
entry = cache.alloc("h", 3)
|
||||
assert entry is not None
|
||||
cache.discard("h")
|
||||
assert cache.get("h") is None
|
||||
# All blocks reclaimed: a fresh 4-block alloc now fits.
|
||||
assert cache.alloc("h2", 4) is not None
|
||||
|
||||
|
||||
def test_discard_rejects_ready_entry():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 1)
|
||||
cache.mark_ready("h")
|
||||
with pytest.raises(AssertionError):
|
||||
cache.discard("h")
|
||||
|
||||
|
||||
def test_discard_rejects_absent_key():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
with pytest.raises(KeyError):
|
||||
cache.discard("nope")
|
||||
|
||||
|
||||
# ── pin_if_ready ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_if_ready_pins_and_returns_entry():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 2)
|
||||
cache.mark_ready("h")
|
||||
entry = cache.pin_if_ready("h")
|
||||
assert entry is not None and entry.ready and len(entry.block_ids) == 2
|
||||
assert cache.alloc("h2", 4) is None # pinned ⇒ not evictable
|
||||
|
||||
|
||||
def test_pin_if_ready_none_when_absent():
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
assert cache.pin_if_ready("nope") is None
|
||||
|
||||
|
||||
def test_pin_if_ready_returns_unpinned_entry_when_not_ready():
|
||||
"""The producer needs "present but not ready" told apart from "absent":
|
||||
the first is NACKed as retryable, the second as a miss."""
|
||||
cache = EmbeddingCache(num_blocks=4)
|
||||
cache.alloc("h", 1) # not-ready
|
||||
entry = cache.pin_if_ready("h")
|
||||
assert entry is not None and not entry.ready
|
||||
assert cache.alloc("h2", 4) is None # not-ready ⇒ still not evictable
|
||||
@@ -1,466 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.config.ec_transfer import ECRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
|
||||
_N = 16
|
||||
_BS = 64
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, identifier=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = identifier if identifier is not None else mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
_counter = 0
|
||||
|
||||
def __init__(self, features, request_id=None):
|
||||
self.mm_features = features
|
||||
if request_id is None:
|
||||
_Request._counter += 1
|
||||
self.request_id = f"req_{_Request._counter}"
|
||||
else:
|
||||
self.request_id = request_id
|
||||
|
||||
|
||||
class _WorkerOutput:
|
||||
"""Stand-in for ECConnectorOutput carrying a completion report.
|
||||
|
||||
`saves` are mm_hashes; `loads` are transfer ids. One entry per reporting
|
||||
rank, so a list holding the same id twice is two ranks reporting it.
|
||||
"""
|
||||
|
||||
def __init__(self, *, saves=None, loads=None):
|
||||
self.ec_connector_worker_meta = ECCPUWorkerMetadata(
|
||||
completed_saves=list(saves or []),
|
||||
completed_loads=list(loads or []),
|
||||
)
|
||||
|
||||
|
||||
def _make_scheduler(
|
||||
monkeypatch,
|
||||
num_blocks=_N,
|
||||
*,
|
||||
ec_role: ECRole = "ec_both",
|
||||
tensor_parallel_size=1,
|
||||
) -> ECCPUScheduler:
|
||||
region = ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=_BS,
|
||||
)
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
return ECCPUScheduler(
|
||||
create_ec_vllm_config(
|
||||
ec_role=ec_role,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _load_ids(meta) -> list[int]:
|
||||
"""Transfer ids the scheduler dispatched in this step's metadata."""
|
||||
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
|
||||
|
||||
|
||||
def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
"""Block ids the scheduler dispatched for one load."""
|
||||
return meta.loads[mm_hash][1]
|
||||
|
||||
|
||||
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
|
||||
"""Pre-populate a ready cache entry backed by real blocks."""
|
||||
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
|
||||
s._cache.mark_ready(mm_hash)
|
||||
|
||||
|
||||
def test_offload_reuse_cycle(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
# Two blocks, so the reload also pins down that every allocated block id
|
||||
# comes back in meta.loads, not just the first.
|
||||
req = _Request([_Feature("h1", length=2)])
|
||||
|
||||
# Step A: first sight — allocate and save.
|
||||
assert s.has_cache_item("h1") is False
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_a = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_a.saves
|
||||
assert meta_a.loads == {}
|
||||
# Not ready until the worker reports the save memcpy complete.
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Worker reports the save copy finished → entry becomes ready.
|
||||
s.update_connector_output(_WorkerOutput(saves=["h1"]))
|
||||
assert s.has_cache_item("h1") is True
|
||||
|
||||
# Step C: reload now works — entry is ready at update_state_after_alloc time.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_c = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_c.loads
|
||||
assert _load_blocks(meta_c, "h1") == meta_a.saves["h1"]
|
||||
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_has_cache_item_false_when_not_consumer(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
|
||||
assert s.has_cache_item("anything") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
|
||||
"""The connector must key the encoder cache on feature.identifier (what
|
||||
has_cache_item is called with), NOT feature.mm_hash."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "ENC_KEY" in meta.saves
|
||||
assert "PROC_KEY" not in meta.saves
|
||||
# Not ready until the completion report.
|
||||
assert s.has_cache_item("ENC_KEY") is False
|
||||
# After the report it becomes ready — keyed on identifier.
|
||||
s.update_connector_output(_WorkerOutput(saves=["ENC_KEY"]))
|
||||
assert s.has_cache_item("ENC_KEY") is True
|
||||
assert s.has_cache_item("PROC_KEY") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_cpu_region_fifo_eviction(monkeypatch):
|
||||
# Region holds exactly 2 one-block encodings.
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
|
||||
# Fill with A and B, marking each ready via a completion report.
|
||||
for h in ("A", "B"):
|
||||
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
s.update_connector_output(_WorkerOutput(saves=[h]))
|
||||
assert s.has_cache_item("A") is True
|
||||
assert s.has_cache_item("B") is True
|
||||
|
||||
# Allocate C: must evict A (oldest) to make room.
|
||||
s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("A") is False # evicted
|
||||
assert s.has_cache_item("B") is True
|
||||
|
||||
|
||||
def test_multiple_mm_items_per_request(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.update_state_after_alloc(req, 1)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert set(meta.saves) == {"h1", "h2"}
|
||||
|
||||
# Worker reports both saves complete.
|
||||
s.update_connector_output(_WorkerOutput(saves=["h1", "h2"]))
|
||||
assert s.has_cache_item("h1") is True
|
||||
assert s.has_cache_item("h2") is True
|
||||
|
||||
# Now reload both.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.update_state_after_alloc(req, 1)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert set(meta2.loads) == {"h1", "h2"}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
_seed_cached(s, "b", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
assert "b" not in meta.loads
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_repeated_reload_same_step_loads_once(monkeypatch):
|
||||
"""The same mm_hash requested twice in one step must appear in
|
||||
meta.loads exactly once."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert list(meta.loads.keys()).count("a") == 1
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_load_not_emitted_for_uncached_entry(monkeypatch):
|
||||
"""A feature that was never saved must not appear in meta.loads."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
s.update_state_after_alloc(_Request([_Feature("missing")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta.loads == {}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_load_pin_protects_blocks_until_completion(monkeypatch):
|
||||
"""Loaded blocks stay pinned until the worker reports the load memcpy
|
||||
complete. Observable: a save that needs eviction cannot reclaim a
|
||||
still-pinned loaded entry, but can once the completion report arrives."""
|
||||
# 2 blocks, both occupied by "a" (ready).
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" — pins it.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
(transfer_id,) = _load_ids(meta)
|
||||
|
||||
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves # can't evict pinned "a"
|
||||
|
||||
# The one participant reports the load complete → "a" unpins. Now "b" can
|
||||
# evict it.
|
||||
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def _can_evict(s: ECCPUScheduler, mm_hash: str, n_blocks: int) -> bool:
|
||||
"""Whether a new save of `mm_hash` can claim `n_blocks`, i.e. whether the
|
||||
blocks currently held by other entries are reclaimable."""
|
||||
s.update_state_after_alloc(_Request([_Feature(mm_hash, length=n_blocks)]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
return mm_hash in meta.saves
|
||||
|
||||
|
||||
def test_load_needs_every_participant_before_unpin(monkeypatch):
|
||||
"""With two participating ranks, one report must not release the pin.
|
||||
|
||||
This is the multi-rank case: each rank copies the same CPU blocks into its
|
||||
own GPU memory, so the blocks stay in use until the last rank is done.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, tensor_parallel_size=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
(transfer_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
|
||||
|
||||
# First rank reports: still in use by the second, so "a" must hold.
|
||||
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
|
||||
assert _can_evict(s, "b", 2) is False
|
||||
assert s.has_pending_push_work() is True
|
||||
|
||||
# Second rank reports: now it is genuinely free.
|
||||
s.update_connector_output(_WorkerOutput(loads=[transfer_id]))
|
||||
assert s.has_pending_push_work() is False
|
||||
assert _can_evict(s, "c", 2) is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_late_report_does_not_release_a_later_load_of_same_hash(monkeypatch):
|
||||
"""A straggling report from one dispatch must not release the pin taken by
|
||||
a subsequent dispatch of the same mm_hash.
|
||||
|
||||
This is what the transfer id is for: mm_hash identifies a cache entry, not
|
||||
a transfer, so it cannot tell the two dispatches apart.
|
||||
"""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, tensor_parallel_size=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# First dispatch, fully reported by both ranks → pin released.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
(first_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
|
||||
s.update_connector_output(_WorkerOutput(loads=[first_id, first_id]))
|
||||
assert s.has_pending_push_work() is False
|
||||
|
||||
# Second dispatch of the same hash takes a fresh pin.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
(second_id,) = _load_ids(s.build_connector_meta(scheduler_output=None))
|
||||
assert second_id != first_id
|
||||
|
||||
# A replayed report for the first dispatch must be ignored entirely.
|
||||
s.update_connector_output(_WorkerOutput(loads=[first_id, first_id]))
|
||||
assert s.has_pending_push_work() is True
|
||||
assert _can_evict(s, "b", 2) is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
"""When the region is fully occupied by pinned entries, new saves are
|
||||
silently skipped and ensure_cache_available never blocks."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=1)
|
||||
_seed_cached(s, "pinned", n_blocks=1)
|
||||
s._cache.pin("pinned")
|
||||
|
||||
req = _Request([_Feature("new", length=1)])
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "new" not in meta.saves
|
||||
assert s.has_cache_item("new") is False
|
||||
|
||||
s._cache.unpin("pinned")
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_producer_only_never_emits_loads(monkeypatch):
|
||||
"""A producer-only scheduler must never populate meta.loads, even when
|
||||
entries are ready."""
|
||||
s = _make_scheduler(monkeypatch, ec_role="ec_producer")
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.saves
|
||||
assert meta.loads == {}
|
||||
|
||||
s.update_connector_output(_WorkerOutput(saves=["h1"]))
|
||||
|
||||
# Even if we re-encounter the same feature, no load.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta2.loads == {}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_consumer_only_never_emits_saves(monkeypatch):
|
||||
"""A consumer-only scheduler must never populate meta.saves."""
|
||||
s = _make_scheduler(monkeypatch, ec_role="ec_consumer")
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
|
||||
req = _Request([_Feature("a", length=1)])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta.saves == {}
|
||||
assert "a" in meta.loads
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_consumer_only_has_cache_item(monkeypatch):
|
||||
"""Consumer-only: has_cache_item works normally for ready entries."""
|
||||
s = _make_scheduler(monkeypatch, ec_role="ec_consumer")
|
||||
assert s.has_cache_item("a") is False
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
assert s.has_cache_item("a") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
"""An entry already in the cache (from a prior save) must not trigger
|
||||
a second allocation or appear in meta.saves again."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
# First encounter → save.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta1 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta1.saves
|
||||
|
||||
# Second encounter (same step or later) → no save.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" not in meta2.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
# ── completion reporting (update_connector_output) ──────────────────────────
|
||||
|
||||
|
||||
def test_update_connector_output_ignores_foreign_meta(monkeypatch):
|
||||
"""A worker output whose payload is not an ECCPUWorkerMetadata is a
|
||||
no-op — no cache mutation, no crash."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
|
||||
class _Foreign:
|
||||
ec_connector_worker_meta = object()
|
||||
|
||||
s.update_connector_output(_Foreign()) # must not raise
|
||||
assert s.has_cache_item("a") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_update_connector_output_ignores_unknown_report(monkeypatch):
|
||||
"""Reports for a never-seen save hash or a transfer id this scheduler never
|
||||
dispatched are dropped rather than mutating unrelated state."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
s.update_connector_output(_WorkerOutput(saves=["gone"], loads=[4242]))
|
||||
assert s.has_cache_item("gone") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
# ── has_pending_push_work ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_has_pending_push_work_tracks_inflight_save(monkeypatch):
|
||||
"""A dispatched-but-unconfirmed save keeps push work pending until the
|
||||
completion report marks it ready."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
assert s.has_pending_push_work() is False
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("h1", length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_pending_push_work() is True # save not yet confirmed
|
||||
|
||||
s.update_connector_output(_WorkerOutput(saves=["h1"]))
|
||||
assert s.has_pending_push_work() is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_has_pending_push_work_tracks_inflight_load(monkeypatch):
|
||||
"""A dispatched-but-unconfirmed load keeps push work pending until the
|
||||
unpin completion report arrives."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
assert s.has_pending_push_work() is False # ready + unpinned
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_pending_push_work() is True # pinned, awaiting unpin
|
||||
|
||||
s.update_connector_output(_WorkerOutput(loads=_load_ids(meta)))
|
||||
assert s.has_pending_push_work() is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
# ── shutdown ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_shutdown_disables_roles_and_cleans_region(monkeypatch):
|
||||
"""After shutdown the scheduler serves no items and the region is
|
||||
cleaned up."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
assert s.has_cache_item("a") is True
|
||||
|
||||
s.shutdown()
|
||||
assert s.has_cache_item("a") is False # consumer role disabled
|
||||
@@ -1,71 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
|
||||
|
||||
def _cfg():
|
||||
ec = MagicMock()
|
||||
ec.is_ec_producer = True
|
||||
ec.is_ec_consumer = True
|
||||
cfg = MagicMock()
|
||||
cfg.ec_transfer_config = ec
|
||||
return cfg
|
||||
|
||||
|
||||
def test_scheduler_role_builds_only_scheduler(monkeypatch):
|
||||
fake_sched = MagicMock()
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
assert c.connector_scheduler is fake_sched
|
||||
assert c.connector_worker is None
|
||||
c.has_cache_item("x")
|
||||
fake_sched.has_cache_item.assert_called_once_with("x")
|
||||
|
||||
|
||||
def test_worker_role_builds_only_worker(monkeypatch):
|
||||
fake_worker = MagicMock()
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER)
|
||||
assert c.connector_worker is fake_worker
|
||||
assert c.connector_scheduler is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", [ECConnectorRole.SCHEDULER, ECConnectorRole.WORKER])
|
||||
def test_v1_model_runner_is_rejected(monkeypatch, role):
|
||||
"""The V1 runner never reports save completions, so refuse to start."""
|
||||
monkeypatch.setattr(
|
||||
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
|
||||
)
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: MagicMock())
|
||||
cfg = _cfg()
|
||||
cfg.use_v2_model_runner = False
|
||||
|
||||
with pytest.raises(ValueError, match="V2 model runner"):
|
||||
ECCPUConnector(cfg, role)
|
||||
|
||||
|
||||
def test_factory_registered():
|
||||
cls = ECConnectorFactory._registry["ECCPUConnector"]()
|
||||
assert cls is ECCPUConnector
|
||||
|
||||
|
||||
def test_request_finished_forwards_to_scheduler(monkeypatch):
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.connector as conn_mod # noqa: F401
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
|
||||
|
||||
fake_sched = MagicMock()
|
||||
fake_sched.request_finished.return_value = (False, {"h1": {"peer_port": 1}})
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
req = MagicMock()
|
||||
assert c.request_finished(req) == (False, {"h1": {"peer_port": 1}})
|
||||
fake_sched.request_finished.assert_called_once_with(req)
|
||||
@@ -1,396 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECSharedRegion (the mmap substrate)."""
|
||||
|
||||
import contextlib
|
||||
import ctypes
|
||||
import errno
|
||||
import mmap
|
||||
import os
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import common
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
_wait_for_file_size,
|
||||
)
|
||||
|
||||
|
||||
def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion:
|
||||
return ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=block_size_bytes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def region() -> ECSharedRegion:
|
||||
r = _make_region()
|
||||
yield r
|
||||
r.cleanup()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _region(**kwargs):
|
||||
"""Context manager: create one region, clean up on exit."""
|
||||
r = _make_region(**kwargs)
|
||||
try:
|
||||
yield r
|
||||
finally:
|
||||
r.cleanup()
|
||||
|
||||
|
||||
def _page_residency(mmap_obj: mmap.mmap, length: int) -> list[bool]:
|
||||
"""Return Linux page-residency bits for a writable mmap."""
|
||||
try:
|
||||
libc = ctypes.CDLL(None, use_errno=True)
|
||||
mincore = libc.mincore
|
||||
except AttributeError:
|
||||
pytest.skip("mincore is unavailable")
|
||||
|
||||
page_count = (length + mmap.PAGESIZE - 1) // mmap.PAGESIZE
|
||||
mincore.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_size_t,
|
||||
ctypes.POINTER(ctypes.c_ubyte),
|
||||
]
|
||||
mincore.restype = ctypes.c_int
|
||||
vector = (ctypes.c_ubyte * page_count)()
|
||||
address = ctypes.addressof(ctypes.c_ubyte.from_buffer(mmap_obj))
|
||||
result = mincore(ctypes.c_void_p(address), length, vector)
|
||||
if result != 0:
|
||||
raise OSError(ctypes.get_errno())
|
||||
return [bool(value & 1) for value in vector]
|
||||
|
||||
|
||||
# ── mmap sharing between two instances ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
try:
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
try:
|
||||
assert r1._is_creator
|
||||
assert not r2._is_creator
|
||||
# Both map the same physical pages; writes via r1 are visible via r2.
|
||||
r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8)
|
||||
assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40]
|
||||
finally:
|
||||
r2.cleanup()
|
||||
finally:
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_dense_dp_engines_get_distinct_regions():
|
||||
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
|
||||
so the region must be keyed by data_parallel_index."""
|
||||
instance_id = str(uuid.uuid4())
|
||||
regions = []
|
||||
try:
|
||||
for dp_index in range(2):
|
||||
parallel_config = ParallelConfig(
|
||||
data_parallel_size=2, data_parallel_rank=dp_index
|
||||
)
|
||||
parallel_config.reconfigure_for_independent_dp_rank()
|
||||
cfg = MagicMock()
|
||||
cfg.instance_id = instance_id
|
||||
cfg.parallel_config = parallel_config
|
||||
cfg.model_config.dtype = torch.float16
|
||||
cfg.model_config.get_inputs_embeds_size.return_value = 8
|
||||
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
|
||||
regions.append(common.create_ec_shared_region(cfg))
|
||||
|
||||
assert all(r._is_creator for r in regions)
|
||||
assert regions[0]._mmap_path != regions[1]._mmap_path
|
||||
finally:
|
||||
for r in regions:
|
||||
r.cleanup()
|
||||
|
||||
|
||||
def test_only_creator_unlinks_file_on_cleanup():
|
||||
"""Critical contract: if the non-creator unlinks, the creator's mmap path
|
||||
becomes a dangling backing file and a third opener would create a new one
|
||||
out from under the creator."""
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
path = r1._mmap_path
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
|
||||
# Non-creator goes away first — file must still be on disk for r1.
|
||||
r2.cleanup()
|
||||
assert os.path.exists(path), "non-creator cleanup must not unlink the file"
|
||||
|
||||
# Creator goes away — file is removed.
|
||||
r1.cleanup()
|
||||
assert not os.path.exists(path), "creator cleanup must unlink the file"
|
||||
|
||||
|
||||
# ── _wait_for_file_size ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_wait_for_file_size_returns_when_already_big_enough(tmp_path):
|
||||
"""The fast path: file already at expected size — return immediately."""
|
||||
p = tmp_path / "f.bin"
|
||||
p.write_bytes(b"\x00" * 128)
|
||||
fd = os.open(str(p), os.O_RDONLY)
|
||||
try:
|
||||
_wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path):
|
||||
p = tmp_path / "f.bin"
|
||||
p.write_bytes(b"")
|
||||
fd = os.open(str(p), os.O_RDONLY)
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
_wait_for_file_size(fd, expected_size=4096, timeout=0.05)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
# ── MADV_POPULATE_WRITE pre-faulting / fallback ──────────────────────────────
|
||||
|
||||
|
||||
def test_madvise_success_selects_madvise_population(monkeypatch):
|
||||
"""A successful probe must keep using MADV_POPULATE_WRITE — the fallback
|
||||
helper must never be invoked on a kernel that accepts the advice.
|
||||
|
||||
Spies on both helpers so we can verify call counts: 1 probe call over a
|
||||
single page + 1 populate call over the whole flat region, fallback 0 times.
|
||||
"""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
madvise_calls: list[tuple[int, int, int]] = []
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
|
||||
def _spy_madvise(mm, off, ln):
|
||||
madvise_calls.append((off, ln, id(mm)))
|
||||
|
||||
def _spy_fallback(mm, off, ln):
|
||||
fallback_calls.append((off, ln))
|
||||
|
||||
monkeypatch.setattr(esr, "_madvise_populate_write", _spy_madvise)
|
||||
monkeypatch.setattr(esr, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
num_blocks, block_size_bytes = 4, mmap.PAGESIZE
|
||||
with _region(num_blocks=num_blocks, block_size_bytes=block_size_bytes):
|
||||
total = num_blocks * block_size_bytes
|
||||
mmap_id = madvise_calls[0][2]
|
||||
assert madvise_calls == [
|
||||
(0, mmap.PAGESIZE, mmap_id), # probe
|
||||
(0, total, mmap_id), # flat pre-fault of the whole region
|
||||
]
|
||||
assert fallback_calls == [], (
|
||||
"native-path constructor must not invoke the fallback helper"
|
||||
)
|
||||
|
||||
|
||||
def test_madvise_einval_selects_fallback_for_whole_region(monkeypatch):
|
||||
"""An EINVAL probe must select the fallback for the whole flat region.
|
||||
|
||||
This is the regression under test: on Linux < 5.14 the constant is absent
|
||||
from the `mmap` module, so advice=23 reaches a kernel that rejects it and
|
||||
every ECSharedRegion construction used to abort with EINVAL.
|
||||
"""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
fallback_calls: list[tuple[int, int]] = []
|
||||
real_fallback = esr._fallback_populate_write
|
||||
|
||||
def _raise_einval(mm, off, ln):
|
||||
raise OSError(errno.EINVAL, "simulated unsupported kernel")
|
||||
|
||||
def _spy_fallback(mm, off, ln):
|
||||
fallback_calls.append((off, ln))
|
||||
return real_fallback(mm, off, ln)
|
||||
|
||||
monkeypatch.setattr(esr, "_madvise_populate_write", _raise_einval)
|
||||
monkeypatch.setattr(esr, "_fallback_populate_write", _spy_fallback)
|
||||
|
||||
num_blocks, block_size_bytes = 4, mmap.PAGESIZE
|
||||
with _region(num_blocks=num_blocks, block_size_bytes=block_size_bytes) as r:
|
||||
assert fallback_calls == [(0, num_blocks * block_size_bytes)]
|
||||
# Region is still fully usable after taking the fallback path.
|
||||
r.blocks[0, :4] = torch.tensor([1, 2, 3, 4], dtype=torch.int8)
|
||||
assert r.blocks[0, :4].tolist() == [1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_non_creator_does_not_probe_or_populate(monkeypatch):
|
||||
"""Only the creator pre-faults. A second opener must not probe the advice
|
||||
nor re-touch pages the creator already populated."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id, num_blocks=4, block_size_bytes=mmap.PAGESIZE
|
||||
)
|
||||
try:
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
esr,
|
||||
"_madvise_populate_write",
|
||||
lambda mm, off, ln: calls.append("madvise"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
esr,
|
||||
"_fallback_populate_write",
|
||||
lambda mm, off, ln: calls.append("fallback"),
|
||||
)
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id, num_blocks=4, block_size_bytes=mmap.PAGESIZE
|
||||
)
|
||||
try:
|
||||
assert not r2._is_creator
|
||||
assert calls == []
|
||||
finally:
|
||||
r2.cleanup()
|
||||
finally:
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_fallback_populate_write_preserves_bytes_and_faults_pages():
|
||||
"""Fallback preserves existing bytes and touches every target page.
|
||||
|
||||
`|= 0` rather than `= 0` matters here: a peer worker may already have
|
||||
written EC data into the shared mmap before we pre-fault it.
|
||||
"""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
size = 3 * mmap.PAGESIZE
|
||||
if not hasattr(mmap, "MADV_DONTNEED"):
|
||||
pytest.skip("MADV_DONTNEED is unavailable")
|
||||
|
||||
mmap_obj = mmap.mmap(
|
||||
-1,
|
||||
size,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
try:
|
||||
mmap_obj.madvise(mmap.MADV_DONTNEED, 0, size)
|
||||
if any(_page_residency(mmap_obj, size)):
|
||||
pytest.skip("kernel did not discard anonymous mmap pages")
|
||||
|
||||
mmap_obj[0] = 0xAB
|
||||
assert _page_residency(mmap_obj, size) == [True, False, False]
|
||||
|
||||
esr._fallback_populate_write(mmap_obj, 0, size)
|
||||
|
||||
assert _page_residency(mmap_obj, size) == [True, True, True]
|
||||
assert mmap_obj[0] == 0xAB
|
||||
finally:
|
||||
mmap_obj.close()
|
||||
|
||||
|
||||
def test_madvise_unexpected_oserror_propagates(monkeypatch):
|
||||
"""Only EINVAL triggers the fallback. Other OSErrors (e.g. EIO) must
|
||||
propagate out of __init__, not be silently masked by the fallback branch.
|
||||
"""
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu import ec_shared_region as esr
|
||||
|
||||
monkeypatch.setattr(
|
||||
esr,
|
||||
"_madvise_populate_write",
|
||||
lambda mm, off, ln: (_ for _ in ()).throw(
|
||||
OSError(errno.EIO, "simulated I/O failure")
|
||||
),
|
||||
)
|
||||
engine_id = str(uuid.uuid4())
|
||||
try:
|
||||
with pytest.raises(OSError) as exc_info:
|
||||
ECSharedRegion(
|
||||
engine_id=engine_id, num_blocks=4, block_size_bytes=mmap.PAGESIZE
|
||||
)
|
||||
assert exc_info.value.errno == errno.EIO
|
||||
finally:
|
||||
# __init__ raised past the unlink-owning cleanup(), so drop the
|
||||
# backing file here rather than leaking it into /dev/shm.
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
os.unlink(f"/dev/shm/vllm_ec_{engine_id}.mmap")
|
||||
|
||||
|
||||
# ── pin_memory ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_memory_success_sets_flag(region):
|
||||
"""When cudaHostRegister returns 0, _is_pinned flips to True
|
||||
and cleanup will correspondingly call cudaHostUnregister."""
|
||||
fake_cudart = MagicMock()
|
||||
success = MagicMock()
|
||||
success.value = 0
|
||||
fake_cudart.cudaHostRegister.return_value = success
|
||||
fake_cudart.cudaHostUnregister.return_value = success
|
||||
|
||||
with (
|
||||
patch("torch.cuda.is_available", return_value=True),
|
||||
patch("torch.cuda.cudart", return_value=fake_cudart),
|
||||
):
|
||||
region.pin_memory()
|
||||
assert region._is_pinned is True
|
||||
# cleanup must pair with cudaHostUnregister exactly once.
|
||||
region.cleanup()
|
||||
fake_cudart.cudaHostUnregister.assert_called_once()
|
||||
|
||||
|
||||
def test_pin_memory_failure_leaves_flag_false():
|
||||
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered."""
|
||||
r = _make_region()
|
||||
try:
|
||||
fake_cudart = MagicMock()
|
||||
fail = MagicMock()
|
||||
fail.value = 1 # non-zero == error
|
||||
fake_cudart.cudaHostRegister.return_value = fail
|
||||
|
||||
with (
|
||||
patch("torch.cuda.is_available", return_value=True),
|
||||
patch("torch.cuda.cudart", return_value=fake_cudart),
|
||||
):
|
||||
r.pin_memory()
|
||||
assert r._is_pinned is False
|
||||
# Now run cleanup and verify cudaHostUnregister was NOT called.
|
||||
r.cleanup()
|
||||
fake_cudart.cudaHostUnregister.assert_not_called()
|
||||
finally:
|
||||
r.cleanup()
|
||||
|
||||
|
||||
def test_pin_memory_noop_without_cuda(region):
|
||||
"""pin_memory is a no-op when CUDA is not available."""
|
||||
with patch("torch.cuda.is_available", return_value=False):
|
||||
region.pin_memory()
|
||||
assert region._is_pinned is False
|
||||
|
||||
|
||||
# ── cleanup idempotency ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_cleanup_is_idempotent(region):
|
||||
region.cleanup()
|
||||
region.cleanup() # fixture calls a third time — must not raise
|
||||
@@ -1,285 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for control-layer ABCs, ZmqClientConnection, and ECSession."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import zmq
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
|
||||
ControlConnection,
|
||||
)
|
||||
|
||||
|
||||
class _ConcreteConn(ControlConnection):
|
||||
"""Minimal concrete subclass for ABC-instantiation tests."""
|
||||
|
||||
def __init__(self):
|
||||
self._alive = True
|
||||
self._sent = []
|
||||
self._inbox = []
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
def send(self, msg: bytes) -> None:
|
||||
self._sent.append(msg)
|
||||
|
||||
def recv(self) -> list[bytes]:
|
||||
msgs, self._inbox = self._inbox, []
|
||||
return msgs
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
def close(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
|
||||
def test_control_connection_abc_cannot_be_instantiated_directly():
|
||||
with pytest.raises(TypeError):
|
||||
ControlConnection() # type: ignore[abstract]
|
||||
|
||||
|
||||
def test_concrete_connection_send_recv():
|
||||
conn = _ConcreteConn()
|
||||
assert conn.alive
|
||||
conn.send(b"hello")
|
||||
conn._inbox.append(b"world")
|
||||
assert conn.recv() == [b"world"]
|
||||
assert conn._sent == [b"hello"]
|
||||
|
||||
|
||||
def test_mark_dead_sets_alive_false():
|
||||
conn = _ConcreteConn()
|
||||
conn.mark_dead()
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
def test_close_sets_alive_false():
|
||||
conn = _ConcreteConn()
|
||||
conn.close()
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
# ── ZmqClientConnection ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import ( # noqa: E402
|
||||
ZmqClientConnection,
|
||||
ZmqClientTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr # noqa: E402
|
||||
|
||||
|
||||
def _conn(messages: list[list[bytes]], monitor=None) -> ZmqClientConnection:
|
||||
"""Build a ZmqClientConnection whose dealer yields the given frame-lists."""
|
||||
dealer = MagicMock()
|
||||
dealer.recv_multipart.side_effect = [*messages, zmq.Again()]
|
||||
return ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
|
||||
|
||||
def test_zmq_client_connection_recv_extracts_payloads_from_envelopes():
|
||||
conn = _conn([[b"", b"msg1"], [b"", b"msg2"]])
|
||||
assert conn.recv() == [b"msg1", b"msg2"]
|
||||
|
||||
|
||||
def test_zmq_client_connection_recv_skips_frames_without_empty_delimiter():
|
||||
# A frame that does not start with b"" is malformed and must be dropped.
|
||||
conn = _conn([[b"", b"ok"], [b"no-delimiter", b"bad"]])
|
||||
assert conn.recv() == [b"ok"]
|
||||
|
||||
|
||||
def test_zmq_client_connection_recv_breaks_on_non_again_exception():
|
||||
# A socket error should stop the drain and return whatever was collected.
|
||||
dealer = MagicMock()
|
||||
dealer.recv_multipart.side_effect = [[b"", b"first"], OSError("broken")]
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
assert conn.recv() == [b"first"]
|
||||
|
||||
|
||||
def test_zmq_client_connection_send_prefixes_empty_delimiter():
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.send(b"payload")
|
||||
dealer.send_multipart.assert_called_once_with([b"", b"payload"])
|
||||
|
||||
|
||||
def test_zmq_client_connection_mark_dead_does_not_close_socket():
|
||||
# mark_dead signals liveness only; resource teardown is close()'s job.
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.mark_dead()
|
||||
assert not conn.alive
|
||||
dealer.close.assert_not_called()
|
||||
|
||||
|
||||
def test_zmq_client_connection_close_without_monitor():
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.close()
|
||||
dealer.close.assert_called_once_with(linger=0)
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
def test_zmq_client_connection_close_with_monitor_disables_then_closes_both():
|
||||
# The disable_monitor → monitor.close → dealer.close sequence must be
|
||||
# respected: closing the dealer before disabling the monitor can cause
|
||||
# a zmq assertion in some versions.
|
||||
dealer = MagicMock()
|
||||
monitor = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
conn.close()
|
||||
dealer.disable_monitor.assert_called_once()
|
||||
monitor.close.assert_called_once_with(linger=0)
|
||||
dealer.close.assert_called_once_with(linger=0)
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
def test_zmq_client_connection_close_is_idempotent():
|
||||
dealer = MagicMock()
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=None)
|
||||
conn.close()
|
||||
conn.close() # must not raise
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
# ── ZmqClientTransport ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_transport() -> tuple[ZmqClientTransport, MagicMock]:
|
||||
"""Return a ZmqClientTransport with a mocked ZMQ context."""
|
||||
ctx = MagicMock()
|
||||
ctx.socket.return_value = MagicMock()
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context",
|
||||
return_value=ctx,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path",
|
||||
return_value="tcp://host:0",
|
||||
),
|
||||
):
|
||||
transport = ZmqClientTransport()
|
||||
transport._ctx = ctx
|
||||
return transport, ctx
|
||||
|
||||
|
||||
def _inject_conn(transport: ZmqClientTransport, addr: PeerAddr) -> ZmqClientConnection:
|
||||
"""Inject a ZmqClientConnection directly into the transport pool."""
|
||||
conn = ZmqClientConnection(dealer=MagicMock(), monitor=MagicMock())
|
||||
transport._connections[addr] = conn
|
||||
return conn
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
|
||||
def test_zmq_client_transport_connect_caches_connection(
|
||||
mock_ctx_cls, mock_path, mock_sock
|
||||
):
|
||||
mock_ctx_cls.return_value = MagicMock()
|
||||
mock_ctx_cls.return_value.socket.return_value = MagicMock()
|
||||
mock_sock.return_value = MagicMock()
|
||||
transport = ZmqClientTransport()
|
||||
addr: PeerAddr = ("host", 1234)
|
||||
assert transport.connect(addr) is transport.connect(addr)
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
|
||||
def test_zmq_client_transport_connect_creates_independent_conns_per_addr(
|
||||
mock_ctx_cls, mock_path, mock_sock
|
||||
):
|
||||
mock_ctx_cls.return_value = MagicMock()
|
||||
mock_ctx_cls.return_value.socket.return_value = MagicMock()
|
||||
mock_sock.return_value = MagicMock()
|
||||
transport = ZmqClientTransport()
|
||||
conn_a = transport.connect(("host-a", 1111))
|
||||
conn_b = transport.connect(("host-b", 2222))
|
||||
assert conn_a is not conn_b
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
|
||||
def test_zmq_client_transport_poll_dead_detects_disconnected_peer(mock_recv_mon):
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 1234)
|
||||
conn = _inject_conn(transport, addr)
|
||||
mock_recv_mon.side_effect = [{"event": zmq.EVENT_DISCONNECTED}, zmq.Again()]
|
||||
|
||||
dead = transport.poll_dead()
|
||||
|
||||
assert addr in dead
|
||||
assert not conn.alive
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
|
||||
def test_zmq_client_transport_poll_dead_skips_conn_without_monitor(mock_recv_mon):
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 9999)
|
||||
conn = ZmqClientConnection(dealer=MagicMock(), monitor=None)
|
||||
transport._connections[addr] = conn
|
||||
|
||||
dead = transport.poll_dead()
|
||||
|
||||
assert dead == []
|
||||
mock_recv_mon.assert_not_called()
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.recv_monitor_message")
|
||||
def test_zmq_client_transport_poll_dead_ignores_non_disconnect_events(mock_recv_mon):
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 1234)
|
||||
conn = _inject_conn(transport, addr)
|
||||
# A CONNECTED event should not trigger a dead notification.
|
||||
mock_recv_mon.side_effect = [{"event": zmq.EVENT_CONNECTED}, zmq.Again()]
|
||||
|
||||
dead = transport.poll_dead()
|
||||
|
||||
assert dead == []
|
||||
assert conn.alive
|
||||
|
||||
|
||||
def test_zmq_client_transport_remove_closes_and_evicts_conn():
|
||||
transport, _ = _make_transport()
|
||||
addr: PeerAddr = ("host", 5678)
|
||||
conn = _inject_conn(transport, addr)
|
||||
|
||||
removed = transport.remove(addr)
|
||||
|
||||
assert removed is conn
|
||||
assert addr not in transport._connections
|
||||
conn.dealer.close.assert_called()
|
||||
|
||||
|
||||
def test_zmq_client_transport_remove_returns_none_for_unknown_addr():
|
||||
transport, _ = _make_transport()
|
||||
assert transport.remove(("ghost", 0)) is None
|
||||
|
||||
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_socket")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.make_zmq_path")
|
||||
@patch("vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq.zmq.Context")
|
||||
def test_zmq_client_transport_close_closes_all_connections(
|
||||
mock_ctx_cls, mock_path, mock_sock
|
||||
):
|
||||
mock_ctx = MagicMock()
|
||||
mock_ctx_cls.return_value = mock_ctx
|
||||
mock_ctx.socket.return_value = MagicMock()
|
||||
dealer_a, dealer_b = MagicMock(), MagicMock()
|
||||
mock_sock.side_effect = [dealer_a, dealer_b]
|
||||
transport = ZmqClientTransport()
|
||||
transport.connect(("a", 1))
|
||||
transport.connect(("b", 2))
|
||||
transport.close()
|
||||
dealer_a.close.assert_called()
|
||||
dealer_b.close.assert_called()
|
||||
assert transport._connections == {}
|
||||
@@ -1,184 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for DataTransport ABC and NixlDataTransport."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
|
||||
_NIXL_PATH = "vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl"
|
||||
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_nixl_mock(prep_side_effect=None) -> MagicMock:
|
||||
nixl = MagicMock()
|
||||
nixl.get_agent_metadata.return_value = b"agent-meta"
|
||||
nixl.get_reg_descs.return_value = MagicMock()
|
||||
nixl.get_xfer_descs.return_value = MagicMock()
|
||||
nixl.prep_xfer_dlist.side_effect = prep_side_effect or [77, 88]
|
||||
nixl.add_remote_agent.return_value = "remote-agent-1"
|
||||
nixl.get_new_notifs.return_value = {}
|
||||
nixl.check_xfer_state.return_value = "PROC"
|
||||
return nixl
|
||||
|
||||
|
||||
def _make_transport(nixl_mock=None):
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
|
||||
mock = nixl_mock or _make_nixl_mock()
|
||||
with (
|
||||
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=mock),
|
||||
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
|
||||
):
|
||||
t = NixlDataTransport("agent", 0, 4, 64, 256)
|
||||
t._nixl = mock
|
||||
return t
|
||||
|
||||
|
||||
# ── ABC ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_data_transport_abc_cannot_be_instantiated():
|
||||
with pytest.raises(TypeError):
|
||||
DataTransport() # type: ignore[abstract]
|
||||
|
||||
|
||||
# ── NixlDataTransport construction ───────────────────────────────────────────
|
||||
|
||||
|
||||
def test_nixl_data_transport_registers_memory_on_init():
|
||||
nixl = _make_nixl_mock()
|
||||
_make_transport(nixl)
|
||||
nixl.register_memory.assert_called_once()
|
||||
|
||||
|
||||
def test_nixl_data_transport_unwinds_registration_if_init_fails():
|
||||
"""A failure after register_memory must not leave the region pinned.
|
||||
|
||||
__init__ propagates, so the caller never gets a transport to deregister.
|
||||
"""
|
||||
nixl = _make_nixl_mock(prep_side_effect=RuntimeError("no dlist"))
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(f"{_NIXL_PATH}.NixlWrapper", return_value=nixl),
|
||||
patch(f"{_NIXL_PATH}.nixl_agent_config", return_value=MagicMock()),
|
||||
pytest.raises(RuntimeError, match="no dlist"),
|
||||
):
|
||||
NixlDataTransport("agent", 0, 4, 64, 256)
|
||||
|
||||
nixl.deregister_memory.assert_called_once_with(nixl.get_reg_descs.return_value)
|
||||
|
||||
|
||||
def test_nixl_data_transport_raises_if_nixl_unavailable():
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
|
||||
with (
|
||||
patch(f"{_NIXL_PATH}.NixlWrapper", None),
|
||||
patch(f"{_NIXL_PATH}.nixl_agent_config", None),
|
||||
pytest.raises(RuntimeError, match="requires NIXL"),
|
||||
):
|
||||
NixlDataTransport("a", 0, 1, 64, 64)
|
||||
|
||||
|
||||
# ── public accessors ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_agent_metadata_returns_nixl_blob():
|
||||
assert _make_transport().get_agent_metadata() == b"agent-meta"
|
||||
|
||||
|
||||
def test_get_mem_descriptor_roundtrips_block_descs():
|
||||
t = _make_transport()
|
||||
raw = t.get_mem_descriptor()
|
||||
assert isinstance(raw, bytes) and len(raw) > 0
|
||||
|
||||
|
||||
# ── add_remote_peer ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_add_remote_peer_returns_agent_name_and_stores_handle_internally():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
agent_name = t.add_remote_peer(b"meta", mem_desc)
|
||||
assert agent_name == "remote-agent-1"
|
||||
# Dlist handle stored internally; accessible via _peer_handles.
|
||||
assert t._peer_handles["remote-agent-1"] == 88
|
||||
|
||||
|
||||
def test_add_remote_peer_registers_remote_agent():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
t.add_remote_peer(b"fresh-meta", mem_desc)
|
||||
nixl.add_remote_agent.assert_called_once_with(b"fresh-meta")
|
||||
|
||||
|
||||
# ── remove_remote_peer ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_remove_remote_peer_delegates_to_nixl():
|
||||
t = _make_transport()
|
||||
t.remove_remote_peer("agent-x")
|
||||
t._nixl.remove_remote_agent.assert_called_once_with("agent-x")
|
||||
|
||||
|
||||
def test_remove_remote_peer_swallows_exceptions():
|
||||
t = _make_transport()
|
||||
t._nixl.remove_remote_agent.side_effect = RuntimeError("gone")
|
||||
t.remove_remote_peer("agent-x") # must not raise
|
||||
|
||||
|
||||
# ── post_read ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_post_read_raises_on_index_count_mismatch():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
agent = t.add_remote_peer(b"meta", mem_desc)
|
||||
with pytest.raises(ValueError, match="mismatch"):
|
||||
t.post_read([0, 1], agent, [5], notif_msg=b"sid")
|
||||
|
||||
|
||||
def test_post_read_calls_make_prepped_xfer_and_transfer():
|
||||
nixl = _make_nixl_mock(prep_side_effect=[77, 88])
|
||||
t = _make_transport(nixl)
|
||||
mem_desc = serialize_mem_descriptor([(0, 64, 0)])
|
||||
agent = t.add_remote_peer(b"meta", mem_desc)
|
||||
nixl.make_prepped_xfer.return_value = 999
|
||||
handle = t.post_read([0, 1], agent, [5, 6], notif_msg=b"sid")
|
||||
nixl.make_prepped_xfer.assert_called_once()
|
||||
nixl.transfer.assert_called_once_with(999)
|
||||
assert handle == 999
|
||||
|
||||
|
||||
# ── release_xfer_handle / deregister ─────────────────────────────────────────
|
||||
|
||||
|
||||
def test_release_xfer_handle_swallows_exceptions():
|
||||
t = _make_transport()
|
||||
t._nixl.release_xfer_handle.side_effect = RuntimeError("stale")
|
||||
t.release_xfer_handle(42) # must not raise
|
||||
|
||||
|
||||
def test_deregister_swallows_exceptions():
|
||||
t = _make_transport()
|
||||
t._nixl.deregister_memory.side_effect = RuntimeError("already gone")
|
||||
t.deregister() # must not raise
|
||||
@@ -1,85 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the connector-agnostic EC connector metrics plumbing.
|
||||
|
||||
Covers only the base infra: no concrete connector type implements these
|
||||
hooks yet, so every assertion here is about the documented no-op defaults
|
||||
and the container contracts, mirroring the equivalent KV connector tests.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
|
||||
ECExampleConnector,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import (
|
||||
ECConnectorLogging,
|
||||
ECConnectorProm,
|
||||
ECConnectorStats,
|
||||
)
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def example_connector(tmp_path):
|
||||
config = Mock(spec=VllmConfig)
|
||||
config.ec_transfer_config = Mock()
|
||||
config.ec_transfer_config.get_from_extra_config = Mock(return_value=str(tmp_path))
|
||||
config.ec_transfer_config.is_ec_producer = True
|
||||
config.ec_transfer_config.is_ec_consumer = False
|
||||
return ECExampleConnector(vllm_config=config, role=ECConnectorRole.SCHEDULER)
|
||||
|
||||
|
||||
def test_base_stats_methods_raise_not_implemented():
|
||||
"""The base container has no aggregation logic of its own."""
|
||||
stats = ECConnectorStats()
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.reset()
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.aggregate(stats)
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.reduce()
|
||||
with pytest.raises(NotImplementedError):
|
||||
stats.is_empty()
|
||||
|
||||
|
||||
def test_connector_hooks_default_to_none(example_connector):
|
||||
"""No connector type overrides these yet, so every hook is a no-op."""
|
||||
assert example_connector.get_ec_connector_stats() is None
|
||||
assert ECExampleConnector.build_ec_connector_stats() is None
|
||||
assert (
|
||||
ECExampleConnector.build_prom_metrics(
|
||||
vllm_config=Mock(spec=VllmConfig),
|
||||
metric_types={},
|
||||
labelnames=[],
|
||||
per_engine_labelvalues={},
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_ec_connector_output_is_empty_reflects_stats_field():
|
||||
output = ECConnectorOutput()
|
||||
assert output.is_empty()
|
||||
|
||||
output.ec_connector_stats = ECConnectorStats(data={"x": 1})
|
||||
assert not output.is_empty()
|
||||
|
||||
|
||||
def test_logging_and_prom_are_no_ops_without_ec_transfer_config():
|
||||
"""A normal run with no EC connector configured must stay silent."""
|
||||
logging = ECConnectorLogging(ec_transfer_config=None)
|
||||
logging.log() # Should not raise, nothing to log.
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = None
|
||||
prom = ECConnectorProm(vllm_config, labelnames=[], per_engine_labelvalues={})
|
||||
assert prom.prom_metrics is None
|
||||
prom.observe({}) # Should not raise.
|
||||
@@ -1,747 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECExampleConnector."""
|
||||
|
||||
import os
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import safetensors
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.example_connector import (
|
||||
ECExampleConnector,
|
||||
ECExampleConnectorMetadata,
|
||||
MMMeta,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
|
||||
# ------------------ Mock Classes ------------------ #
|
||||
class MockRequest:
|
||||
def __init__(self, request_id, mm_hashes: list[str], token_counts: list[int]):
|
||||
assert len(mm_hashes) == len(token_counts)
|
||||
self.request_id = request_id
|
||||
self._token_counts = token_counts
|
||||
self.mm_features = []
|
||||
for i, mm_hash in enumerate(mm_hashes):
|
||||
feature = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="image",
|
||||
identifier=mm_hash,
|
||||
mm_position=PlaceholderRange(offset=0, length=self._token_counts[i]),
|
||||
)
|
||||
self.mm_features.append(feature)
|
||||
|
||||
def get_num_encoder_embeds(self, input_id: int) -> int:
|
||||
assert input_id < len(self._token_counts)
|
||||
return self._token_counts[input_id]
|
||||
|
||||
|
||||
def test_metadata_preserves_repeated_item_positions():
|
||||
request = MockRequest("request", ["A", "B", "A"], [4, 4, 4])
|
||||
resolver = Mock()
|
||||
resolver.fields_for.return_value = {"image_grid_thw"}
|
||||
for feature in request.mm_features:
|
||||
feature.data = Mock()
|
||||
feature.data.get_data.return_value = {"image_grid_thw": torch.tensor([1, 2, 2])}
|
||||
items = collect_ec_item_metadata(request.mm_features, resolver)
|
||||
assert items == {
|
||||
"A": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [0, 2]},
|
||||
"B": {"metadata": {"image_grid_thw": [1, 2, 2]}, "item_indices": [1]},
|
||||
}
|
||||
|
||||
|
||||
def test_audio_metadata_survives_processor_cache_hit():
|
||||
resolver = Mock()
|
||||
resolver.fields_for.return_value = {"audio_num_tokens"}
|
||||
feature = MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality="audio",
|
||||
identifier="audio",
|
||||
mm_position=PlaceholderRange(
|
||||
offset=0, length=4, is_embed=torch.tensor([False, True, True, False])
|
||||
),
|
||||
)
|
||||
assert collect_ec_item_metadata([feature], resolver) == {
|
||||
"audio": {"metadata": {"audio_num_tokens": [2]}, "item_indices": [0]}
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_storage(tmp_path):
|
||||
"""Fixture providing temporary storage path."""
|
||||
return str(tmp_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_vllm_config_producer(temp_storage):
|
||||
"""Fixture providing mock VllmConfig for producer role."""
|
||||
config = Mock(spec=VllmConfig)
|
||||
config.ec_transfer_config = Mock()
|
||||
config.ec_transfer_config.get_from_extra_config = Mock(return_value=temp_storage)
|
||||
config.ec_transfer_config.is_ec_producer = True
|
||||
return config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_vllm_config_consumer(temp_storage):
|
||||
"""Fixture providing mock VllmConfig for consumer role."""
|
||||
config = Mock(spec=VllmConfig)
|
||||
config.ec_transfer_config = Mock()
|
||||
config.ec_transfer_config.get_from_extra_config = Mock(return_value=temp_storage)
|
||||
config.ec_transfer_config.is_ec_producer = False
|
||||
return config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_request_with_3_mm():
|
||||
"""Fixture providing mock Request with 3 multimodal items."""
|
||||
request_id = "test_req_123"
|
||||
mm_hashes = ["img_hash_1", "img_hash_2", "img_hash_3"]
|
||||
token_counts = [100, 150, 200]
|
||||
|
||||
request = MockRequest(request_id, mm_hashes, token_counts)
|
||||
return request
|
||||
|
||||
|
||||
# ------------------ Unit Tests ------------------ #
|
||||
class TestECExampleConnectorBasics:
|
||||
"""Test basic EC connector functionality."""
|
||||
|
||||
def test_initialization_producer(self, mock_vllm_config_producer, temp_storage):
|
||||
"""Test connector initializes correctly as producer."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
assert connector.role == ECConnectorRole.SCHEDULER
|
||||
assert connector.is_producer
|
||||
assert connector._storage_path == temp_storage
|
||||
assert connector._mm_datas_need_loads == {}
|
||||
|
||||
def test_initialization_consumer(self, mock_vllm_config_consumer, temp_storage):
|
||||
"""Test connector initializes correctly as consumer."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
assert connector.role == ECConnectorRole.WORKER
|
||||
assert not connector.is_producer
|
||||
assert connector._storage_path == temp_storage
|
||||
|
||||
def test_role_assignment(self, mock_vllm_config_producer):
|
||||
"""Test role is correctly assigned."""
|
||||
scheduler_connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
worker_connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
assert scheduler_connector.role == ECConnectorRole.SCHEDULER
|
||||
assert worker_connector.role == ECConnectorRole.WORKER
|
||||
|
||||
|
||||
class TestCacheExistence:
|
||||
"""Test cache existence checking using has_cache_item() API."""
|
||||
|
||||
def test_has_cache_item_all_exist_3_items(
|
||||
self,
|
||||
mock_vllm_config_producer,
|
||||
mock_vllm_config_consumer,
|
||||
mock_request_with_3_mm,
|
||||
):
|
||||
"""Test has_cache_item returns True when all 3 caches exist."""
|
||||
# Test for producer first
|
||||
producer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Create cache files using save_caches (proper way)
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
|
||||
for mm_feature in mock_request_with_3_mm.mm_features:
|
||||
mm_hash = mm_feature.identifier
|
||||
encoder_cache[mm_hash] = torch.randn(10, 768)
|
||||
producer.save_caches(encoder_cache, mm_hash)
|
||||
|
||||
# Test using has_cache_item API
|
||||
producer_result = [
|
||||
producer.has_cache_item(mm_feature.identifier)
|
||||
for mm_feature in mock_request_with_3_mm.mm_features
|
||||
]
|
||||
|
||||
# Assert
|
||||
assert len(producer_result) == 3
|
||||
assert all(producer_result), f"Expected all True, got {producer_result}"
|
||||
|
||||
# Also test consumer can check if cache exists
|
||||
consumer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Test using has_cache_item API
|
||||
consumer_result = [
|
||||
consumer.has_cache_item(mm_feature.identifier)
|
||||
for mm_feature in mock_request_with_3_mm.mm_features
|
||||
]
|
||||
|
||||
# Assert
|
||||
assert len(consumer_result) == 3
|
||||
assert all(consumer_result), f"Expected all True, got {consumer_result}"
|
||||
|
||||
def test_has_cache_item_none_exist(
|
||||
self, mock_vllm_config_producer, mock_request_with_3_mm
|
||||
):
|
||||
"""Test has_caches returns False when no caches exist."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Test without creating any files
|
||||
result = [
|
||||
connector.has_cache_item(mm_feature.identifier)
|
||||
for mm_feature in mock_request_with_3_mm.mm_features
|
||||
]
|
||||
|
||||
# Assert
|
||||
assert len(result) == 3
|
||||
assert not any(result), f"Expected all False, got {result}"
|
||||
|
||||
def test_has_cache_item_partial_exist(
|
||||
self, mock_vllm_config_producer, mock_request_with_3_mm
|
||||
):
|
||||
"""Test has_caches with some caches existing (1 of 3)."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Create only the second cache file
|
||||
mm_hash_second = mock_request_with_3_mm.mm_features[1].identifier
|
||||
encoder_cache = {mm_hash_second: torch.randn(10, 768)}
|
||||
connector.save_caches(encoder_cache, mm_hash_second)
|
||||
|
||||
# Test
|
||||
result = [
|
||||
connector.has_cache_item(mm_feature.identifier)
|
||||
for mm_feature in mock_request_with_3_mm.mm_features
|
||||
]
|
||||
|
||||
# Assert
|
||||
assert len(result) == 3
|
||||
assert not result[0] # First doesn't exist
|
||||
assert result[1] # Second exists
|
||||
assert not result[2] # Third doesn't exist
|
||||
|
||||
def test_has_cache_item_miss_creates_no_folder(
|
||||
self, mock_vllm_config_consumer, temp_storage
|
||||
):
|
||||
"""Test a cache miss leaves no empty folder in the shared storage."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
assert not connector.has_cache_item("missing_hash")
|
||||
assert os.listdir(temp_storage) == []
|
||||
|
||||
|
||||
class TestStateManagement:
|
||||
"""Test connector state management."""
|
||||
|
||||
def test_update_state_after_alloc_3_items(
|
||||
self, mock_vllm_config_producer, mock_request_with_3_mm
|
||||
):
|
||||
"""Test state update after allocation for 3 MM items."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Initial state should be empty
|
||||
assert len(connector._mm_datas_need_loads) == 0
|
||||
|
||||
# Update state for all 3 items (mock cache existence)
|
||||
with patch.object(connector, "has_cache_item", return_value=True):
|
||||
for i in range(3):
|
||||
connector.update_state_after_alloc(mock_request_with_3_mm, index=i)
|
||||
|
||||
# Check state updated for all 3
|
||||
assert len(connector._mm_datas_need_loads) == 3
|
||||
assert "img_hash_1" in connector._mm_datas_need_loads
|
||||
assert "img_hash_2" in connector._mm_datas_need_loads
|
||||
assert "img_hash_3" in connector._mm_datas_need_loads
|
||||
assert connector._mm_datas_need_loads["img_hash_1"] == 100
|
||||
assert connector._mm_datas_need_loads["img_hash_2"] == 150
|
||||
assert connector._mm_datas_need_loads["img_hash_3"] == 200
|
||||
|
||||
def test_build_connector_meta_3_items(
|
||||
self, mock_vllm_config_producer, mock_request_with_3_mm
|
||||
):
|
||||
"""Test metadata building for 3 MM items."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Setup state for all 3 items (mock cache existence)
|
||||
with patch.object(connector, "has_cache_item", return_value=True):
|
||||
for i in range(3):
|
||||
connector.update_state_after_alloc(mock_request_with_3_mm, index=i)
|
||||
|
||||
# Build metadata
|
||||
scheduler_output = Mock(spec=SchedulerOutput)
|
||||
metadata = connector.build_connector_meta(scheduler_output)
|
||||
|
||||
# Assert
|
||||
assert isinstance(metadata, ECExampleConnectorMetadata)
|
||||
assert len(metadata.mm_datas) == 3
|
||||
assert metadata.mm_datas[0].mm_hash == "img_hash_1"
|
||||
assert metadata.mm_datas[0].num_token == 100
|
||||
assert metadata.mm_datas[1].mm_hash == "img_hash_2"
|
||||
assert metadata.mm_datas[1].num_token == 150
|
||||
assert metadata.mm_datas[2].mm_hash == "img_hash_3"
|
||||
assert metadata.mm_datas[2].num_token == 200
|
||||
|
||||
# State should be cleared after building
|
||||
assert len(connector._mm_datas_need_loads) == 0
|
||||
|
||||
def test_build_connector_meta_empty(self, mock_vllm_config_producer):
|
||||
"""Test metadata building with empty state."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
scheduler_output = Mock(spec=SchedulerOutput)
|
||||
metadata = connector.build_connector_meta(scheduler_output)
|
||||
|
||||
assert isinstance(metadata, ECExampleConnectorMetadata)
|
||||
assert len(metadata.mm_datas) == 0
|
||||
|
||||
def test_state_cleared_after_metadata_build(
|
||||
self, mock_vllm_config_producer, mock_request_with_3_mm
|
||||
):
|
||||
"""Test that state is properly cleared after building metadata."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
# Add state (mock cache existence)
|
||||
with patch.object(connector, "has_cache_item", return_value=True):
|
||||
for i in range(3):
|
||||
connector.update_state_after_alloc(mock_request_with_3_mm, index=i)
|
||||
assert len(connector._mm_datas_need_loads) == 3
|
||||
|
||||
# Build metadata (should clear state)
|
||||
scheduler_output = Mock(spec=SchedulerOutput)
|
||||
connector.build_connector_meta(scheduler_output)
|
||||
|
||||
# State should be empty
|
||||
assert len(connector._mm_datas_need_loads) == 0
|
||||
|
||||
# Build again should return empty metadata
|
||||
metadata2 = connector.build_connector_meta(scheduler_output)
|
||||
assert len(metadata2.mm_datas) == 0
|
||||
|
||||
|
||||
class TestCacheSaving:
|
||||
"""Test encoder cache saving (producer only)."""
|
||||
|
||||
def test_save_caches_producer_3_items(
|
||||
self, mock_vllm_config_producer, mock_request_with_3_mm, temp_storage
|
||||
):
|
||||
"""Test cache saving as producer for 3 different MM items."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
# Create and save 3 different caches
|
||||
mm_hashes = [f.identifier for f in mock_request_with_3_mm.mm_features]
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
|
||||
for mm_hash in mm_hashes:
|
||||
encoder_cache[mm_hash] = torch.randn(10, 768)
|
||||
connector.save_caches(encoder_cache, mm_hash)
|
||||
|
||||
# Verify all files exist using has_cache_item
|
||||
result = [
|
||||
connector.has_cache_item(mm_feature.identifier)
|
||||
for mm_feature in mock_request_with_3_mm.mm_features
|
||||
]
|
||||
assert all(result), f"Not all caches were saved: {result}"
|
||||
|
||||
# Verify each file's content
|
||||
for mm_hash in mm_hashes:
|
||||
filename = connector._generate_filename_debug(mm_hash)
|
||||
loaded = safetensors.torch.load_file(filename)
|
||||
assert "ec_cache" in loaded
|
||||
assert torch.allclose(loaded["ec_cache"], encoder_cache[mm_hash].cpu())
|
||||
|
||||
def test_save_caches_consumer_skips(self, mock_vllm_config_consumer):
|
||||
"""Test cache saving is skipped for consumer."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
mm_hash = "test_hash_consumer"
|
||||
encoder_cache = {mm_hash: torch.randn(10, 768)}
|
||||
|
||||
# Save should not raise but also not create file
|
||||
connector.save_caches(encoder_cache, mm_hash)
|
||||
|
||||
# Verify file doesn't exist using has_cache_item
|
||||
result = connector.has_cache_item(mm_hash)
|
||||
assert not result, "Consumer should not save caches"
|
||||
|
||||
|
||||
class TestCacheLoading:
|
||||
"""Test encoder cache loading (consumer)."""
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
def test_start_load_caches_consumer_3_items(
|
||||
self,
|
||||
mock_vllm_config_producer,
|
||||
mock_vllm_config_consumer,
|
||||
mock_request_with_3_mm,
|
||||
temp_storage,
|
||||
):
|
||||
"""Test consumer loads 3 caches from storage."""
|
||||
# First, create producer to save caches
|
||||
producer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
# Producer saves 3 caches
|
||||
mm_hashes = [f.identifier for f in mock_request_with_3_mm.mm_features]
|
||||
saved_caches = {}
|
||||
for mm_hash in mm_hashes:
|
||||
saved_caches[mm_hash] = torch.randn(10, 768)
|
||||
producer.save_caches(saved_caches, mm_hash)
|
||||
|
||||
# Now consumer loads
|
||||
consumer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
# Setup metadata for all 3
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
for mm_hash in mm_hashes:
|
||||
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
|
||||
consumer.bind_connector_metadata(metadata)
|
||||
|
||||
# Load
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
consumer.start_load_caches(encoder_cache=encoder_cache)
|
||||
|
||||
# Verify all 3 loaded
|
||||
assert len(encoder_cache) == 3
|
||||
for mm_hash in mm_hashes:
|
||||
assert mm_hash in encoder_cache, f"{mm_hash} missing in encoder_cache"
|
||||
assert encoder_cache[mm_hash].is_cuda, (
|
||||
f"{mm_hash} cache is in {encoder_cache[mm_hash].device}"
|
||||
)
|
||||
assert torch.allclose(
|
||||
encoder_cache[mm_hash].cpu(), saved_caches[mm_hash]
|
||||
), f"{mm_hash} cache saved and loaded tesnor are not the same"
|
||||
|
||||
def test_start_load_caches_skip_existing(
|
||||
self, mock_vllm_config_producer, mock_vllm_config_consumer, temp_storage
|
||||
):
|
||||
"""Test cache loading skips already cached items."""
|
||||
# Setup: producer saves cache
|
||||
producer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
mm_hash = "existing_hash"
|
||||
saved_cache = torch.randn(10, 768)
|
||||
producer.save_caches({mm_hash: saved_cache}, mm_hash)
|
||||
|
||||
# Consumer setup
|
||||
consumer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
|
||||
consumer.bind_connector_metadata(metadata)
|
||||
|
||||
# Pre-populate encoder_cache with different value
|
||||
existing_cache = torch.randn(5, 512)
|
||||
encoder_cache = {mm_hash: existing_cache}
|
||||
|
||||
# Load (should skip since already exists)
|
||||
with patch("safetensors.torch.load_file") as mock_load:
|
||||
consumer.start_load_caches(encoder_cache=encoder_cache)
|
||||
# Should not call load_file since cache exists
|
||||
mock_load.assert_not_called()
|
||||
|
||||
# Verify original cache unchanged
|
||||
assert torch.equal(encoder_cache[mm_hash], existing_cache)
|
||||
|
||||
def test_start_load_caches_empty_metadata(self, mock_vllm_config_consumer):
|
||||
"""Test loading with empty metadata does nothing."""
|
||||
consumer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
# Setup empty metadata
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
consumer.bind_connector_metadata(metadata)
|
||||
|
||||
# Load (should not raise)
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
consumer.start_load_caches(encoder_cache=encoder_cache)
|
||||
|
||||
# Cache should remain empty
|
||||
assert len(encoder_cache) == 0
|
||||
|
||||
def test_start_load_caches_pins_device_to_current_rank(
|
||||
self, mock_vllm_config_producer, mock_vllm_config_consumer
|
||||
):
|
||||
"""Load must pin to the selected device under tensor parallelism."""
|
||||
# Produce a real cache file so the load path has something to open.
|
||||
producer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
mm_hash = "rank_pinning_hash"
|
||||
producer.save_caches({mm_hash: torch.zeros(4, 8)}, mm_hash)
|
||||
|
||||
consumer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
|
||||
consumer.bind_connector_metadata(metadata)
|
||||
|
||||
mock_platform = Mock()
|
||||
mock_platform.device_type = "cuda"
|
||||
mock_platform.is_cuda_alike.return_value = True
|
||||
mock_platform.current_device.return_value = 2
|
||||
|
||||
# Simulate running on TP rank 2 without requiring a multi-GPU host.
|
||||
captured: dict = {}
|
||||
|
||||
def fake_load_file(filename, device):
|
||||
captured["device"] = device
|
||||
return {"ec_cache": torch.zeros(4, 8)}
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform", mock_platform),
|
||||
patch("safetensors.torch.load_file", side_effect=fake_load_file),
|
||||
):
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
consumer.start_load_caches(encoder_cache=encoder_cache)
|
||||
|
||||
assert captured["device"] == "cuda:2", (
|
||||
f"Expected device='cuda:2' (pinned to current rank), "
|
||||
f"got {captured['device']!r}. A bare 'cuda' string makes "
|
||||
"safetensors load onto cuda:0, which breaks non-rank-0 TP workers."
|
||||
)
|
||||
|
||||
def test_start_load_caches_non_cuda_platform_uses_device_type(
|
||||
self, mock_vllm_config_producer, mock_vllm_config_consumer
|
||||
):
|
||||
"""Non-CUDA platforms pass `device_type` through unchanged (no rank suffix)."""
|
||||
producer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
mm_hash = "cpu_device_hash"
|
||||
producer.save_caches({mm_hash: torch.zeros(2, 4)}, mm_hash)
|
||||
|
||||
consumer = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
metadata.add_mm_data(MMMeta.make_meta(mm_hash, 100))
|
||||
consumer.bind_connector_metadata(metadata)
|
||||
|
||||
mock_platform = Mock()
|
||||
mock_platform.device_type = "cpu"
|
||||
mock_platform.is_cuda_alike.return_value = False
|
||||
|
||||
captured: dict = {}
|
||||
|
||||
def fake_load_file(filename, device):
|
||||
captured["device"] = device
|
||||
return {"ec_cache": torch.zeros(2, 4)}
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform", mock_platform),
|
||||
patch("safetensors.torch.load_file", side_effect=fake_load_file),
|
||||
):
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
consumer.start_load_caches(encoder_cache=encoder_cache)
|
||||
|
||||
assert captured["device"] == "cpu"
|
||||
|
||||
|
||||
class TestFilenameGeneration:
|
||||
"""Test filename and path generation."""
|
||||
|
||||
def test_generate_foldername(self, mock_vllm_config_producer, temp_storage):
|
||||
"""Test folder name generation."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
mm_hash = "test_folder_hash"
|
||||
folder = connector._generate_foldername_debug(mm_hash)
|
||||
|
||||
assert folder == os.path.join(temp_storage, mm_hash)
|
||||
assert os.path.isdir(folder) # Should be created
|
||||
|
||||
def test_generate_filename(self, mock_vllm_config_producer, temp_storage):
|
||||
"""Test filename generation."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
mm_hash = "test_file_hash"
|
||||
filename = connector._generate_filename_debug(mm_hash)
|
||||
|
||||
expected = os.path.join(temp_storage, mm_hash, "encoder_cache.safetensors")
|
||||
assert filename == expected
|
||||
assert os.path.isdir(os.path.dirname(filename)) # Folder created
|
||||
|
||||
def test_generate_filename_consistency(self, mock_vllm_config_producer):
|
||||
"""Test filename generation is consistent."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
mm_hash = "consistency_hash"
|
||||
filename1 = connector._generate_filename_debug(mm_hash)
|
||||
filename2 = connector._generate_filename_debug(mm_hash)
|
||||
|
||||
assert filename1 == filename2
|
||||
|
||||
|
||||
class TestMetadataBindingLifecycle:
|
||||
"""Test metadata binding and clearing lifecycle."""
|
||||
|
||||
def test_bind_connector_metadata(self, mock_vllm_config_consumer):
|
||||
"""Test binding connector metadata."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
metadata.add_mm_data(MMMeta.make_meta("hash_1", 100))
|
||||
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
assert connector._connector_metadata is metadata
|
||||
|
||||
def test_clear_connector_metadata(self, mock_vllm_config_consumer):
|
||||
"""Test clearing connector metadata."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
connector.clear_connector_metadata()
|
||||
|
||||
assert connector._connector_metadata is None
|
||||
|
||||
def test_get_connector_metadata(self, mock_vllm_config_consumer):
|
||||
"""Test getting connector metadata."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
retrieved = connector._get_connector_metadata()
|
||||
|
||||
assert retrieved is metadata
|
||||
|
||||
def test_get_connector_metadata_not_set(self, mock_vllm_config_consumer):
|
||||
"""Test getting metadata when not set raises."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
connector._get_connector_metadata()
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
"""Test edge cases and error handling."""
|
||||
|
||||
def test_save_empty_cache(self, mock_vllm_config_producer):
|
||||
"""Test saving empty tensor."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
mm_hash = "empty_hash"
|
||||
encoder_cache = {mm_hash: torch.empty(0)}
|
||||
|
||||
# Should not raise
|
||||
connector.save_caches(encoder_cache, mm_hash)
|
||||
|
||||
def test_load_nonexistent_cache(self, mock_vllm_config_consumer):
|
||||
"""Test loading cache that doesn't exist raises error."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.WORKER,
|
||||
)
|
||||
|
||||
metadata = ECExampleConnectorMetadata()
|
||||
metadata.add_mm_data(MMMeta.make_meta("nonexistent_hash", 100))
|
||||
connector.bind_connector_metadata(metadata)
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
|
||||
# Should raise FileNotFoundError
|
||||
with pytest.raises(FileNotFoundError):
|
||||
connector.start_load_caches(encoder_cache=encoder_cache)
|
||||
|
||||
def test_has_cache_item_empty_request(self, mock_vllm_config_producer):
|
||||
"""Test has_cache_item with a nonexistent identifier."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_producer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
result = connector.has_cache_item("nonexistent_hash")
|
||||
|
||||
assert result is False
|
||||
@@ -1,131 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECOutputAggregator."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorWorkerMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.metrics import ECConnectorStats
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import ECOutputAggregator
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import KVOutputAggregator
|
||||
from vllm.v1.outputs import (
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class FakeWorkerMeta(ECConnectorWorkerMetadata):
|
||||
"""Records merge order. `aggregate` returns a new object, as the base class
|
||||
declares: an aggregator discarding the return value would lose the merge.
|
||||
"""
|
||||
|
||||
def __init__(self, saves: list[str]):
|
||||
self.saves = saves
|
||||
|
||||
def aggregate(self, other: "FakeWorkerMeta") -> "FakeWorkerMeta":
|
||||
return FakeWorkerMeta(self.saves + other.saves)
|
||||
|
||||
|
||||
class FakeConnectorStats(ECConnectorStats):
|
||||
"""Records merge order, mirroring FakeWorkerMeta above."""
|
||||
|
||||
def __init__(self, saves: list[str]):
|
||||
self.saves = saves
|
||||
|
||||
def aggregate(self, other: "FakeConnectorStats") -> "FakeConnectorStats":
|
||||
return FakeConnectorStats(self.saves + other.saves)
|
||||
|
||||
|
||||
def _worker_output(ec_output: ECConnectorOutput | None) -> ModelRunnerOutput:
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[], req_id_to_index={}, ec_connector_output=ec_output
|
||||
)
|
||||
|
||||
|
||||
def test_aggregate_folds_every_rank_onto_output_rank():
|
||||
"""EC work done on any rank reaches the scheduler via output_rank's output.
|
||||
|
||||
The middle rank reports no worker metadata or stats: it must neither seed
|
||||
nor clobber either accumulator.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(
|
||||
ECConnectorOutput(
|
||||
finished_sending={"mm0"},
|
||||
ec_connector_stats=FakeConnectorStats(["mm0"]),
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm0"]),
|
||||
)
|
||||
),
|
||||
_worker_output(ECConnectorOutput(finished_recving={"mm1"})),
|
||||
_worker_output(
|
||||
ECConnectorOutput(
|
||||
ec_connector_stats=FakeConnectorStats(["mm2"]),
|
||||
ec_connector_worker_meta=FakeWorkerMeta(["mm2"]),
|
||||
)
|
||||
),
|
||||
]
|
||||
|
||||
result = ECOutputAggregator().aggregate(outputs, output_rank=2)
|
||||
|
||||
assert result is outputs[2]
|
||||
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||
assert result.ec_connector_output.finished_recving == {"mm1"}
|
||||
assert result.ec_connector_output.ec_connector_stats.saves == ["mm0", "mm2"]
|
||||
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0", "mm2"]
|
||||
|
||||
|
||||
def test_aggregate_leaves_no_ec_output_when_no_worker_reported():
|
||||
"""Empty per-worker reports must not reach the scheduler as an empty object."""
|
||||
outputs = [_worker_output(ECConnectorOutput()), _worker_output(ECConnectorOutput())]
|
||||
|
||||
result = ECOutputAggregator().aggregate(outputs, output_rank=0)
|
||||
|
||||
assert result is outputs[0]
|
||||
assert result.ec_connector_output is None
|
||||
assert ECOutputAggregator().aggregate([None], output_rank=0) is None
|
||||
|
||||
|
||||
def test_aggregate_does_not_write_through_the_shared_empty_output():
|
||||
"""A rank with nothing to report yields the shared empty output singleton.
|
||||
|
||||
Folding another rank's metadata onto it must not write through to the
|
||||
module-level object, which every later step would then carry.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(
|
||||
ECConnectorOutput(ec_connector_worker_meta=FakeWorkerMeta(["mm0"]))
|
||||
),
|
||||
EMPTY_MODEL_RUNNER_OUTPUT,
|
||||
]
|
||||
|
||||
result = ECOutputAggregator().aggregate(outputs, output_rank=1)
|
||||
|
||||
assert EMPTY_MODEL_RUNNER_OUTPUT.ec_connector_output is None
|
||||
assert result is not EMPTY_MODEL_RUNNER_OUTPUT
|
||||
assert result.ec_connector_output.ec_connector_worker_meta.saves == ["mm0"]
|
||||
|
||||
|
||||
def test_chaining_with_kv_aggregator_preserves_both_outputs():
|
||||
"""MultiprocExecutor chains both aggregators and keeps only the last result,
|
||||
so each must merge onto the same output_rank output rather than replace it.
|
||||
"""
|
||||
outputs = [
|
||||
_worker_output(ECConnectorOutput(finished_sending={"mm0"})),
|
||||
_worker_output(None),
|
||||
]
|
||||
outputs[1].kv_connector_output = KVConnectorOutput(invalid_block_ids={7})
|
||||
|
||||
result = None
|
||||
for aggregator in (
|
||||
KVOutputAggregator(expected_finished_count=1),
|
||||
ECOutputAggregator(),
|
||||
):
|
||||
result = aggregator.aggregate(outputs, output_rank=1)
|
||||
|
||||
assert result is outputs[1]
|
||||
assert result.kv_connector_output.invalid_block_ids == {7}
|
||||
assert result.ec_connector_output.finished_sending == {"mm0"}
|
||||
@@ -1,129 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sanity tests for ec_transfer_params protocol plumbing.
|
||||
|
||||
No running engine required.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
# tests.v1.core.utils transitively imports the example KV connector, which
|
||||
# requires the compiled CUDA flash-attention extensions; skip cleanly rather
|
||||
# than erroring at collection when those extensions aren't built.
|
||||
_core_utils = pytest.importorskip(
|
||||
"tests.v1.core.utils",
|
||||
reason="requires compiled CUDA flash-attention extensions",
|
||||
exc_type=ImportError,
|
||||
)
|
||||
create_scheduler = _core_utils.create_scheduler
|
||||
|
||||
EC_PARAMS: dict = {"mm_hash_abc": {"peer_host": "10.0.0.1", "peer_port": 5501}}
|
||||
|
||||
|
||||
def test_ec_transfer_params_routed_to_sampling_params_extra_args():
|
||||
"""ec_transfer_params on the request must land in SamplingParams.extra_args."""
|
||||
req = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
max_tokens=5,
|
||||
ec_transfer_params=EC_PARAMS,
|
||||
)
|
||||
sp = req.to_sampling_params(max_tokens=5, default_sampling_params={})
|
||||
assert sp.extra_args is not None
|
||||
assert sp.extra_args.get("ec_transfer_params") == EC_PARAMS
|
||||
|
||||
|
||||
def test_request_output_add_propagates_ec_transfer_params():
|
||||
"""RequestOutput.add() must carry ec_transfer_params forward to the caller."""
|
||||
|
||||
def _out(ec_params):
|
||||
return RequestOutput(
|
||||
request_id="r1",
|
||||
prompt="p",
|
||||
prompt_token_ids=[1],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=[],
|
||||
cumulative_logprob=0.0,
|
||||
logprobs=None,
|
||||
finish_reason=None,
|
||||
)
|
||||
],
|
||||
finished=False,
|
||||
ec_transfer_params=ec_params,
|
||||
)
|
||||
|
||||
accumulated = _out(None)
|
||||
accumulated.add(_out(EC_PARAMS), aggregate=True)
|
||||
assert accumulated.ec_transfer_params == EC_PARAMS
|
||||
|
||||
|
||||
def test_request_reads_ec_transfer_params_from_extra_args():
|
||||
"""v1 Request must pull ec_transfer_params out of SamplingParams.extra_args."""
|
||||
sp = SamplingParams(extra_args={"ec_transfer_params": EC_PARAMS})
|
||||
req = Request(
|
||||
request_id="r1",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=sp,
|
||||
pooling_params=None,
|
||||
)
|
||||
assert req.ec_transfer_params == EC_PARAMS
|
||||
|
||||
|
||||
def test_free_request_calls_ec_connector_and_surfaces_params():
|
||||
"""_free_request must call ec_connector.request_finished() and return its params."""
|
||||
sp = SamplingParams(max_tokens=1)
|
||||
sp.update_from_generation_config({}, 50256)
|
||||
request = Request(
|
||||
request_id="test-req",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=sp,
|
||||
pooling_params=None,
|
||||
client_index=0,
|
||||
)
|
||||
scheduler = create_scheduler(use_ec_connector=True, ec_role="ec_producer")
|
||||
scheduler.add_request(request)
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
|
||||
mock_ec = MagicMock()
|
||||
mock_ec.request_finished.return_value = (False, EC_PARAMS)
|
||||
scheduler.ec_connector = mock_ec
|
||||
|
||||
kv_params, ec_params = scheduler._free_request(request)
|
||||
|
||||
mock_ec.request_finished.assert_called_once_with(request)
|
||||
assert ec_params == EC_PARAMS
|
||||
assert kv_params is None
|
||||
|
||||
|
||||
def test_free_request_without_ec_connector_returns_none():
|
||||
"""When no EC connector is configured, ec_transfer_params must be None."""
|
||||
sp = SamplingParams(max_tokens=1)
|
||||
sp.update_from_generation_config({}, 50256)
|
||||
request = Request(
|
||||
request_id="test-req",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
sampling_params=sp,
|
||||
pooling_params=None,
|
||||
client_index=0,
|
||||
)
|
||||
scheduler = create_scheduler(use_ec_connector=True, ec_role="ec_producer")
|
||||
scheduler.add_request(request)
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
|
||||
kv_params, ec_params = scheduler._free_request(request)
|
||||
|
||||
assert ec_params is None
|
||||
assert kv_params is None
|
||||
@@ -1,133 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encoder fan-out fairness in the disaggregated EPD proxy.
|
||||
|
||||
Exercises the REAL ``encoder_rr_assignment`` helper loaded from its
|
||||
``examples/`` path -- no routing logic is re-implemented here, so a future
|
||||
change to that function is what these tests exercise.
|
||||
|
||||
Regression target: the fan-out cursor used to restart at e_urls[0] on every
|
||||
incoming request (``e_urls[i % len(e_urls)] for i in range(len(mm_items))``),
|
||||
so single-item requests -- the common case -- always hit the first encoder
|
||||
instance and left the rest idle. ``encoder_rr_assignment`` threads a cursor
|
||||
across calls so consecutive requests keep advancing through every URL.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
|
||||
|
||||
|
||||
def _load_proxy_module():
|
||||
path = Path(__file__).parents[4] / PROXY_REL
|
||||
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_under_test", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy():
|
||||
return _load_proxy_module()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def assign(proxy):
|
||||
return proxy.encoder_rr_assignment
|
||||
|
||||
|
||||
def _drive(assign, e_urls, counts):
|
||||
"""Feed a sequence of request item-counts through the cursor, as
|
||||
fanout_encoder_primer does one request at a time."""
|
||||
cursor = 0
|
||||
all_urls = []
|
||||
for count in counts:
|
||||
urls, cursor = assign(e_urls, cursor, count)
|
||||
assert len(urls) == count
|
||||
all_urls.append(urls)
|
||||
return all_urls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_urls", [1, 2, 3, 5])
|
||||
def test_full_url_space_is_covered_uniformly(assign, n_urls):
|
||||
e_urls = [f"E{i}" for i in range(n_urls)]
|
||||
# Three full cycles of single-item requests.
|
||||
urls = _drive(assign, e_urls, counts=[1] * (n_urls * 3))
|
||||
hits = Counter(u for req in urls for u in req)
|
||||
assert set(hits) == set(e_urls)
|
||||
assert max(hits.values()) == min(hits.values())
|
||||
|
||||
|
||||
def test_single_item_requests_rotate_through_all_encoders(assign):
|
||||
# Exact scenario from the bug report: 3 encoders, one image/request.
|
||||
e_urls = ["E0", "E1", "E2"]
|
||||
routed = _drive(assign, e_urls, counts=[1] * 6)
|
||||
assert [urls[0] for urls in routed] == ["E0", "E1", "E2", "E0", "E1", "E2"]
|
||||
|
||||
|
||||
def test_cursor_stays_contiguous_across_varying_item_counts(assign):
|
||||
e_urls = ["E0", "E1", "E2"]
|
||||
routed = _drive(assign, e_urls, counts=[2, 1, 3, 1])
|
||||
assert routed == [
|
||||
["E0", "E1"],
|
||||
["E2"],
|
||||
["E0", "E1", "E2"],
|
||||
["E0"],
|
||||
]
|
||||
|
||||
|
||||
def test_single_encoder_always_resolves_to_it(assign):
|
||||
urls, next_cursor = assign(["E0"], 0, 4)
|
||||
assert urls == ["E0"] * 4
|
||||
assert next_cursor == 0
|
||||
|
||||
|
||||
def test_generic_e_p_d_routing_without_mooncake_is_supported(proxy):
|
||||
proxy.validate_ec_consumer_routing(["http://prefill"], [])
|
||||
|
||||
|
||||
def test_mooncake_e_pd_routing_is_supported(proxy):
|
||||
proxy.validate_ec_consumer_routing([], ["tcp://decode:19019"])
|
||||
|
||||
|
||||
def test_mooncake_independent_prefill_routing_fails_fast(proxy):
|
||||
with pytest.raises(ValueError, match=r"supports E\+PD only"):
|
||||
proxy.validate_ec_consumer_routing(["http://prefill"], ["tcp://decode:19019"])
|
||||
|
||||
|
||||
def test_decode_rewrite_preserves_engine_reported_ec_hash(proxy):
|
||||
request = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
rewritten = proxy.rewrite_for_decode(
|
||||
request,
|
||||
{
|
||||
0: {
|
||||
"mm_hash": "proxy-uuid",
|
||||
"ec_mm_hash": "engine-derived-hash",
|
||||
"transfer_id": "transfer",
|
||||
"image_grid_thw": [1, 2, 3],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
assert rewritten["messages"][0]["content"][0]["uuid"] == "proxy-uuid"
|
||||
assert rewritten["messages"][0]["content"][0]["image_embeds"] == {
|
||||
"image_grid_thw": [1, 2, 3]
|
||||
}
|
||||
assert rewritten["ec_transfer_params"]["ec_items"] == [
|
||||
{"mm_hash": "engine-derived-hash", "transfer_id": "transfer"}
|
||||
]
|
||||
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECCPUConnectorMetadata."""
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_is_ec_connector_metadata():
|
||||
"""ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
assert isinstance(meta, ECConnectorMetadata)
|
||||
|
||||
|
||||
def test_metadata_initialization():
|
||||
"""ECCPUConnectorMetadata initializes with empty saves and loads dicts."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
assert hasattr(meta, "saves")
|
||||
assert hasattr(meta, "loads")
|
||||
assert meta.saves == {}
|
||||
assert meta.loads == {}
|
||||
|
||||
|
||||
def test_metadata_saves_dict_operations():
|
||||
"""The saves dict supports standard dict operations."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
# Add multiple entries
|
||||
meta.saves["mm_hash_1"] = [0, 1, 2]
|
||||
meta.saves["mm_hash_2"] = [10, 11]
|
||||
|
||||
assert "mm_hash_1" in meta.saves
|
||||
assert "mm_hash_2" in meta.saves
|
||||
assert len(meta.saves) == 2
|
||||
assert meta.saves["mm_hash_1"] == [0, 1, 2]
|
||||
assert meta.saves["mm_hash_2"] == [10, 11]
|
||||
|
||||
|
||||
def test_metadata_loads_carry_a_transfer_id_and_blocks():
|
||||
"""Each load entry is a (transfer_id, block_ids) pair, so the worker can
|
||||
report the transfer rather than the mm_hash."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.loads["mm_hash_1"] = (0, [5, 6, 7])
|
||||
meta.loads["mm_hash_2"] = (1, [100])
|
||||
|
||||
assert len(meta.loads) == 2
|
||||
assert meta.loads["mm_hash_1"] == (0, [5, 6, 7])
|
||||
transfer_id, block_ids = meta.loads["mm_hash_2"]
|
||||
assert transfer_id == 1
|
||||
assert block_ids == [100]
|
||||
|
||||
|
||||
def test_metadata_saves_and_loads_are_independent():
|
||||
"""Saves and loads dicts are independent."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["key"] = [1, 2]
|
||||
meta.loads["key"] = (0, [3, 4])
|
||||
|
||||
# Same key but different dicts should not interfere
|
||||
assert meta.saves["key"] == [1, 2]
|
||||
assert meta.loads["key"] == (0, [3, 4])
|
||||
|
||||
# Modifying one should not affect the other
|
||||
meta.saves["other"] = [99]
|
||||
assert "other" not in meta.loads
|
||||
|
||||
|
||||
def test_metadata_block_indices_are_integers():
|
||||
"""Block indices in saves/loads lists should be integers."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["hash"] = [0, 100, 999, 65535]
|
||||
meta.loads["hash"] = (0, [1, 2, 3])
|
||||
|
||||
for idx in meta.saves["hash"]:
|
||||
assert isinstance(idx, int)
|
||||
for idx in meta.loads["hash"][1]:
|
||||
assert isinstance(idx, int)
|
||||
|
||||
|
||||
def test_metadata_multiple_instances_are_independent():
|
||||
"""Different ECCPUConnectorMetadata instances should not share state."""
|
||||
meta1 = ECCPUConnectorMetadata()
|
||||
meta2 = ECCPUConnectorMetadata()
|
||||
|
||||
meta1.saves["x"] = [1]
|
||||
meta2.saves["y"] = [2]
|
||||
|
||||
assert "x" in meta1.saves
|
||||
assert "x" not in meta2.saves
|
||||
assert "y" not in meta1.saves
|
||||
assert "y" in meta2.saves
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
EC_CONNECTOR_VERSION,
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
compute_ec_compatibility_hash,
|
||||
)
|
||||
|
||||
|
||||
def test_xferreq_roundtrip():
|
||||
req = XferReq(mm_hash="h1", compatibility_hash="c1", session_id="s1")
|
||||
data = msgspec.msgpack.encode(req)
|
||||
out = msgspec.msgpack.decode(data, type=XferReq)
|
||||
assert out.mm_hash == "h1"
|
||||
assert out.compatibility_hash == "c1"
|
||||
assert out.connector_version == EC_CONNECTOR_VERSION
|
||||
|
||||
|
||||
def test_xferack_roundtrip_ok():
|
||||
ack = XferAck(
|
||||
mm_hash="h1",
|
||||
status=XferStatus.OK,
|
||||
src_block_indices=[1, 2],
|
||||
agent_metadata=b"meta",
|
||||
mem_descriptor=b"desc",
|
||||
)
|
||||
out = msgspec.msgpack.decode(msgspec.msgpack.encode(ack), type=XferAck)
|
||||
assert out.status == XferStatus.OK
|
||||
assert out.src_block_indices == [1, 2]
|
||||
|
||||
|
||||
def test_compat_hash_deterministic_and_sensitive():
|
||||
a = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
|
||||
b = compute_ec_compatibility_hash("0.1", "m", "float16", 64)
|
||||
c = compute_ec_compatibility_hash("0.1", "m", "float16", 128)
|
||||
assert a == b
|
||||
assert a != c
|
||||
@@ -1,461 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
)
|
||||
|
||||
_N, _BS, _HID, _ES = 16, 64, 32, 2
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, data=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.data = data
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, features, params=None, req_id="r1"):
|
||||
self.mm_features = features
|
||||
self.request_id = req_id
|
||||
self.ec_transfer_params = params
|
||||
|
||||
|
||||
class _FakeResults:
|
||||
def __init__(self):
|
||||
self.completed = set()
|
||||
self.tombstoned = set()
|
||||
self.quarantined = set()
|
||||
self.retryable = set()
|
||||
self.settled = []
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
"""Records started transfers; lets the test flip them to completed."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = []
|
||||
self._results = _FakeResults()
|
||||
|
||||
def start_xfer(self, mm_hash, block_indices, deadline):
|
||||
self.started.append(mm_hash)
|
||||
|
||||
def poll(self, messages, now):
|
||||
pass
|
||||
|
||||
def take_results(self):
|
||||
r, self._results = self._results, _FakeResults()
|
||||
return r
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeTransport:
|
||||
"""Stand-in for ZmqClientTransport so _poll_step runs without ZMQ."""
|
||||
|
||||
def poll(self):
|
||||
return {}
|
||||
|
||||
def poll_dead(self):
|
||||
return []
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _consumer_sched(monkeypatch):
|
||||
def _region(cfg):
|
||||
return ECSharedRegion(
|
||||
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
|
||||
)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
|
||||
|
||||
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
|
||||
# below flip on the NIXL consumer state directly.
|
||||
# No measured widths: a consumer takes each shape from the announcement.
|
||||
s = ECCPUScheduler(
|
||||
create_ec_vllm_config(ec_role="ec_consumer", encoder_output_widths={})
|
||||
)
|
||||
# Turn on NIXL consumer state without constructing real transports.
|
||||
s._nixl_enabled = True
|
||||
s._transport = _FakeTransport()
|
||||
s._data = None
|
||||
s._compat_hash = "c"
|
||||
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
|
||||
return s
|
||||
|
||||
|
||||
def _params(mm_hash, length):
|
||||
return {
|
||||
mm_hash: {
|
||||
"peer_host": "h",
|
||||
"peer_port": 1,
|
||||
"size_bytes": length * _HID * _ES,
|
||||
"shape": [length, _HID],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
"""The read is sized from the announced shape, not a local width."""
|
||||
width = 4 * _HID
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
|
||||
# still reserve a real not-ready cache entry so mark_ready works.
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
assert size == width * _ES
|
||||
assert shape == (1, width)
|
||||
entry = s._cache.alloc(mm_hash, size // _BS, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
params = _params("h1", 1)
|
||||
params["h1"]["size_bytes"] = width * _ES
|
||||
params["h1"]["shape"] = [1, width]
|
||||
req = _Request([_Feature("h1", 1)], params=params)
|
||||
|
||||
# Step 1: unseen remote item -> read started, request deferred.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert "h1" in s._in_flight
|
||||
assert "h1" in fake.started
|
||||
s.build_connector_meta(scheduler_output=None) # re-arms _first_in_batch
|
||||
|
||||
# Session reports completion this step.
|
||||
fake._results.completed.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
# Step 2: poll drains completion -> entry marked ready, request admitted.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert "h1" in s._step_completed
|
||||
entry = s._cache.get("h1")
|
||||
assert entry is not None and entry.ready
|
||||
# The scheduler now allocates encoder input; the ready entry loads through
|
||||
# the same local path as a natively cached encoding.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.loads
|
||||
assert meta.loads["h1"][2] == (1, width)
|
||||
assert "h1" not in s._step_completed # cleared by promote
|
||||
|
||||
# Step 3: still cached -> admitted directly, no new transfer.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("params", [None, {"x": 123}])
|
||||
def test_no_remote_announcement_is_ready(monkeypatch, params):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
assert s.ensure_cache_available(_Request([_Feature("x")], params=params), 0)
|
||||
assert s.take_unavailable_requests() == set()
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [None, "metadata", "pixels", "embeds", "shm"])
|
||||
def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
|
||||
"""A remote miss is fatal only when local model input is unavailable."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
entry = s._cache.alloc(mm_hash, 1, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
fields = {
|
||||
"metadata": {"image_grid_thw": torch.tensor([1, 2, 2])},
|
||||
"pixels": {"pixel_values": torch.ones(1, 3)},
|
||||
"embeds": {"image_embeds": torch.ones(1, _HID)},
|
||||
"shm": {"address": 123, "monotonic_id": 1},
|
||||
}
|
||||
data = (
|
||||
MultiModalKwargsItem(
|
||||
{
|
||||
key: MultiModalFieldElem(value, MultiModalSharedField(batch_size=1))
|
||||
for key, value in fields[payload].items()
|
||||
}
|
||||
)
|
||||
if payload is not None
|
||||
else None
|
||||
)
|
||||
req = _Request([_Feature("h1", 1, data)], params=_params("h1", 1))
|
||||
|
||||
# Step 1: start the read.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
fake._results.tombstoned.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
can_fallback = payload in ("pixels", "embeds", "shm")
|
||||
assert s.ensure_cache_available(req, 0) is can_fallback
|
||||
assert s._cache.get("h1") is None
|
||||
assert "h1" not in s._in_flight
|
||||
assert s.take_unavailable_requests() == (set() if can_fallback else {"r1"})
|
||||
# Drained by the read, so the scheduler cannot abort it twice.
|
||||
assert s.take_unavailable_requests() == set()
|
||||
if can_fallback:
|
||||
# A later scheduling step must not retry the failed remote source.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.ensure_cache_available(req, 0)
|
||||
assert fake.started == ["h1"]
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("retry", [False, True], ids=["in-flight", "retryable"])
|
||||
def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
|
||||
"""Neither retries nor delayed scheduling may restart the request budget."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
def start(mm_hash, info, size, shape):
|
||||
assert s._cache.alloc(mm_hash, 1, shape) is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", start)
|
||||
now = [1000.0]
|
||||
monkeypatch.setattr(time, "monotonic", lambda: now[0])
|
||||
req = _Request([_Feature("h1")], params=_params("h1", 1))
|
||||
assert not s.ensure_cache_available(req, 0)
|
||||
for elapsed in (20, 40, 121):
|
||||
now[0] = 1000.0 + elapsed
|
||||
if retry:
|
||||
fake._results.retryable.add("h1")
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert not s.ensure_cache_available(req, 0)
|
||||
assert s.take_unavailable_requests() == ({"r1"} if elapsed > 60 else set())
|
||||
assert len(fake.started) == (3 if retry else 1)
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
"""A not-ready NACK defers the request and re-requests the read.
|
||||
|
||||
Unlike a tombstone it never admits the request, because an admitted request
|
||||
whose media was rewritten away upstream has no embedding and no way to
|
||||
recompute one.
|
||||
"""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
entry = s._cache.alloc(mm_hash, 1, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Producer's save has not landed: retryable, not tombstoned.
|
||||
fake._results.retryable.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
# The entry is released and the read re-issued in the same step, so the
|
||||
# request stays deferred rather than being admitted unfetched.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert "h1" in s._in_flight
|
||||
assert "h1" not in s._tombstones
|
||||
assert fake.started.count("h1") == 2
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fields",
|
||||
[
|
||||
{"size_bytes": 999},
|
||||
{"size_bytes": None},
|
||||
{"size_bytes": "big"},
|
||||
{"size_bytes": _MISSING},
|
||||
{"shape": _MISSING},
|
||||
{"shape": [2, _HID]},
|
||||
],
|
||||
ids=[
|
||||
"size-mismatch",
|
||||
"null-size",
|
||||
"text-size",
|
||||
"no-size",
|
||||
"no-shape",
|
||||
"wrong-rows",
|
||||
],
|
||||
)
|
||||
def test_invalid_announcement_fails_the_request(monkeypatch, fields):
|
||||
"""An unusable announced size or shape fails the request, not the engine."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
announced = {**_params("h1", 1)["h1"], **fields}
|
||||
announced = {k: v for k, v in announced.items() if v is not _MISSING}
|
||||
req = _Request([_Feature("h1", 1)], params={"h1": announced})
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert s.take_unavailable_requests() == {"r1"}
|
||||
assert "h1" not in s._in_flight
|
||||
assert s._cache.get("h1") is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_deferral_budget_is_per_request(monkeypatch):
|
||||
"""Shared hashes have independent wait budgets and cancellation cleanup."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n, shape=None: None)
|
||||
params = _params("h1", 1)
|
||||
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
|
||||
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
|
||||
|
||||
now = [1000.0]
|
||||
monkeypatch.setattr(time, "monotonic", lambda: now[0])
|
||||
assert s.ensure_cache_available(old, 0) is False
|
||||
now[0] += sched_mod._ADMIT_DEFER_TIMEOUT_S - 1
|
||||
assert s.ensure_cache_available(new, 0) is False
|
||||
assert s.take_unavailable_requests() == set()
|
||||
|
||||
now[0] += 2
|
||||
# `old` is now past its budget; `new` is not.
|
||||
assert s.ensure_cache_available(new, 0) is False
|
||||
assert s.take_unavailable_requests() == set()
|
||||
assert s.request_finished(new) == (False, None)
|
||||
assert set(s._deferred_since) == {("old", "h1")}
|
||||
assert s.ensure_cache_available(old, 0) is False
|
||||
assert s.take_unavailable_requests() == {"old"}
|
||||
assert "h1" not in s._in_flight
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
|
||||
# A producer that reported placeholder metadata but had no cache entry
|
||||
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
|
||||
# must fall back to local compute rather than treating that as a size
|
||||
# mismatch.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
metadata_only = {"h1": {"image_grid_thw": [1, 2, 3]}}
|
||||
req = _Request([_Feature("h1", 1)], params=metadata_only)
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert s._cache.get("h1") is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_orphan_not_ready_entry_defers_no_realloc(monkeypatch):
|
||||
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
|
||||
# entry exists for the mm_hash, but the hash is tracked in none of
|
||||
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
|
||||
# quarantined/settling DMA and must not be re-allocated.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
entry = s._cache.alloc("h1", 1)
|
||||
assert entry is not None and not entry.ready
|
||||
assert "h1" not in s._in_flight
|
||||
assert "h1" not in s._step_completed
|
||||
assert "h1" not in s._tombstones
|
||||
|
||||
# Spy on alloc: record calls, delegate to the real allocator (which
|
||||
# asserts on a duplicate key). A re-alloc for "h1" is the bug.
|
||||
real_alloc = s._cache.alloc
|
||||
calls: list[str] = []
|
||||
|
||||
def _spy_alloc(key, n):
|
||||
calls.append(key)
|
||||
return real_alloc(key, n)
|
||||
|
||||
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# Must not raise; the request is deferred while the DMA settles, the
|
||||
# orphan entry is untouched, and alloc is never called again.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert calls == []
|
||||
assert s._cache.get("h1") is entry
|
||||
assert not entry.ready
|
||||
assert s.take_unavailable_requests() == set()
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_already_computed_feature_admits_without_transfer(monkeypatch):
|
||||
# A feature entirely within num_computed_tokens is skipped by admit; no
|
||||
# cache lookup/alloc or transfer should be triggered for it.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
calls: list[str] = []
|
||||
real_alloc = s._cache.alloc
|
||||
|
||||
def _spy_alloc(key, n):
|
||||
calls.append(key)
|
||||
return real_alloc(key, n)
|
||||
|
||||
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
|
||||
# Feature spans [0, 4); with num_computed_tokens=4 it is fully computed.
|
||||
req = _Request([_Feature("h1", 4)], params=_params("h1", 4))
|
||||
|
||||
assert s.ensure_cache_available(req, 4) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert calls == []
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_local_ready_entry_admits_without_transfer(monkeypatch):
|
||||
# A READY entry already present in the cache is a local hit: admit
|
||||
# returns True and no new transfer is started for that mm_hash.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
entry = s._cache.alloc("h1", 1)
|
||||
assert entry is not None
|
||||
s._cache.mark_ready("h1")
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
|
||||
# A mm_hash already tracked in _in_flight must defer the request rather
|
||||
# than starting a duplicate transfer.
|
||||
s = _consumer_sched(monkeypatch)
|
||||
started: list[str] = []
|
||||
|
||||
def _spy_start_xfer(mm_hash, info, size, shape):
|
||||
started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _spy_start_xfer)
|
||||
s._in_flight.add("h1")
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert started == []
|
||||
s.shutdown()
|
||||
@@ -1,146 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
import vllm.distributed.nixl_utils as nixl_utils
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.config.ec_transfer import ECTransferConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
from vllm.distributed.nixl_utils import NixlWrapper
|
||||
|
||||
_N = 16
|
||||
_BS = 64
|
||||
|
||||
|
||||
def _region() -> ECSharedRegion:
|
||||
return ECSharedRegion(
|
||||
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
|
||||
)
|
||||
|
||||
|
||||
def test_ec_enable_nixl_absent_from_extra_config_by_default():
|
||||
cfg = ECTransferConfig()
|
||||
assert cfg.get_from_extra_config("ec_enable_nixl", False) is False
|
||||
|
||||
|
||||
def test_ec_enable_nixl_read_from_extra_config():
|
||||
cfg = ECTransferConfig(
|
||||
ec_connector="ECCPUConnector",
|
||||
ec_role="ec_both",
|
||||
ec_connector_extra_config={"ec_enable_nixl": True},
|
||||
)
|
||||
assert cfg.get_from_extra_config("ec_enable_nixl", False) is True
|
||||
|
||||
|
||||
def test_gate_off_builds_no_nixl(monkeypatch):
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
|
||||
# ec_enable_nixl defaults to False.
|
||||
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
|
||||
assert s._nixl_enabled is False
|
||||
assert getattr(s, "_data", None) is None
|
||||
assert getattr(s, "_producer_session", None) is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_string_false_in_extra_config_leaves_nixl_off(monkeypatch):
|
||||
"""Extra config is not type coerced, and bool("false") is True."""
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
|
||||
cfg = create_ec_vllm_config(ec_role="ec_both")
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = "false"
|
||||
s = ECCPUScheduler(cfg)
|
||||
assert s._nixl_enabled is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_missing_nixl_releases_shared_region(monkeypatch):
|
||||
"""A failed constructor leaves no scheduler for the caller to shut down."""
|
||||
# Import before patching to avoid caching the missing-NIXL values.
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data import nixl # noqa: F401
|
||||
|
||||
region = _region()
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
monkeypatch.setattr(nixl_utils, "NixlWrapper", None)
|
||||
monkeypatch.setattr(nixl_utils, "nixl_agent_config", None)
|
||||
cfg = create_ec_vllm_config(ec_role="ec_both")
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="ec_enable_nixl requires NIXL"):
|
||||
ECCPUScheduler(cfg)
|
||||
|
||||
assert not Path(region._mmap_path).exists()
|
||||
assert region._fd is None
|
||||
assert region._mmap_obj is None
|
||||
finally:
|
||||
region.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_control_transport_failure_releases_nixl_and_shared_region(monkeypatch):
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq as zmq_mod
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl as data_mod
|
||||
|
||||
class RegisteredData:
|
||||
registered = True
|
||||
|
||||
def deregister(self):
|
||||
self.registered = False
|
||||
|
||||
data = RegisteredData()
|
||||
error = RuntimeError("control transport failed")
|
||||
|
||||
def fail_control_transport():
|
||||
raise error
|
||||
|
||||
region = _region()
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
monkeypatch.setattr(nixl_utils, "NixlWrapper", object())
|
||||
monkeypatch.setattr(nixl_utils, "nixl_agent_config", object())
|
||||
monkeypatch.setattr(data_mod, "NixlDataTransport", lambda **kwargs: data)
|
||||
monkeypatch.setattr(zmq_mod, "ZmqClientTransport", fail_control_transport)
|
||||
cfg = create_ec_vllm_config(ec_role="ec_consumer")
|
||||
cfg.model_config.hf_config = None
|
||||
cfg.model_config.get_inputs_embeds_size.return_value = 8
|
||||
cfg.model_config.model = "test-model"
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
|
||||
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="control transport failed") as exc:
|
||||
ECCPUScheduler(cfg)
|
||||
|
||||
assert exc.value is error
|
||||
assert not data.registered
|
||||
assert not Path(region._mmap_path).exists()
|
||||
assert region._fd is None
|
||||
assert region._mmap_obj is None
|
||||
finally:
|
||||
region.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.skipif(NixlWrapper is None, reason="Requires NIXL package")
|
||||
def test_gate_on_wires_data_transport_and_producer_session(monkeypatch):
|
||||
# Port 0 lets the OS pick an ephemeral port so the real ZMQ ROUTER bind
|
||||
# in ProducerSession.start() cannot collide with another test/process.
|
||||
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_HOST", "127.0.0.1")
|
||||
monkeypatch.setenv("VLLM_EC_SIDE_CHANNEL_PORT", "0")
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: _region())
|
||||
cfg = create_ec_vllm_config(ec_role="ec_both")
|
||||
cfg.ec_transfer_config.ec_connector_extra_config["ec_enable_nixl"] = True
|
||||
s = ECCPUScheduler(cfg)
|
||||
try:
|
||||
assert s._nixl_enabled is True
|
||||
assert s._data is not None
|
||||
assert s._compat_hash is not None
|
||||
assert s._producer_session is not None
|
||||
assert s._peer_host == "127.0.0.1"
|
||||
assert s._transport is not None # is_ec_consumer is also True
|
||||
finally:
|
||||
s.shutdown()
|
||||
@@ -1,116 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
|
||||
_N, _BS = 16, 64
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.data = None
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, features, req_id="r1"):
|
||||
self.mm_features = features
|
||||
self.request_id = req_id
|
||||
|
||||
|
||||
def _sched_gate_off(monkeypatch):
|
||||
def _region(cfg):
|
||||
return ECSharedRegion(
|
||||
engine_id="eng-" + str(uuid.uuid4()), num_blocks=_N, block_size_bytes=_BS
|
||||
)
|
||||
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", _region)
|
||||
return ECCPUScheduler(create_ec_vllm_config(ec_role="ec_both"))
|
||||
|
||||
|
||||
def test_request_finished_gate_off_returns_none(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
assert s.request_finished(_Request([_Feature("h1")])) == (False, None)
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_request_finished_producer_emits_params(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
|
||||
entry = s._cache.alloc("h1", 2, (2, 32))
|
||||
assert entry is not None
|
||||
s._cache.mark_ready("h1")
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
assert entry.evictable
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
"item_indices": [0],
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
"shape": [2, 32],
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
"""A save whose GPU->mmap copy hasn't been confirmed complete yet is
|
||||
still announced: the entry can't be evicted before it's ready, and a read
|
||||
arriving too early is NACKed NACK_NOT_READY for the consumer to retry —
|
||||
protecting it is not this method's job."""
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._cache.alloc("h1", 2, (2, 32)) # allocated but not marked ready
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
"item_indices": [0],
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
"shape": [2, 32],
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_request_finished_skips_unallocated_entry(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
# No alloc() for "h1" at all — e.g. the cache was full at save time.
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
# The item's placeholder metadata is still reported even though there's
|
||||
# no cache entry to transfer (empty "metadata": no fields, no transfer).
|
||||
assert params == {"h1": {"metadata": {}, "item_indices": [0]}}
|
||||
s.shutdown()
|
||||
@@ -1,471 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ProducerXfer, ProducerSession, ConsumerXfer, ConsumerSession."""
|
||||
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
_CONSUMER_QUARANTINE_TIMEOUT_S,
|
||||
ConsumerSession,
|
||||
ConsumerXfer,
|
||||
ProducerSession,
|
||||
ProducerXfer,
|
||||
XferState,
|
||||
)
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_data(xfer_state: str = "PROC") -> MagicMock:
|
||||
data = MagicMock()
|
||||
data.check_xfer_state.return_value = xfer_state
|
||||
data.post_read.return_value = 99
|
||||
return data
|
||||
|
||||
|
||||
def _ok_ack(
|
||||
mm_hash: str = "h1",
|
||||
session_id: str = "sess-1",
|
||||
src_indices: list | None = None,
|
||||
) -> XferAck:
|
||||
return XferAck(
|
||||
mm_hash=mm_hash,
|
||||
status=XferStatus.OK,
|
||||
session_id=session_id,
|
||||
src_block_indices=src_indices or [0, 1],
|
||||
agent_metadata=b"meta",
|
||||
mem_descriptor=b"desc",
|
||||
)
|
||||
|
||||
|
||||
def _xfer(
|
||||
data: MagicMock | None = None,
|
||||
deadline: float | None = None,
|
||||
consumer_session_id: str = "sess-1",
|
||||
) -> ConsumerXfer:
|
||||
return ConsumerXfer(
|
||||
mm_hash="h1",
|
||||
block_indices=[10, 11],
|
||||
addr=("host", 1234),
|
||||
deadline=deadline if deadline is not None else time.monotonic() + 60,
|
||||
data=data or _make_data(),
|
||||
consumer_session_id=consumer_session_id,
|
||||
)
|
||||
|
||||
|
||||
def _started(
|
||||
data: MagicMock | None = None, deadline: float | None = None
|
||||
) -> ConsumerXfer:
|
||||
"""ConsumerXfer that has already received an OK XferAck."""
|
||||
x = _xfer(data, deadline)
|
||||
x.handle_ack(_ok_ack(session_id="sess-1", src_indices=[5, 6]), agent_name="agent-1")
|
||||
return x
|
||||
|
||||
|
||||
# ── ProducerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_producer_xfer_not_expired_before_deadline():
|
||||
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() + 60)
|
||||
assert not x.is_expired()
|
||||
|
||||
|
||||
def test_producer_xfer_expired_after_deadline():
|
||||
x = ProducerXfer("h1", [0, 1], deadline=time.monotonic() - 1)
|
||||
assert x.is_expired()
|
||||
|
||||
|
||||
# ── ProducerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _xfer_req(mm_hash: str = "h1", session_id: str = "sess-1") -> XferReq:
|
||||
return XferReq(
|
||||
mm_hash=mm_hash, compatibility_hash="hash-abc", session_id=session_id
|
||||
)
|
||||
|
||||
|
||||
def _make_producer_session(cache: EmbeddingCache | None = None) -> ProducerSession:
|
||||
data = _make_data()
|
||||
data.get_agent_metadata.return_value = b"meta"
|
||||
data.get_mem_descriptor.return_value = b"desc"
|
||||
return ProducerSession(
|
||||
transport=MagicMock(),
|
||||
data=data,
|
||||
cache=cache or EmbeddingCache(num_blocks=8),
|
||||
compat_hash="hash-abc",
|
||||
)
|
||||
|
||||
|
||||
def test_producer_session_grant_pins_ready_entry_and_returns_blocks():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
|
||||
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
|
||||
|
||||
assert ack.status == XferStatus.OK
|
||||
entry = cache.get("h1")
|
||||
assert not entry.evictable # pinned by the grant
|
||||
assert list(ack.src_block_indices) == list(entry.block_ids)
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_missing_entry():
|
||||
s = _make_producer_session()
|
||||
ack = s._grant_or_nack(_xfer_req(mm_hash="nope"))
|
||||
assert ack.status == XferStatus.NACK_MISSING
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_not_ready_entry_as_retryable():
|
||||
"""An announced entry whose save is still in flight is distinguished from a
|
||||
miss, so the consumer retries instead of falling back to local compute."""
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2) # not ready
|
||||
s = _make_producer_session(cache)
|
||||
ack = s._grant_or_nack(_xfer_req(mm_hash="h1"))
|
||||
assert ack.status == XferStatus.NACK_NOT_READY
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_version_mismatch():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
req = XferReq(
|
||||
mm_hash="h1",
|
||||
compatibility_hash="hash-abc",
|
||||
session_id="sess-1",
|
||||
connector_version=-1,
|
||||
)
|
||||
|
||||
ack = s._grant_or_nack(req)
|
||||
|
||||
assert ack.status == XferStatus.NACK_VERSION
|
||||
entry = cache.get("h1")
|
||||
assert entry.evictable # nothing was pinned
|
||||
|
||||
|
||||
def test_producer_session_grant_nacks_compat_hash_mismatch():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
req = XferReq(mm_hash="h1", compatibility_hash="wrong-hash", session_id="sess-1")
|
||||
|
||||
ack = s._grant_or_nack(req)
|
||||
|
||||
assert ack.status == XferStatus.NACK_INCOMPAT
|
||||
entry = cache.get("h1")
|
||||
assert entry.evictable # nothing was pinned
|
||||
|
||||
|
||||
def test_producer_session_notif_unpins_via_cache():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
|
||||
entry = cache.get("h1")
|
||||
assert not entry.evictable # pinned
|
||||
|
||||
s._data.get_new_notifs.return_value = {"agent": [b"sess-1:h1"]}
|
||||
s._drain_notifs()
|
||||
|
||||
assert entry.evictable # unpinned via cache
|
||||
assert s._active_xfers == {}
|
||||
|
||||
|
||||
def test_producer_session_sweep_timeouts_unpins_via_cache():
|
||||
cache = EmbeddingCache(num_blocks=8)
|
||||
cache.alloc("h1", 2)
|
||||
cache.mark_ready("h1")
|
||||
s = _make_producer_session(cache)
|
||||
s._grant_or_nack(_xfer_req(mm_hash="h1", session_id="sess-1"))
|
||||
key = next(iter(s._active_xfers))
|
||||
s._active_xfers[key].deadline = time.monotonic() - 1
|
||||
|
||||
s._sweep_timeouts()
|
||||
|
||||
assert cache.get("h1").evictable
|
||||
assert s._active_xfers == {}
|
||||
|
||||
|
||||
# ── ConsumerXfer.handle_ack ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_ok_starts_read():
|
||||
data = _make_data()
|
||||
x = _xfer(data)
|
||||
assert x.handle_ack(_ok_ack(), agent_name="agent-1") is True
|
||||
assert x.transfer_handle == 99
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_notif_msg_is_session_id_colon_mm_hash():
|
||||
data = _make_data()
|
||||
x = _xfer(data, consumer_session_id="my-sess")
|
||||
x.handle_ack(_ok_ack(mm_hash="h1"), agent_name="agent-1")
|
||||
_, kwargs = data.post_read.call_args
|
||||
assert kwargs["notif_msg"] == b"my-sess:h1"
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_forwards_correct_args():
|
||||
data = _make_data()
|
||||
x = _xfer(data)
|
||||
x.handle_ack(_ok_ack(src_indices=[5, 6]), agent_name="agent-1")
|
||||
args, _ = data.post_read.call_args
|
||||
assert args[0] == [10, 11] # local block_indices
|
||||
assert args[1] == "agent-1" # agent_name (not a raw handle)
|
||||
assert args[2] == [5, 6] # remote src_block_indices
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_resets_deadline():
|
||||
original = time.monotonic() + 2
|
||||
x = _xfer(deadline=original)
|
||||
x.handle_ack(_ok_ack(), agent_name="agent-1")
|
||||
assert x.deadline > original
|
||||
|
||||
|
||||
def test_consumer_xfer_handle_ack_nack_returns_false():
|
||||
data = _make_data()
|
||||
x = _xfer(data)
|
||||
assert (
|
||||
x.handle_ack(XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING), "a")
|
||||
is False
|
||||
)
|
||||
assert x.transfer_handle is None
|
||||
data.post_read.assert_not_called()
|
||||
|
||||
|
||||
# ── ConsumerXfer.poll — WAITING_ACK ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_waiting_ack_before_deadline():
|
||||
assert _xfer().poll(time.monotonic()) == XferState.WAITING_ACK
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_waiting_ack_timeout():
|
||||
assert (
|
||||
_xfer(deadline=time.monotonic() - 1).poll(time.monotonic())
|
||||
== XferState.ACK_TIMEOUT
|
||||
)
|
||||
|
||||
|
||||
# ── ConsumerXfer.poll — READING ───────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_proc():
|
||||
assert _started().poll(time.monotonic()) == XferState.READING
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_done_releases_handle():
|
||||
data = _make_data(xfer_state="DONE")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic()) == XferState.DONE
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
assert x.transfer_handle is None
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_unexpected_state_fails():
|
||||
data = _make_data(xfer_state="ERR")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic()) == XferState.READ_FAILED
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_reading_exception_fails_gracefully():
|
||||
data = _make_data()
|
||||
data.check_xfer_state.side_effect = RuntimeError("nixl crash")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic()) == XferState.READ_FAILED
|
||||
|
||||
|
||||
def test_consumer_xfer_poll_timeout_quarantines_without_releasing():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
assert x.poll(time.monotonic() + 9999) == XferState.QUARANTINED
|
||||
data.release_xfer_handle.assert_not_called()
|
||||
assert x.transfer_handle == 99
|
||||
|
||||
|
||||
# ── ConsumerXfer.poll — QUARANTINED ───────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_proc_stays():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
x.poll(time.monotonic() + 9999) # → QUARANTINED
|
||||
assert x.poll(time.monotonic()) == XferState.QUARANTINED
|
||||
data.release_xfer_handle.assert_not_called()
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_terminal_settles():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
x.poll(time.monotonic() + 9999) # → QUARANTINED
|
||||
data.check_xfer_state.return_value = "DONE"
|
||||
assert x.poll(time.monotonic()) == XferState.SETTLED
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
assert x.transfer_handle is None
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_exception_settles():
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
x.poll(time.monotonic() + 9999)
|
||||
data.check_xfer_state.side_effect = RuntimeError("gone")
|
||||
assert x.poll(time.monotonic()) == XferState.SETTLED
|
||||
|
||||
|
||||
def test_consumer_xfer_quarantined_timeout_evicts_blocks():
|
||||
"""Quarantine timeout should release handle and return READ_FAILED."""
|
||||
data = _make_data(xfer_state="PROC")
|
||||
x = _started(data)
|
||||
# First poll transitions to QUARANTINED (deadline extended by 60s)
|
||||
x.poll(time.monotonic() + 9999)
|
||||
# Second poll with expired quarantine deadline should evict
|
||||
expired_time = time.monotonic() + 9999 + _CONSUMER_QUARANTINE_TIMEOUT_S + 1
|
||||
assert x.poll(expired_time) == XferState.READ_FAILED
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
assert x.transfer_handle is None
|
||||
|
||||
|
||||
# ── ConsumerXfer.cancel / release ────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_consumer_xfer_cancel_valid_in_waiting_ack():
|
||||
_xfer().cancel() # must not raise
|
||||
|
||||
|
||||
def test_consumer_xfer_cancel_raises_when_read_started():
|
||||
with pytest.raises(AssertionError):
|
||||
_started().cancel()
|
||||
|
||||
|
||||
def test_consumer_xfer_release_releases_handle():
|
||||
data = _make_data()
|
||||
x = _started(data)
|
||||
x.release()
|
||||
data.release_xfer_handle.assert_called_once_with(99)
|
||||
|
||||
|
||||
def test_consumer_xfer_release_noop_when_no_handle():
|
||||
data = _make_data()
|
||||
_xfer(data).release()
|
||||
data.release_xfer_handle.assert_not_called()
|
||||
|
||||
|
||||
# ── ConsumerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_consumer_session(data=None):
|
||||
zmq_conn = MagicMock()
|
||||
zmq_conn.recv.return_value = []
|
||||
return ConsumerSession(
|
||||
addr=("host", 1234),
|
||||
zmq_conn=zmq_conn,
|
||||
transport=MagicMock(),
|
||||
data=data or _make_data(),
|
||||
compat_hash="hash-abc",
|
||||
)
|
||||
|
||||
|
||||
def test_consumer_session_has_stable_session_id():
|
||||
s = _make_consumer_session()
|
||||
assert isinstance(s._session_id, str) and len(s._session_id) > 0
|
||||
assert s._session_id == s._session_id # stable
|
||||
|
||||
|
||||
def test_consumer_session_start_xfer_sends_req_with_session_id():
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
raw = s._zmq.send.call_args[0][0]
|
||||
req = msgspec.msgpack.decode(raw, type=XferReq)
|
||||
assert req.mm_hash == "h1"
|
||||
assert req.session_id == s._session_id
|
||||
|
||||
|
||||
def test_consumer_session_poll_done_goes_to_completed():
|
||||
import msgspec
|
||||
|
||||
data = _make_data(xfer_state="PROC")
|
||||
s = _make_consumer_session(data)
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
|
||||
ack = XferAck(
|
||||
mm_hash="h1",
|
||||
status=XferStatus.OK,
|
||||
session_id=s._session_id,
|
||||
src_block_indices=[5, 6],
|
||||
agent_metadata=b"meta",
|
||||
mem_descriptor=b"desc",
|
||||
)
|
||||
data.add_remote_peer.return_value = "agent-1"
|
||||
raw_ack = msgspec.msgpack.encode(ack)
|
||||
|
||||
data.check_xfer_state.return_value = "DONE"
|
||||
s.poll([raw_ack], time.monotonic())
|
||||
|
||||
results = s.take_results()
|
||||
assert "h1" in results.completed
|
||||
assert not results.tombstoned
|
||||
|
||||
|
||||
def test_consumer_session_poll_nack_goes_to_tombstoned():
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_MISSING)
|
||||
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
|
||||
results = s.take_results()
|
||||
assert "h1" in results.tombstoned
|
||||
assert not results.completed
|
||||
|
||||
|
||||
def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
|
||||
"""A not-ready NACK must not tombstone: tombstoning admits the request with
|
||||
no embedding, which is fatal once the media has been rewritten away."""
|
||||
import msgspec
|
||||
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 10)
|
||||
nack = XferAck(mm_hash="h1", status=XferStatus.NACK_NOT_READY)
|
||||
s.poll([msgspec.msgpack.encode(nack)], time.monotonic())
|
||||
results = s.take_results()
|
||||
assert "h1" in results.retryable
|
||||
assert not results.tombstoned
|
||||
assert not results.completed
|
||||
|
||||
|
||||
def test_consumer_session_ack_timeout_reports_retry_once():
|
||||
"""A delayed producer ACK is retryable, and its result is drained once."""
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
|
||||
s.poll([], time.monotonic())
|
||||
r1 = s.take_results()
|
||||
assert "h1" in r1.retryable
|
||||
assert not r1.tombstoned
|
||||
r2 = s.take_results()
|
||||
assert not r2.retryable # cleared after first take
|
||||
|
||||
|
||||
def test_consumer_session_on_peer_down_cancels_waiting_ack():
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() + 60)
|
||||
s.on_peer_down()
|
||||
results = s.take_results()
|
||||
assert "h1" in results.retryable
|
||||
assert not results.tombstoned
|
||||
@@ -1,54 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECCPUConnector scheduler utilities."""
|
||||
|
||||
import msgspec
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
build_block_descs,
|
||||
deserialize_mem_descriptor,
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
|
||||
# ── build_block_descs ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_id,expected_dev", [(7, 7), (None, 0)])
|
||||
def test_build_block_descs(device_id, expected_dev):
|
||||
kwargs = dict(base_ptr=1000, num_blocks=4, block_size_bytes=256)
|
||||
if device_id is not None:
|
||||
kwargs["device_id"] = device_id
|
||||
descs = build_block_descs(**kwargs)
|
||||
assert len(descs) == 4
|
||||
for i, (addr, size, dev) in enumerate(descs):
|
||||
assert addr == 1000 + i * 256
|
||||
assert size == 256
|
||||
assert dev == expected_dev
|
||||
|
||||
|
||||
def test_build_block_descs_zero_blocks_returns_empty():
|
||||
assert build_block_descs(base_ptr=100, num_blocks=0, block_size_bytes=64) == []
|
||||
|
||||
|
||||
# ── serialize / deserialize_mem_descriptor ───────────────────────────────────
|
||||
|
||||
|
||||
def test_mem_descriptor_roundtrip():
|
||||
descs = [(100, 64, 0), (164, 64, 0), (228, 64, 1)]
|
||||
assert deserialize_mem_descriptor(serialize_mem_descriptor(descs)) == descs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_value",
|
||||
[
|
||||
[(1, 2)], # 2-tuple instead of 3
|
||||
[("a", "b", "c")], # strings, not ints
|
||||
],
|
||||
)
|
||||
def test_mem_descriptor_rejects_malformed_payload(bad_value):
|
||||
"""Malformed descriptor lists must fail to decode."""
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
bad_payload = encoder.encode(bad_value)
|
||||
with pytest.raises((msgspec.DecodeError, msgspec.ValidationError)):
|
||||
deserialize_mem_descriptor(bad_payload)
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the V2 GPU model runner's EC connector wrapper."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT
|
||||
from vllm.v1.worker.ec_connector_model_runner_mixin import (
|
||||
ECConnectorModelRunnerMixin,
|
||||
)
|
||||
from vllm.v1.worker.gpu.ec_connector import NO_OP_EC_CONNECTOR, ActiveECConnector
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
WORKER_META = object()
|
||||
CONNECTOR_STATS = object()
|
||||
|
||||
|
||||
def _scheduler_output() -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
ec_connector_metadata=ECConnectorMetadata(), finished_req_ids=frozenset()
|
||||
)
|
||||
|
||||
|
||||
def _connector(
|
||||
encoder_cache: dict | None = None,
|
||||
is_producer: bool = True,
|
||||
is_consumer: bool = False,
|
||||
) -> tuple[ActiveECConnector, MagicMock]:
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
fake.is_producer = is_producer
|
||||
fake.is_consumer = is_consumer
|
||||
fake.get_finished.return_value = (None, None)
|
||||
fake.build_connector_worker_meta.return_value = WORKER_META
|
||||
fake.get_ec_connector_stats.return_value = CONNECTOR_STATS
|
||||
with patch("vllm.v1.worker.gpu.ec_connector.get_ec_transfer", return_value=fake):
|
||||
return ActiveECConnector(SimpleNamespace(), encoder_cache or {}), fake
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("is_producer", "is_consumer"), [(True, False), (True, True), (False, True)]
|
||||
)
|
||||
def test_saves_newly_added_caches_for_every_producer(is_producer, is_consumer):
|
||||
"""An ec_both node is also a producer: it must offload what it just computed."""
|
||||
encoder_cache = {"mm_old": None}
|
||||
connector, fake = _connector(encoder_cache, is_producer, is_consumer)
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()):
|
||||
encoder_cache["mm_new"] = None
|
||||
|
||||
saved = [call.kwargs["mm_hash"] for call in fake.save_caches.call_args_list]
|
||||
assert saved == (["mm_new"] if is_producer else [])
|
||||
assert fake.start_save_caches.called == is_producer
|
||||
if is_producer:
|
||||
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
|
||||
assert fake.start_load_caches.called == is_consumer
|
||||
|
||||
|
||||
def test_worker_meta_is_reported_on_context_exit():
|
||||
"""Reported in the finally block, so is_empty() sees it only after the exit."""
|
||||
connector, fake = _connector()
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()) as output:
|
||||
assert output.ec_connector_worker_meta is None
|
||||
|
||||
assert output.ec_connector_worker_meta is WORKER_META
|
||||
assert fake.clear_connector_metadata.called
|
||||
|
||||
|
||||
def test_connector_stats_are_reported_on_context_exit():
|
||||
"""Reported in the finally block, alongside the worker meta."""
|
||||
connector, fake = _connector()
|
||||
|
||||
with connector.maybe_get_output(_scheduler_output()) as output:
|
||||
assert output.ec_connector_stats is None
|
||||
|
||||
assert output.ec_connector_stats is CONNECTOR_STATS
|
||||
|
||||
|
||||
def test_v1_producer_receives_existing_encoder_cache():
|
||||
encoder_cache = {"mm_cached": None}
|
||||
fake = MagicMock(spec=ECConnectorBase)
|
||||
fake.is_producer = True
|
||||
fake.is_consumer = False
|
||||
fake.get_finished.return_value = (None, None)
|
||||
|
||||
module = "vllm.v1.worker.ec_connector_model_runner_mixin"
|
||||
with (
|
||||
patch(f"{module}.has_ec_transfer", return_value=True),
|
||||
patch(f"{module}.get_ec_transfer", return_value=fake),
|
||||
ECConnectorModelRunnerMixin.maybe_get_ec_connector_output(
|
||||
_scheduler_output(), encoder_cache
|
||||
),
|
||||
):
|
||||
pass
|
||||
|
||||
assert fake.start_save_caches.call_args.kwargs["encoder_cache"] is encoder_cache
|
||||
|
||||
|
||||
def test_no_forward_reports_without_running_the_model():
|
||||
connector, _ = _connector()
|
||||
|
||||
output = connector.no_forward(_scheduler_output())
|
||||
|
||||
assert output.ec_connector_output.ec_connector_worker_meta is WORKER_META
|
||||
|
||||
empty = NO_OP_EC_CONNECTOR.no_forward(_scheduler_output())
|
||||
assert empty is EMPTY_MODEL_RUNNER_OUTPUT
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared config builders for EC connector unit tests."""
|
||||
|
||||
import uuid
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.config.ec_transfer import ECRole, ECTransferConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
|
||||
# The scheduler derives its per-load ack count from the PCP size only under the
|
||||
# `mp` backend -- which also keeps the world size off the local GPU count, so
|
||||
# TP > 1 stays expressible on a single-GPU host.
|
||||
_EXECUTOR_BACKEND = "mp"
|
||||
_PCP_SIZE = 1
|
||||
|
||||
|
||||
_EMBED_WIDTH = 32
|
||||
|
||||
|
||||
def create_ec_vllm_config(
|
||||
*,
|
||||
ec_role: ECRole = "ec_both",
|
||||
tensor_parallel_size: int = 1,
|
||||
rank: int = 0,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
encoder_output_widths: dict[str, int] | None = None,
|
||||
) -> Mock:
|
||||
"""Build a `VllmConfig` stand-in for EC connector unit tests.
|
||||
|
||||
`ec_transfer_config` and `parallel_config` are the real config objects, so
|
||||
role derivation and the rank arithmetic the connector depends on behave as
|
||||
they do in production.
|
||||
|
||||
`model_config` supplies a dtype and embedding width without resolving an
|
||||
HF model or inspecting its architecture.
|
||||
|
||||
Args:
|
||||
ec_role: EC role this instance plays. `is_ec_producer` and
|
||||
`is_ec_consumer` are derived from it, as in production.
|
||||
tensor_parallel_size: TP degree.
|
||||
rank: Global rank of this worker.
|
||||
dtype: Encoder cache dtype.
|
||||
encoder_output_widths: Per-modality encoder output width, as a
|
||||
producer measures it at startup. Defaults to the embedding width.
|
||||
|
||||
Returns:
|
||||
A `VllmConfig`-specced mock carrying the configs above.
|
||||
|
||||
"""
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
prefill_context_parallel_size=_PCP_SIZE,
|
||||
distributed_executor_backend=_EXECUTOR_BACKEND,
|
||||
rank=rank,
|
||||
)
|
||||
ec_transfer_config = ECTransferConfig(
|
||||
# The role properties report False unless a connector is named.
|
||||
ec_connector="ECCPUConnector",
|
||||
ec_role=ec_role,
|
||||
engine_id=str(uuid.uuid4()),
|
||||
)
|
||||
ec_transfer_config.mm_encoder_output_widths = (
|
||||
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
|
||||
if encoder_output_widths is None
|
||||
else encoder_output_widths
|
||||
)
|
||||
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.dtype = dtype
|
||||
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = ec_transfer_config
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = model_config
|
||||
vllm_config.instance_id = f"ec-test-{uuid.uuid4()}"
|
||||
return vllm_config
|
||||
Reference in New Issue
Block a user