[Mini-VLLM] Remove upstream CI, docs, and lint surfaces
- Delete .buildkite: fork runs no Buildkite pipelines - Delete mkdocs.yaml, .readthedocs.yaml: docs/ is gone - Delete rust-toolchain.toml: Rust frontend is gone - Delete .pre-commit-config.yaml: all hooks call deleted tools/pre_commit - Delete .dockerignore: docker/ is gone - Drop three tests importing deleted quark and mooncake modules Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,168 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.quantization.quark.utils import deep_compare
|
||||
|
||||
|
||||
def test_cuda_empty_vs_unset_configs(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test that configs created with normal (untouched) CUDA_VISIBLE_DEVICES
|
||||
and CUDA_VISIBLE_DEVICES="" are equivalent. This ensures consistent
|
||||
behavior regardless of whether GPU visibility is disabled via empty string
|
||||
or left in its normal state.
|
||||
"""
|
||||
|
||||
def create_config():
|
||||
engine_args = EngineArgs(
|
||||
model="deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True
|
||||
)
|
||||
return engine_args.create_engine_config()
|
||||
|
||||
# Create config with CUDA_VISIBLE_DEVICES set normally
|
||||
normal_config = create_config()
|
||||
|
||||
# Create config with CUDA_VISIBLE_DEVICES=""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("CUDA_VISIBLE_DEVICES", "")
|
||||
empty_config = create_config()
|
||||
|
||||
normal_config_dict = vars(normal_config)
|
||||
empty_config_dict = vars(empty_config)
|
||||
|
||||
# Remove instance_id before comparison as it's expected to be different
|
||||
normal_config_dict.pop("instance_id", None)
|
||||
empty_config_dict.pop("instance_id", None)
|
||||
|
||||
assert deep_compare(normal_config_dict, empty_config_dict), (
|
||||
'Configs with normal CUDA_VISIBLE_DEVICES and CUDA_VISIBLE_DEVICES=""'
|
||||
" should be equivalent"
|
||||
)
|
||||
|
||||
|
||||
def test_ray_runtime_env(monkeypatch: pytest.MonkeyPatch):
|
||||
# In testing, this method needs to be nested inside as ray does not
|
||||
# see the test module.
|
||||
def create_config():
|
||||
engine_args = EngineArgs(
|
||||
model="deepseek-ai/DeepSeek-V2-Lite", trust_remote_code=True
|
||||
)
|
||||
return engine_args.create_engine_config()
|
||||
|
||||
config = create_config()
|
||||
parallel_config = config.parallel_config
|
||||
assert parallel_config.ray_runtime_env is None
|
||||
|
||||
import ray
|
||||
|
||||
ray.init()
|
||||
|
||||
runtime_env = {
|
||||
"env_vars": {
|
||||
"TEST_ENV_VAR": "test_value",
|
||||
# In future ray versions, this will be default, so when setting a
|
||||
# task or actor with num_gpus=None/0, the visible devices env var
|
||||
# won't be overridden resulting in no GPUs being visible on a gpu
|
||||
# machine.
|
||||
"RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO": "0",
|
||||
},
|
||||
}
|
||||
|
||||
config_ref = ray.remote(create_config).options(runtime_env=runtime_env).remote()
|
||||
|
||||
config = ray.get(config_ref)
|
||||
parallel_config = config.parallel_config
|
||||
assert parallel_config.ray_runtime_env is not None
|
||||
assert (
|
||||
parallel_config.ray_runtime_env.env_vars().get("TEST_ENV_VAR") == "test_value"
|
||||
)
|
||||
|
||||
ray.shutdown()
|
||||
|
||||
|
||||
def test_unrecognized_env(monkeypatch):
|
||||
import os
|
||||
|
||||
from vllm.envs import environment_variables
|
||||
|
||||
# Remove any existing unrecognized VLLM env vars that might interfere
|
||||
for env in list(os.environ):
|
||||
if env.startswith("VLLM_") and env not in environment_variables:
|
||||
monkeypatch.delenv(env, raising=False)
|
||||
|
||||
# Test that if fail_on_environ_validation is True, then an error
|
||||
# is raised when an unrecognized vLLM environment variable is set
|
||||
monkeypatch.setenv("VLLM_UNRECOGNIZED_ENV_VAR", "some_value")
|
||||
engine_args = EngineArgs(
|
||||
fail_on_environ_validation=True,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Unknown vLLM environment variable detected"):
|
||||
engine_args.create_engine_config()
|
||||
|
||||
# Test that if fail_on_environ_validation is False, then no error is raised
|
||||
engine_args = EngineArgs()
|
||||
engine_args.create_engine_config()
|
||||
|
||||
# Test that when the unrecognized env var is removed, no error is raised
|
||||
monkeypatch.delenv("VLLM_UNRECOGNIZED_ENV_VAR")
|
||||
engine_args = EngineArgs(
|
||||
fail_on_environ_validation=True,
|
||||
)
|
||||
engine_args.create_engine_config()
|
||||
|
||||
|
||||
def test_get_diff_sampling_param_includes_penalties():
|
||||
"""``--override-generation-config`` must apply presence_penalty and
|
||||
frequency_penalty as server-side defaults, like the other sampling
|
||||
params. Regression test for the whitelist silently dropping them.
|
||||
|
||||
Called unbound with a lightweight stand-in so no model is loaded:
|
||||
generation_config="vllm" makes the method skip try_get_generation_config
|
||||
and read only override_generation_config.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
|
||||
fake = SimpleNamespace(
|
||||
generation_config="vllm",
|
||||
override_generation_config={
|
||||
"presence_penalty": 1.5,
|
||||
"frequency_penalty": 0.5,
|
||||
"temperature": 0.6,
|
||||
},
|
||||
)
|
||||
|
||||
diff = ModelConfig.get_diff_sampling_param(fake)
|
||||
|
||||
assert diff["presence_penalty"] == 1.5
|
||||
assert diff["frequency_penalty"] == 0.5
|
||||
assert diff["temperature"] == 0.6
|
||||
|
||||
|
||||
def test_generation_config_penalties_survive_diff_sampling_filter():
|
||||
"""``generation_config.json`` penalties must survive the diff-sampling
|
||||
whitelist (regression for #50767, auto path).
|
||||
|
||||
Covers the ``generation_config="auto"`` branch, where the override
|
||||
dict is empty and the values come from ``try_get_generation_config``.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
|
||||
fake = SimpleNamespace(
|
||||
generation_config="auto",
|
||||
override_generation_config={},
|
||||
try_get_generation_config=lambda: {
|
||||
"presence_penalty": 1.1,
|
||||
"frequency_penalty": 0.7,
|
||||
"temperature": 0.9,
|
||||
},
|
||||
)
|
||||
|
||||
diff = ModelConfig.get_diff_sampling_param(fake)
|
||||
|
||||
assert diff["presence_penalty"] == 1.1
|
||||
assert diff["frequency_penalty"] == 0.7
|
||||
assert diff["temperature"] == 0.9
|
||||
@@ -1,194 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests quantization configuration matching utilities."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
|
||||
should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
_find_matching_targets,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.utils import (
|
||||
should_ignore_layer as quark_should_ignore_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.config_utils import (
|
||||
find_matching_patterns,
|
||||
get_layer_name_after_index,
|
||||
is_equal_or_regex_match,
|
||||
)
|
||||
|
||||
|
||||
def test_is_equal_or_regex_match():
|
||||
assert is_equal_or_regex_match(
|
||||
"model.layers.0.mlp.down_proj", "model.layers.0.mlp.down_proj"
|
||||
)
|
||||
assert is_equal_or_regex_match("model.layers.0.mlp.down_proj", r"re:.*down_proj")
|
||||
assert not is_equal_or_regex_match("model.layers.0.mlp.down_proj", "other")
|
||||
|
||||
|
||||
def test_fnmatch_is_opt_in():
|
||||
layer_name = "model.layers.0.mlp.experts"
|
||||
pattern = "*mlp.experts*"
|
||||
|
||||
assert not is_equal_or_regex_match(layer_name, pattern)
|
||||
assert is_equal_or_regex_match(layer_name, pattern, use_fnmatch=True)
|
||||
assert not should_ignore_layer(layer_name, [pattern])
|
||||
assert not quark_should_ignore_layer(layer_name, [pattern])
|
||||
assert should_ignore_layer(layer_name, [pattern], use_fnmatch=True)
|
||||
|
||||
|
||||
def test_online_targets_support_fnmatch_patterns():
|
||||
layer_name = "model.layers.0.mlp.experts"
|
||||
targets = {"*mlp.experts*": "mxfp4"}
|
||||
|
||||
assert _find_matching_targets(layer_name, targets) == ["*mlp.experts*"]
|
||||
|
||||
|
||||
def test_online_ignore_supports_fnmatch_patterns():
|
||||
layer_name = "model.layers.0.mlp.experts"
|
||||
ignore = ["*mlp.experts*"]
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={"*mlp.experts*": "mxfp4"}, ignore=ignore)
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="matches both quantization_config.ignore"):
|
||||
config.resolve_quant_method_cls(LinearBase, layer_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"layer_name,expected",
|
||||
[
|
||||
("model.layers.1.self_attn.qkv_proj", "self_attn.qkv_proj"),
|
||||
("model.layers.2.mlp.down_proj", "mlp.down_proj"),
|
||||
("lm_head", "lm_head"),
|
||||
],
|
||||
)
|
||||
def test_get_layer_name_after_index(layer_name, expected):
|
||||
assert get_layer_name_after_index(layer_name) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns",
|
||||
[
|
||||
[r"re:.*qkv_proj.*"],
|
||||
[r"re:.*\.[qkv]_proj$"],
|
||||
],
|
||||
ids=["direct_fused_regex", "fused_shard_regexes"],
|
||||
)
|
||||
def test_find_matching_patterns_for_fused_regexes(patterns):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
matches = find_matching_patterns(layer_name, patterns, fused_mapping)
|
||||
assert all(len(shard_matches) == 1 for shard_matches in matches)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns,expected",
|
||||
[
|
||||
(
|
||||
[r"re:.*\.qkv_proj$"],
|
||||
[{r"re:.*\.qkv_proj$"}],
|
||||
),
|
||||
(
|
||||
[r"re:.*\.q_proj$", r"re:.*\.k_proj$", r"re:.*\.v_proj$"],
|
||||
[
|
||||
{r"re:.*\.q_proj$"},
|
||||
{r"re:.*\.k_proj$"},
|
||||
{r"re:.*\.v_proj$"},
|
||||
],
|
||||
),
|
||||
],
|
||||
ids=["direct_fused_match", "individual_shard_matches"],
|
||||
)
|
||||
def test_find_matching_patterns_distinguishes_direct_and_shard_matches(
|
||||
patterns, expected
|
||||
):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert find_matching_patterns(layer_name, patterns, fused_mapping) == expected
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
|
||||
|
||||
def test_should_ignore_layer_rejects_partially_matched_fused_layer():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
patterns = [r"re:.*\.q_proj$"]
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert find_matching_patterns(layer_name, patterns, fused_mapping) == [
|
||||
{r"re:.*\.q_proj$"},
|
||||
set(),
|
||||
set(),
|
||||
]
|
||||
with pytest.raises(ValueError, match="different quantization schemes"):
|
||||
should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
|
||||
|
||||
def test_should_ignore_layer_returns_false_when_no_fused_pattern_matches():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert find_matching_patterns(layer_name, [], fused_mapping) == [
|
||||
set(),
|
||||
set(),
|
||||
set(),
|
||||
]
|
||||
assert not should_ignore_layer(layer_name, [], fused_mapping)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns",
|
||||
[
|
||||
[r"re:.*qkv_proj.*"],
|
||||
[r"re:.*\.[qkv]_proj$"],
|
||||
],
|
||||
ids=["direct_fused_regex", "fused_shard_regexes"],
|
||||
)
|
||||
def test_ignore_and_targets_match_fused_regexes_identically(patterns):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
targets = {pattern: "fp8_per_block" for pattern in patterns}
|
||||
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
matches = _find_matching_targets(layer_name, targets, fused_mapping)
|
||||
assert len(matches) == 1
|
||||
assert targets[matches[0]] == "fp8_per_block"
|
||||
|
||||
|
||||
def test_ignore_allows_individually_matched_fused_shards():
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
patterns = [
|
||||
"model.layers.0.self_attn.q_proj",
|
||||
"model.layers.0.self_attn.k_proj",
|
||||
"model.layers.0.self_attn.v_proj",
|
||||
]
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"patterns",
|
||||
[
|
||||
["model.layers.0.self_attn.qkv_proj"],
|
||||
[r"re:.*\.qkv_proj$"],
|
||||
[
|
||||
"model.layers.0.self_attn.q_proj",
|
||||
"model.layers.0.self_attn.k_proj",
|
||||
"model.layers.0.self_attn.v_proj",
|
||||
],
|
||||
],
|
||||
ids=["direct_fused_name", "direct_fused_regex", "individual_shards"],
|
||||
)
|
||||
def test_quark_and_compressed_tensors_ignore_fused_layers_identically(patterns):
|
||||
layer_name = "model.layers.0.self_attn.qkv_proj"
|
||||
fused_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
assert should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
assert quark_should_ignore_layer(layer_name, patterns, fused_mapping)
|
||||
@@ -1,629 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Body handling across the EPD proxy's decode retries.
|
||||
|
||||
Exercises the REAL helpers loaded from the ``examples/`` proxy, so a future
|
||||
change to them is what these tests catch.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import msgspec
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
from aiohttp.test_utils import TestServer
|
||||
|
||||
PROXY_REL = "examples/disaggregated/disaggregated_encoder/disagg_epd_proxy.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def proxy():
|
||||
path = Path(__file__).parents[4] / PROXY_REL
|
||||
spec = importlib.util.spec_from_file_location("disagg_epd_proxy_retry", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class _Response:
|
||||
def __init__(self, params):
|
||||
self._params = params
|
||||
|
||||
async def read(self):
|
||||
return msgspec.json.encode({"kv_transfer_params": self._params})
|
||||
|
||||
|
||||
def test_maybe_prefill_leaves_the_caller_body_untouched(proxy, monkeypatch):
|
||||
"""A decode retry re-enters this function with the body it was given.
|
||||
|
||||
Mutating that body in place let one attempt's `remote_block_ids` survive
|
||||
into the next, so a retry whose prefill returns nothing sent decode blocks
|
||||
the prefiller may already have freed.
|
||||
"""
|
||||
served = [{"remote_block_ids": [1, 2]}, {}]
|
||||
|
||||
async def _stage(req_data, p_url, req_id, dp_rank=None):
|
||||
assert "kv_transfer_params" not in req_data
|
||||
return _Response(served.pop(0))
|
||||
|
||||
monkeypatch.setattr(proxy, "process_prefill_stage", _stage)
|
||||
|
||||
body = {"messages": [], "stream": False}
|
||||
first = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
|
||||
assert first["kv_transfer_params"] == {"remote_block_ids": [1, 2]}
|
||||
assert "kv_transfer_params" not in body
|
||||
|
||||
second = asyncio.run(proxy.maybe_prefill(body, "http://prefill", "r1"))
|
||||
assert "kv_transfer_params" not in second
|
||||
|
||||
|
||||
class _EncoderResponse:
|
||||
def __init__(self, params):
|
||||
self.status = 200
|
||||
self._params = params
|
||||
|
||||
async def read(self):
|
||||
return msgspec.json.encode({"ec_transfer_params": self._params})
|
||||
|
||||
async def text(self):
|
||||
return ""
|
||||
|
||||
|
||||
class _EncoderSession:
|
||||
"""Serve one canned encoder reply per attempt."""
|
||||
|
||||
def __init__(self, replies):
|
||||
self._replies = list(replies)
|
||||
|
||||
async def post(self, url, data=None, headers=None):
|
||||
return _EncoderResponse(self._replies.pop(0))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("second_type", ["image_url", "audio_url", "input_audio"])
|
||||
@pytest.mark.parametrize("transfer", ["example", "nixl", "mooncake"])
|
||||
def test_partial_audio_metadata_falls_back_with_all_transfers(
|
||||
proxy, monkeypatch, second_type, transfer
|
||||
):
|
||||
"""Never mix metadata-only audio with raw media, or lose EC identities."""
|
||||
import copy
|
||||
|
||||
seen = []
|
||||
reported = {}
|
||||
|
||||
class Session:
|
||||
async def post(self, url, data=None, headers=None):
|
||||
body = msgspec.json.decode(data)
|
||||
seen.append(body)
|
||||
item = body["messages"][0]["content"][0]
|
||||
mm_hash = item["uuid"] + "-processed"
|
||||
entry: dict[str, Any] = {
|
||||
"metadata": {"audio_feature_lengths": [100]} if len(seen) == 1 else {},
|
||||
"item_indices": [0],
|
||||
}
|
||||
if transfer == "nixl":
|
||||
entry.update(peer_host="encoder", peer_port=1234, size_bytes=4096)
|
||||
reported[mm_hash] = entry
|
||||
return _EncoderResponse({mm_hash: entry})
|
||||
|
||||
second = (
|
||||
{"type": second_type, "input_audio": {"data": "large", "format": "wav"}}
|
||||
if second_type == "input_audio"
|
||||
else {"type": second_type, second_type: {"url": "large"}}
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {"url": "small"},
|
||||
"uuid": "user-audio",
|
||||
},
|
||||
second,
|
||||
],
|
||||
}
|
||||
],
|
||||
"mm_processor_kwargs": {"sampling_rate": 16000},
|
||||
}
|
||||
original = copy.deepcopy(body)
|
||||
monkeypatch.setattr(proxy, "encode_session", Session())
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", False)
|
||||
consumer = "tcp://consumer:1" if transfer == "mooncake" else None
|
||||
prepared, _, _ = asyncio.run(
|
||||
proxy.prepare_for_decode(body, "r", ["http://encoder"], "", consumer)
|
||||
)
|
||||
assert body == original
|
||||
assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"]
|
||||
content = prepared["messages"][0]["content"]
|
||||
assert [item["type"] for item in content] == ["audio_url", second_type]
|
||||
assert content[0]["uuid"] == "user-audio"
|
||||
for item, encoded in zip(content, seen):
|
||||
assert item == encoded["messages"][0]["content"][0]
|
||||
params = prepared["ec_transfer_params"]
|
||||
for mm_hash, entry in reported.items():
|
||||
assert params[mm_hash] == entry
|
||||
if transfer == "mooncake":
|
||||
assert params["ec_items"] == [
|
||||
{
|
||||
"mm_hash": mm_hash,
|
||||
"transfer_id": encoded["ec_transfer_params"]["ec_items"][0][
|
||||
"transfer_id"
|
||||
],
|
||||
}
|
||||
for mm_hash, encoded in zip(reported, seen)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("no_rewrite", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size, expected_sizes", [(0, [3, 3]), (1, [1] * 6), (2, [2, 2, 1, 1])]
|
||||
)
|
||||
def test_image_batches_preserve_item_identity(
|
||||
proxy, monkeypatch, no_rewrite, batch_size, expected_sizes
|
||||
):
|
||||
"""Rehashed and repeated images retain metadata and per-occurrence transfers."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.scheduler import (
|
||||
ECMooncakeScheduler,
|
||||
)
|
||||
|
||||
seen = []
|
||||
|
||||
class Session:
|
||||
async def post(self, url, data=None, headers=None):
|
||||
body = msgspec.json.decode(data)
|
||||
seen.append((url, body))
|
||||
params: dict[str, dict[str, Any]] = {}
|
||||
for index, item in enumerate(body["messages"][0]["content"]):
|
||||
image = item["image_url"]["url"]
|
||||
entry = params.setdefault(
|
||||
image + "-processed",
|
||||
{
|
||||
"metadata": {"image_grid_thw": [1, 2, ord(image)]},
|
||||
"item_indices": [],
|
||||
"peer_port": 1234,
|
||||
},
|
||||
)
|
||||
entry["item_indices"].append(index)
|
||||
# Response ordering must not affect the image-to-metadata mapping.
|
||||
return _EncoderResponse(dict(reversed(list(params.items()))))
|
||||
|
||||
monkeypatch.setattr(proxy, "encode_session", Session())
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite)
|
||||
monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", batch_size)
|
||||
monkeypatch.setattr(proxy, "encoder_rr_idx", 0)
|
||||
images = ["A", "A", "C", "D", "A", "F"]
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image}}
|
||||
for image in images
|
||||
],
|
||||
}
|
||||
],
|
||||
"mm_processor_kwargs": {"max_pixels": 262144},
|
||||
}
|
||||
meta, handles = asyncio.run(
|
||||
proxy.fanout_encoder_primer(
|
||||
body, ["http://e0", "http://e1"], "r", "tcp://consumer:1"
|
||||
)
|
||||
)
|
||||
assert [len(b["messages"][0]["content"]) for _, b in seen] == expected_sizes
|
||||
assert (
|
||||
len(
|
||||
{
|
||||
x["transfer_id"]
|
||||
for _, b in seen
|
||||
for x in b["ec_transfer_params"]["ec_items"]
|
||||
}
|
||||
)
|
||||
== 6
|
||||
)
|
||||
for url, batch in seen:
|
||||
assert batch["mm_processor_kwargs"] == body["mm_processor_kwargs"]
|
||||
for item in batch["messages"][0]["content"]:
|
||||
assert item["image_url"]["url"] in (
|
||||
{"A", "C"} if url.startswith("http://e0/") else {"A", "D", "F"}
|
||||
)
|
||||
assert [meta[i]["image_grid_thw"][-1] for i in range(6)] == list(map(ord, images))
|
||||
assert [meta[i]["ec_mm_hash"] for i in range(6)] == [
|
||||
x + "-processed" for x in images
|
||||
]
|
||||
assert handles["A-processed"]["peer_port"] == 1234
|
||||
assert "ec_transfer_params" not in body
|
||||
|
||||
transfers_by_encoder: list[list[dict[str, str]]] = [[], []]
|
||||
for url, batch in seen:
|
||||
rank = 0 if url.startswith("http://e0/") else 1
|
||||
transfers_by_encoder[rank].extend(batch["ec_transfer_params"]["ec_items"])
|
||||
expected_transfers = [
|
||||
transfers_by_encoder[i % 2][i // 2]["transfer_id"] for i in range(len(images))
|
||||
]
|
||||
params = (
|
||||
handles
|
||||
if no_rewrite
|
||||
else proxy.rewrite_for_decode(body, meta)["ec_transfer_params"]
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
ec_transfer_params=params,
|
||||
mm_features=[
|
||||
SimpleNamespace(identifier=image + "-processed") for image in images
|
||||
],
|
||||
)
|
||||
assert [
|
||||
ECMooncakeScheduler._request_transfer_id(request, i) for i in range(len(images))
|
||||
] == expected_transfers
|
||||
|
||||
|
||||
def test_batch_rejects_ambiguous_metadata(proxy, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
proxy,
|
||||
"encode_session",
|
||||
_EncoderSession([{"unknown": {"metadata": {"image_grid_thw": [1, 2, 2]}}}]),
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": image}}
|
||||
for image in ["A", "B"]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
with pytest.raises(proxy.HTTPException, match="cannot be matched"):
|
||||
asyncio.run(proxy.fanout_encoder_primer(body, ["http://e0"], "r"))
|
||||
|
||||
|
||||
def test_a_decode_retry_does_not_inherit_the_previous_handles(proxy, monkeypatch):
|
||||
"""The retry loop re-enters `prepare_for_decode` with the same body.
|
||||
|
||||
Recording the encoder's connector handles on that body in place let
|
||||
attempt 1's handle survive into attempt 2, so a second encode that
|
||||
reported nothing still sent decode a handle on an embedding the encoder
|
||||
no longer publishes -- the exact state the retry exists to leave behind.
|
||||
"""
|
||||
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}, "peer_port": 1234}
|
||||
monkeypatch.setattr(
|
||||
proxy,
|
||||
"encode_session",
|
||||
_EncoderSession([{"encoder-side-hash": handle}, {}]),
|
||||
)
|
||||
|
||||
async def _no_prefill(req_data, p_url, req_id, dp_rank=None):
|
||||
return req_data
|
||||
|
||||
monkeypatch.setattr(proxy, "maybe_prefill", _no_prefill)
|
||||
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
],
|
||||
"stream": False,
|
||||
}
|
||||
args = ("r1", ["http://encoder"], "http://prefill", None)
|
||||
|
||||
first, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
reported = first["ec_transfer_params"]
|
||||
assert {key: value for key, value in reported.items() if key != "ec_items"} == {
|
||||
"encoder-side-hash": handle
|
||||
}
|
||||
assert "ec_transfer_params" not in body
|
||||
|
||||
# Attempt 2's encode reports nothing: decode must be told nothing.
|
||||
second, _, _ = asyncio.run(proxy.prepare_for_decode(body, *args))
|
||||
assert "ec_transfer_params" not in second
|
||||
|
||||
|
||||
def test_raw_media_keeps_encoder_transfer_identity(proxy, monkeypatch):
|
||||
handle = {"metadata": {"image_grid_thw": [1, 2, 2]}}
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", True)
|
||||
monkeypatch.setattr(
|
||||
proxy, "encode_session", _EncoderSession([{"encoded-hash": handle}])
|
||||
)
|
||||
body = {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "image"}}],
|
||||
}
|
||||
]
|
||||
}
|
||||
prepared, _, _ = asyncio.run(
|
||||
proxy.prepare_for_decode(
|
||||
body, "request", ["http://encoder"], "", "tcp://consumer:1"
|
||||
)
|
||||
)
|
||||
assert prepared["messages"] == body["messages"]
|
||||
assert "ec_transfer_params" not in body
|
||||
params = prepared["ec_transfer_params"]
|
||||
assert params["encoded-hash"] == handle
|
||||
assert params["ec_items"][0]["mm_hash"] == "encoded-hash"
|
||||
assert params["ec_items"][0]["transfer_id"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("prefill", [False, True])
|
||||
@pytest.mark.parametrize("dynamic", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"request_options",
|
||||
[
|
||||
{},
|
||||
{
|
||||
"mm_processor_kwargs": {"max_pixels": 262144},
|
||||
"media_io_kwargs": {"image": {"image_mode": "RGB"}},
|
||||
"priority": -3,
|
||||
"session_id": "session-123",
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_http_roundtrip_preserves_payload_and_response_bytes(
|
||||
proxy, monkeypatch, stream, prefill, dynamic, request_options
|
||||
):
|
||||
"""Preserve media and scheduling context across HTTP hops and retries."""
|
||||
seen: dict[str, list[dict]] = {"encode": [], "prefill": [], "decode": []}
|
||||
prefix = b'data: {"content":"'
|
||||
# Split a multibyte character at the old 1024-byte forwarding boundary.
|
||||
expected = (
|
||||
prefix + b"x" * (1023 - len(prefix)) + '图片🌍"}\n\ndata: [DONE]\n\n'.encode()
|
||||
if stream
|
||||
else '{ "choices": [{"message": {"content": "图片🌍"}}] }\n'.encode()
|
||||
)
|
||||
|
||||
async def backend(request):
|
||||
stage = request.match_info["stage"]
|
||||
if dynamic:
|
||||
consumer = "prefill" if prefill else "decode"
|
||||
expected_rank = "0" if stage == consumer else None
|
||||
assert request.headers.get("X-data-parallel-rank") == expected_rank
|
||||
assert request.content_type == "application/json"
|
||||
body = await request.json()
|
||||
seen[stage].append(body)
|
||||
if stage == "encode":
|
||||
params: dict[str, dict[str, Any]] = {}
|
||||
for index, item in enumerate(body["messages"][0]["content"]):
|
||||
mm_hash = item["uuid"]
|
||||
if body.get("mm_processor_kwargs"):
|
||||
mm_hash += "-processed"
|
||||
entry = params.setdefault(
|
||||
mm_hash,
|
||||
{
|
||||
"metadata": {"image_grid_thw": [[1, 2, 2]]},
|
||||
"peer_port": len(seen[stage]),
|
||||
"item_indices": [],
|
||||
},
|
||||
)
|
||||
entry["item_indices"].append(index)
|
||||
return web.json_response({"ec_transfer_params": params})
|
||||
if stage == "prefill":
|
||||
assert body["max_tokens"] == 1 and body["stream"] is False
|
||||
return web.json_response(
|
||||
{"kv_transfer_params": {"remote_block_ids": [len(seen[stage])]}}
|
||||
)
|
||||
if len(seen[stage]) == 1:
|
||||
return web.Response(status=500, text="retry")
|
||||
return web.Response(
|
||||
body=expected,
|
||||
content_type="text/event-stream" if stream else "application/json",
|
||||
)
|
||||
|
||||
backend_app = web.Application()
|
||||
backend_app.router.add_post("/{stage}/v1/chat/completions", backend)
|
||||
async with TestServer(backend_app) as server:
|
||||
for stage, field in [
|
||||
("encode", "e_urls"),
|
||||
("prefill", "p_urls"),
|
||||
("decode", "d_urls"),
|
||||
]:
|
||||
urls = [str(server.make_url(f"/{stage}"))]
|
||||
if stage == "prefill" and not prefill:
|
||||
urls = []
|
||||
monkeypatch.setattr(proxy.app.state, field, urls, raising=False)
|
||||
monkeypatch.setattr(proxy.app.state, "d_ec_urls", [], raising=False)
|
||||
monkeypatch.setattr(proxy.app.state, "ec_consumer_dp_size", 1, raising=False)
|
||||
monkeypatch.setattr(proxy, "DECODE_RETRIES", 1)
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", False)
|
||||
target_app = proxy.app
|
||||
if dynamic:
|
||||
monkeypatch.setenv("ADMIN_API_KEY", "test-key")
|
||||
target_app = proxy.build_app(proxy.EPDProxyConfig(probe_interval=0))
|
||||
item = {"type": "image_url", "image_url": {"url": "data:image/png;base64,YWJj"}}
|
||||
body = {
|
||||
"model": "test",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "描述图片🌍"},
|
||||
item,
|
||||
item,
|
||||
],
|
||||
}
|
||||
],
|
||||
"stream": stream,
|
||||
"temperature": 0.01,
|
||||
"max_tokens": 32,
|
||||
"seed": 42,
|
||||
"structured_outputs": {"choice": ["A", "B"]},
|
||||
**request_options,
|
||||
}
|
||||
await proxy.on_startup()
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
transport=httpx.ASGITransport(app=target_app), base_url="http://proxy"
|
||||
) as client:
|
||||
if dynamic:
|
||||
registrations = [("encode", "encode")]
|
||||
registrations += (
|
||||
[("prefill", "prefill"), ("decode", "decode")]
|
||||
if prefill
|
||||
else [("prefill_decode", "decode")]
|
||||
)
|
||||
for role, stage in registrations:
|
||||
consumer = "prefill" if prefill else "decode"
|
||||
registered = await client.post(
|
||||
"/instances",
|
||||
headers={"X-API-Key": "test-key"},
|
||||
json={
|
||||
"role": role,
|
||||
"url": str(server.make_url(f"/{stage}")),
|
||||
"dp_size": 2 if stage == consumer else 1,
|
||||
},
|
||||
)
|
||||
assert registered.status_code == 200
|
||||
response = await client.post("/v1/chat/completions", json=body)
|
||||
if dynamic:
|
||||
removed = await client.delete(
|
||||
"/instances",
|
||||
headers={"X-API-Key": "test-key"},
|
||||
params={"url": str(server.make_url("/decode"))},
|
||||
)
|
||||
assert removed.json() == {"removed": True}
|
||||
unavailable = await client.post("/v1/chat/completions", json=body)
|
||||
assert unavailable.status_code == 503
|
||||
assert response.status_code == 200
|
||||
assert response.content == expected
|
||||
assert response.headers["content-type"].startswith(
|
||||
"text/event-stream" if stream else "application/json"
|
||||
)
|
||||
finally:
|
||||
await proxy.on_shutdown()
|
||||
|
||||
assert len(seen["encode"]) == 2
|
||||
assert len(seen["decode"]) == 2
|
||||
for requests in seen.values():
|
||||
for forwarded in requests:
|
||||
for key in (
|
||||
"mm_processor_kwargs",
|
||||
"media_io_kwargs",
|
||||
"priority",
|
||||
"session_id",
|
||||
):
|
||||
assert forwarded.get(key) == body.get(key)
|
||||
final = seen["decode"][-1]
|
||||
for key in [
|
||||
"model",
|
||||
"stream",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"seed",
|
||||
"structured_outputs",
|
||||
]:
|
||||
assert final[key] == body[key]
|
||||
content = final["messages"][0]["content"]
|
||||
assert content[0] == body["messages"][0]["content"][0]
|
||||
for reference in content[1:]:
|
||||
assert reference == {
|
||||
"type": "image_embeds",
|
||||
"image_embeds": {"image_grid_thw": [1, 2, 2]},
|
||||
"uuid": proxy.content_uuid(item),
|
||||
}
|
||||
mm_hash = proxy.content_uuid(item)
|
||||
if request_options:
|
||||
mm_hash += "-processed"
|
||||
assert set(final["ec_transfer_params"]) - {"ec_items"} == {mm_hash}
|
||||
assert final["ec_transfer_params"][mm_hash]["peer_port"] == 2
|
||||
if prefill:
|
||||
assert final["kv_transfer_params"] == {"remote_block_ids": [2]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("server_keep_alive", ["0.1", "1", "5", "2", "30"])
|
||||
def test_pooled_connections_are_retired_before_the_server_closes_them(
|
||||
proxy, monkeypatch, server_keep_alive
|
||||
):
|
||||
"""The proxy must not hand a request a connection the server has dropped.
|
||||
|
||||
vLLM closes idle keep-alive connections at `VLLM_HTTP_TIMEOUT_KEEP_ALIVE`
|
||||
seconds while aiohttp pools them for 15, so a slow hop leaves a dead
|
||||
connection in the pool. The next request fails with
|
||||
ServerDisconnectedError and the server logs nothing, because it closed the
|
||||
socket before the request arrived.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", server_keep_alive)
|
||||
monkeypatch.setattr(proxy.app.state, "p_urls", [], raising=False)
|
||||
|
||||
asyncio.run(proxy.on_startup())
|
||||
try:
|
||||
# No public accessor for the pool's idle timeout.
|
||||
pooled_for = proxy.encode_session.connector._keepalive_timeout
|
||||
assert pooled_for < float(server_keep_alive)
|
||||
assert proxy.decode_session.connector._keepalive_timeout == pooled_for
|
||||
finally:
|
||||
asyncio.run(proxy.on_shutdown())
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"no_rewrite, transfer",
|
||||
[(False, None), (True, None), (False, "push"), (False, "handle")],
|
||||
)
|
||||
def test_video_audio_fallback(proxy, monkeypatch, no_rewrite, transfer):
|
||||
"""Keep raw video and neighboring images, but never discard transfer handles."""
|
||||
import copy
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import collect_ec_item_metadata
|
||||
|
||||
video = {"type": "video_url", "video_url": {"url": "video-with-audio"}}
|
||||
images = [{"type": "image_url", "image_url": {"url": key}} for key in ("A", "B")]
|
||||
body: dict[str, Any] = {
|
||||
"messages": [{"content": [images[0], video, images[1]]}],
|
||||
"mm_processor_kwargs": {"use_audio_in_video": True},
|
||||
}
|
||||
original = copy.deepcopy(body)
|
||||
# Collector indices describe two processed features from one video item.
|
||||
video_metadata = collect_ec_item_metadata(
|
||||
[
|
||||
SimpleNamespace(identifier=key, modality=key, data=None)
|
||||
for key in ("video", "audio")
|
||||
],
|
||||
SimpleNamespace(fields_for=lambda _: set()),
|
||||
)
|
||||
if transfer == "handle":
|
||||
video_metadata["audio"]["transfer_id"] = "reservation"
|
||||
image_metadata = {
|
||||
key: {"metadata": {"image_grid_thw": [1, 2, size]}, "item_indices": [i]}
|
||||
for i, (key, size) in enumerate((("A", 2), ("B", 4)))
|
||||
}
|
||||
# Images form the first group, the singleton video the second.
|
||||
monkeypatch.setattr(proxy, "ENCODER_MAX_BATCH_SIZE", 0)
|
||||
monkeypatch.setattr(proxy, "NO_REWRITE", no_rewrite)
|
||||
replies = _EncoderSession([image_metadata, video_metadata])
|
||||
monkeypatch.setattr(proxy, "encode_session", replies)
|
||||
consumer = "tcp://consumer:1" if transfer == "push" else None
|
||||
preparation = proxy.prepare_for_decode(body, "r", ["http://e0"], "", consumer)
|
||||
if transfer:
|
||||
with pytest.raises(proxy.HTTPException, match="cannot be matched"):
|
||||
asyncio.run(preparation)
|
||||
return
|
||||
|
||||
prepared, _, _ = asyncio.run(preparation)
|
||||
assert body == original
|
||||
assert prepared["mm_processor_kwargs"] == body["mm_processor_kwargs"]
|
||||
final = prepared["messages"][0]["content"]
|
||||
assert final[1] == video
|
||||
handles = prepared["ec_transfer_params"]
|
||||
assert not video_metadata.keys() & handles.keys()
|
||||
if no_rewrite:
|
||||
assert final == original["messages"][0]["content"]
|
||||
assert "ec_items" not in handles
|
||||
else:
|
||||
for position, image, key in ((0, images[0], "A"), (2, images[1], "B")):
|
||||
assert final[position] == {
|
||||
"type": "image_embeds",
|
||||
"image_embeds": image_metadata[key]["metadata"],
|
||||
"uuid": proxy.content_uuid(image),
|
||||
}
|
||||
assert [item["mm_hash"] for item in handles["ec_items"]] == ["A", "B"]
|
||||
Reference in New Issue
Block a user