[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 15:59:54 -07:00
committed by GitHub
parent 38ebd97bca
commit b30291cf0c
22 changed files with 904 additions and 45 deletions
+174
View File
@@ -4,10 +4,13 @@
from unittest.mock import MagicMock, patch
import pytest
import torch
from transformers import PretrainedConfig
from vllm.config.ec_transfer import ECRole, ECTransferConfig
from vllm.config.model import ModelConfig
from vllm.config.multimodal import MultiModalConfig
from vllm.config.vllm import VllmConfig
from vllm.transformers_utils.model_arch_config_convertor import (
ModelArchConfigConvertorBase,
)
@@ -173,3 +176,174 @@ def test_sticky_cache_survives_text_subconfig_regeneration():
side_effect=AssertionError("must not re-query registry"),
):
assert text_config._supports_multimodal_for_mm_prefix() is False
@pytest.mark.parametrize(
("device", "expected"),
[
(None, None),
("cpu", "cpu"),
("cuda", "cuda"),
# Callers compare against a bare device type, so an indexed device and a
# torch.device must normalise to the same thing as "cuda".
("cuda:1", "cuda"),
(torch.device("cuda", 1), "cuda"),
],
)
def test_mm_processor_device_type_normalizes(device: object, expected: str | None):
kwargs = {} if device is None else {"device": device}
config = MultiModalConfig(mm_processor_kwargs=kwargs)
assert config.get_mm_processor_device_type() == expected
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
ec_config = (
None
if ec_role is None
# `is_ec_producer`/`is_ec_consumer` are False without a connector, so an
# unset one would make every role look like "no EC role at all".
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
)
MultiModalConfig(
mm_processor_kwargs={"device": device}
).validate_mm_processor_device(ec_config)
@pytest.mark.parametrize("ec_role", [None, "ec_producer"])
def test_bad_mm_processor_device_rejected(ec_role: ECRole | None):
"""A bad device must fail during startup, not mid-request.
Rejected for every role, and on a CPU-only platform too, so a typo can never
silently fall through to running the processor somewhere unintended.
"""
with (
patch("vllm.platforms.current_platform.device_type", "cpu"),
pytest.raises(ValueError, match='Invalid "device" in mm_processor_kwargs'),
):
_validate_mm_processor_device(device="not-a-device", ec_role=ec_role)
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
def test_accelerator_mm_processor_rejected_outside_encoder_instance(
ec_role: ECRole | None,
):
"""Only an encode-only EPD instance has the device to itself.
Every other role runs the language model in the same process, so frontend
accelerator work would contend with the forward pass and allocate outside
the memory profiled for the KV cache.
"""
with (
patch("vllm.platforms.current_platform.device_type", "cuda"),
pytest.raises(ValueError, match="also runs the language model"),
):
_validate_mm_processor_device(device="cuda", ec_role=ec_role)
def test_accelerator_mm_processor_allowed_on_encoder_instance():
with patch("vllm.platforms.current_platform.device_type", "cuda"):
_validate_mm_processor_device(device="cuda", ec_role="ec_producer")
def test_cpu_mm_processor_needs_no_ec_role():
"""The gate only applies to the accelerator, so CPU stays unrestricted."""
with patch("vllm.platforms.current_platform.device_type", "cuda"):
_validate_mm_processor_device(device="cpu", ec_role=None)
@pytest.mark.parametrize(
("flag", "kwargs", "expected"),
[
# "auto" is only settled once the EC role is known, so it must leave no
# device behind for `VllmConfig` to mistake for an explicit request.
("auto", None, None),
(None, None, None),
("cpu", None, "cpu"),
("cuda", None, "cuda"),
# Any explicit value other than "cpu" means "whatever this platform
# calls its accelerator".
("gpu", None, "cuda"),
# An explicit device in the kwargs wins over the convenience flag.
("cpu", {"device": "cuda"}, "cuda"),
("cuda", {"device": "cpu"}, "cpu"),
],
)
def test_fold_mm_processor_device(
flag: str | None, kwargs: dict | None, expected: str | None
):
with patch("vllm.platforms.current_platform.device_type", "cuda"):
folded = MultiModalConfig.fold_mm_processor_device(kwargs, flag)
config = MultiModalConfig(mm_processor_kwargs=folded or {})
assert config.get_mm_processor_device_type() == expected
def _resolve_mm_processor_device(
*,
ec_role: ECRole | None,
mm_tensor_ipc: str = "torch_shm",
device: str | None = None,
) -> str | None:
"""Run the `auto` resolution and report where the processor ended up."""
mm_config = MultiModalConfig(
mm_processor_kwargs={} if device is None else {"device": device},
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
)
model_config = MagicMock(spec=ModelConfig)
model_config.multimodal_config = mm_config
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.model_config = model_config
vllm_config.ec_transfer_config = (
None
if ec_role is None
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
)
with patch("vllm.platforms.current_platform.device_type", "cuda"):
VllmConfig._resolve_mm_processor_device(vllm_config)
return mm_config.get_mm_processor_device_type()
def test_auto_mm_processor_device_uses_accelerator_on_encoder_instance():
"""The one deployment that has the device to itself and can hand it over."""
assert _resolve_mm_processor_device(ec_role="ec_producer") == "cuda"
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
def test_auto_mm_processor_device_stays_on_cpu_off_encoder_instance(
ec_role: ECRole | None,
):
"""Every other role runs the language model in the same process."""
assert _resolve_mm_processor_device(ec_role=ec_role) is None
def test_auto_mm_processor_device_needs_a_device_capable_transport():
"""Other transports serialize host bytes, so the output is copied back."""
assert (
_resolve_mm_processor_device(ec_role="ec_producer", mm_tensor_ipc="direct_rpc")
is None
)
def test_auto_mm_processor_device_leaves_an_explicit_request_alone():
assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu"
def test_vllm_config_runs_the_mm_processor_device_check():
"""Startup must reach the check; the rule itself is covered above.
Guards the wiring, which no other test would notice going missing.
"""
model_config = MagicMock(spec=ModelConfig)
model_config.multimodal_config = MultiModalConfig(
mm_processor_kwargs={"device": "cuda"}
)
vllm_config = MagicMock(spec=VllmConfig)
vllm_config.model_config = model_config
vllm_config.ec_transfer_config = None
with (
patch("vllm.platforms.current_platform.device_type", "cuda"),
pytest.raises(ValueError, match="also runs the language model"),
):
VllmConfig._validate_mm_processor_device(vllm_config)