[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -4,10 +4,13 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.ec_transfer import ECRole, ECTransferConfig
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.config.vllm import VllmConfig
|
||||
from vllm.transformers_utils.model_arch_config_convertor import (
|
||||
ModelArchConfigConvertorBase,
|
||||
)
|
||||
@@ -173,3 +176,174 @@ def test_sticky_cache_survives_text_subconfig_regeneration():
|
||||
side_effect=AssertionError("must not re-query registry"),
|
||||
):
|
||||
assert text_config._supports_multimodal_for_mm_prefix() is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("device", "expected"),
|
||||
[
|
||||
(None, None),
|
||||
("cpu", "cpu"),
|
||||
("cuda", "cuda"),
|
||||
# Callers compare against a bare device type, so an indexed device and a
|
||||
# torch.device must normalise to the same thing as "cuda".
|
||||
("cuda:1", "cuda"),
|
||||
(torch.device("cuda", 1), "cuda"),
|
||||
],
|
||||
)
|
||||
def test_mm_processor_device_type_normalizes(device: object, expected: str | None):
|
||||
kwargs = {} if device is None else {"device": device}
|
||||
config = MultiModalConfig(mm_processor_kwargs=kwargs)
|
||||
assert config.get_mm_processor_device_type() == expected
|
||||
|
||||
|
||||
def _validate_mm_processor_device(*, device: str, ec_role: ECRole | None) -> None:
|
||||
ec_config = (
|
||||
None
|
||||
if ec_role is None
|
||||
# `is_ec_producer`/`is_ec_consumer` are False without a connector, so an
|
||||
# unset one would make every role look like "no EC role at all".
|
||||
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
|
||||
)
|
||||
MultiModalConfig(
|
||||
mm_processor_kwargs={"device": device}
|
||||
).validate_mm_processor_device(ec_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", [None, "ec_producer"])
|
||||
def test_bad_mm_processor_device_rejected(ec_role: ECRole | None):
|
||||
"""A bad device must fail during startup, not mid-request.
|
||||
|
||||
Rejected for every role, and on a CPU-only platform too, so a typo can never
|
||||
silently fall through to running the processor somewhere unintended.
|
||||
"""
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.device_type", "cpu"),
|
||||
pytest.raises(ValueError, match='Invalid "device" in mm_processor_kwargs'),
|
||||
):
|
||||
_validate_mm_processor_device(device="not-a-device", ec_role=ec_role)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
|
||||
def test_accelerator_mm_processor_rejected_outside_encoder_instance(
|
||||
ec_role: ECRole | None,
|
||||
):
|
||||
"""Only an encode-only EPD instance has the device to itself.
|
||||
|
||||
Every other role runs the language model in the same process, so frontend
|
||||
accelerator work would contend with the forward pass and allocate outside
|
||||
the memory profiled for the KV cache.
|
||||
"""
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.device_type", "cuda"),
|
||||
pytest.raises(ValueError, match="also runs the language model"),
|
||||
):
|
||||
_validate_mm_processor_device(device="cuda", ec_role=ec_role)
|
||||
|
||||
|
||||
def test_accelerator_mm_processor_allowed_on_encoder_instance():
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
_validate_mm_processor_device(device="cuda", ec_role="ec_producer")
|
||||
|
||||
|
||||
def test_cpu_mm_processor_needs_no_ec_role():
|
||||
"""The gate only applies to the accelerator, so CPU stays unrestricted."""
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
_validate_mm_processor_device(device="cpu", ec_role=None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("flag", "kwargs", "expected"),
|
||||
[
|
||||
# "auto" is only settled once the EC role is known, so it must leave no
|
||||
# device behind for `VllmConfig` to mistake for an explicit request.
|
||||
("auto", None, None),
|
||||
(None, None, None),
|
||||
("cpu", None, "cpu"),
|
||||
("cuda", None, "cuda"),
|
||||
# Any explicit value other than "cpu" means "whatever this platform
|
||||
# calls its accelerator".
|
||||
("gpu", None, "cuda"),
|
||||
# An explicit device in the kwargs wins over the convenience flag.
|
||||
("cpu", {"device": "cuda"}, "cuda"),
|
||||
("cuda", {"device": "cpu"}, "cpu"),
|
||||
],
|
||||
)
|
||||
def test_fold_mm_processor_device(
|
||||
flag: str | None, kwargs: dict | None, expected: str | None
|
||||
):
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
folded = MultiModalConfig.fold_mm_processor_device(kwargs, flag)
|
||||
|
||||
config = MultiModalConfig(mm_processor_kwargs=folded or {})
|
||||
assert config.get_mm_processor_device_type() == expected
|
||||
|
||||
|
||||
def _resolve_mm_processor_device(
|
||||
*,
|
||||
ec_role: ECRole | None,
|
||||
mm_tensor_ipc: str = "torch_shm",
|
||||
device: str | None = None,
|
||||
) -> str | None:
|
||||
"""Run the `auto` resolution and report where the processor ended up."""
|
||||
mm_config = MultiModalConfig(
|
||||
mm_processor_kwargs={} if device is None else {"device": device},
|
||||
mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
|
||||
)
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = mm_config
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config = model_config
|
||||
vllm_config.ec_transfer_config = (
|
||||
None
|
||||
if ec_role is None
|
||||
else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
|
||||
)
|
||||
|
||||
with patch("vllm.platforms.current_platform.device_type", "cuda"):
|
||||
VllmConfig._resolve_mm_processor_device(vllm_config)
|
||||
return mm_config.get_mm_processor_device_type()
|
||||
|
||||
|
||||
def test_auto_mm_processor_device_uses_accelerator_on_encoder_instance():
|
||||
"""The one deployment that has the device to itself and can hand it over."""
|
||||
assert _resolve_mm_processor_device(ec_role="ec_producer") == "cuda"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
|
||||
def test_auto_mm_processor_device_stays_on_cpu_off_encoder_instance(
|
||||
ec_role: ECRole | None,
|
||||
):
|
||||
"""Every other role runs the language model in the same process."""
|
||||
assert _resolve_mm_processor_device(ec_role=ec_role) is None
|
||||
|
||||
|
||||
def test_auto_mm_processor_device_needs_a_device_capable_transport():
|
||||
"""Other transports serialize host bytes, so the output is copied back."""
|
||||
assert (
|
||||
_resolve_mm_processor_device(ec_role="ec_producer", mm_tensor_ipc="direct_rpc")
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_auto_mm_processor_device_leaves_an_explicit_request_alone():
|
||||
assert _resolve_mm_processor_device(ec_role="ec_producer", device="cpu") == "cpu"
|
||||
|
||||
|
||||
def test_vllm_config_runs_the_mm_processor_device_check():
|
||||
"""Startup must reach the check; the rule itself is covered above.
|
||||
|
||||
Guards the wiring, which no other test would notice going missing.
|
||||
"""
|
||||
model_config = MagicMock(spec=ModelConfig)
|
||||
model_config.multimodal_config = MultiModalConfig(
|
||||
mm_processor_kwargs={"device": "cuda"}
|
||||
)
|
||||
vllm_config = MagicMock(spec=VllmConfig)
|
||||
vllm_config.model_config = model_config
|
||||
vllm_config.ec_transfer_config = None
|
||||
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.device_type", "cuda"),
|
||||
pytest.raises(ValueError, match="also runs the language model"),
|
||||
):
|
||||
VllmConfig._validate_mm_processor_device(vllm_config)
|
||||
|
||||
Reference in New Issue
Block a user