[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 15:59:54 -07:00
committed by GitHub
parent 38ebd97bca
commit b30291cf0c
22 changed files with 904 additions and 45 deletions
+10
View File
@@ -103,5 +103,15 @@ class ECTransferConfig:
def is_ec_consumer(self) -> bool:
return self.ec_connector is not None and self.ec_role in get_args(ECConsumer)
@property
def is_encode_only(self) -> bool:
"""Whether this instance encodes but does not run the language model.
It allocates no KV cache either -- `GPUModelRunner.get_kv_cache_spec`
returns {} for it -- so it is the one role that can spend accelerator
time and memory on frontend work.
"""
return self.is_ec_producer and not self.is_ec_consumer
def get_from_extra_config(self, key, default) -> Any:
return self.ec_connector_extra_config.get(key, default)
+7
View File
@@ -18,6 +18,7 @@ from vllm.config.multimodal import (
MMCacheType,
MMEncoderTPMode,
MMHasherAlgorithm,
MMProcessorDevice,
MMTensorIPC,
MultiModalConfig,
)
@@ -395,6 +396,7 @@ class ModelConfig:
video_pruning_method: InitVar[str | None] = None
mm_tensor_ipc: InitVar[MMTensorIPC] = None
mm_ipc_gpu_memory_gb: InitVar[float | None] = None
mm_processor_device: InitVar[MMProcessorDevice | None] = None
def compute_hash(self) -> str:
"""
@@ -525,6 +527,7 @@ class ModelConfig:
video_pruning_method: str | None,
mm_tensor_ipc: MMTensorIPC,
mm_ipc_gpu_memory_gb: float | None,
mm_processor_device: MMProcessorDevice | None,
) -> None:
# Keep set served_model_name before maybe_model_redirect(self.model)
self.served_model_name = get_served_model_name(
@@ -763,6 +766,10 @@ class ModelConfig:
)
mm_encoder_tp_mode = "weights"
mm_processor_kwargs = MultiModalConfig.fold_mm_processor_device(
mm_processor_kwargs, mm_processor_device
)
mm_config_kwargs = dict(
language_model_only=language_model_only,
limit_per_prompt=limit_mm_per_prompt,
+136
View File
@@ -5,14 +5,19 @@ from collections.abc import Mapping
from pathlib import Path
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
import torch
from pydantic import ConfigDict, Field, field_validator, model_validator
from pydantic.dataclasses import dataclass
import vllm.envs as envs
from vllm.config.ec_transfer import ECTransferConfig
from vllm.config.utils import config, get_from_deprecated_env_if_set
from vllm.logger import init_logger
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
@dataclass
class BaseDummyOptions:
@@ -64,6 +69,10 @@ MMCacheType = Literal["shm", "lru"]
VideoPruningMethod = Literal["evs", "vidcom2"]
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
MMProcessorDevice: TypeAlias = str
"""`"auto"`, `"cpu"`, or the platform's own accelerator name
(`current_platform.device_type`, e.g. `"cuda"` on CUDA and ROCm,
`"xpu"` on XPU). Validated against that set by the CLI."""
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
@@ -222,6 +231,20 @@ class MultiModalConfig:
- "direct_rpc": Use msgspec serialization via RPC
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
Defaults to "direct_rpc". """
mm_embeds_from_ec_connector: bool = False
"""Whether a pre-computed-embedding input may omit the `*_embeds` tensor.
In an encode/prefill/decode (EPD) deployment the encoder instance publishes
embeddings through the EC connector, so the request that reaches the
prefill/decode instance only needs to carry the grid/size metadata that
sizes the placeholder range — the embeddings themselves come from the
connector, keyed by `mm_hash`.
Derived, not user-settable: `VllmConfig.__post_init__` sets this to True
exactly on EC consumers. Everywhere else it stays False so that a request
which forgets its embeddings still fails fast in the frontend, with a clear
error, rather than deep inside the model."""
mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
"""Amount of GPU memory (in GiB) sequestered on the engine's device for
GPU-side multimodal work in the API-server (frontend) process, such as
@@ -319,6 +342,119 @@ class MultiModalConfig:
)
return self
@staticmethod
def fold_mm_processor_device(
mm_processor_kwargs: dict[str, Any] | None,
mm_processor_device: MMProcessorDevice | None,
) -> dict[str, Any] | None:
"""Fold the `mm_processor_device` convenience flag into the kwargs.
The flag keeps no state of its own: `mm_processor_kwargs["device"]` is
the only representation of where the processor runs, so an explicit
`device` there always wins and `"auto"` stays unresolved for
`VllmConfig`, which is where the EC role needed to resolve it lives.
Args:
mm_processor_kwargs: The kwargs as given, or None.
mm_processor_device: The flag's value, or None when unset.
Returns:
The kwargs to build the config with, unchanged unless the flag adds
a `device`.
"""
if mm_processor_device in (None, "auto"):
return mm_processor_kwargs
if (mm_processor_kwargs or {}).get("device") is not None:
return mm_processor_kwargs
from vllm.platforms import current_platform
# Any explicit value other than "cpu" means "the accelerator", so a
# programmatically-set "cuda" still works on a platform whose device type
# is named differently ("xpu"), and degrades to CPU where there is none.
device = (
"cpu"
if mm_processor_device == "cpu"
else (current_platform.device_type or "cpu")
)
return {**(mm_processor_kwargs or {}), "device": device}
def get_mm_processor_device_type(self) -> str | None:
"""The torch device type `mm_processor_kwargs["device"]` names.
`mm_processor_kwargs` is untyped, so `device` may be any form torch
accepts -- `"cuda"`, `"cuda:1"`, `torch.device(...)`, or a bare index.
Normalising through torch rather than parsing the string keeps the
non-string forms from slipping past a caller's comparison.
Returns:
The device type, or None when no device is requested.
Raises:
ValueError: If `device` is not something `torch.device` accepts.
`validate_mm_processor_device` is what surfaces this during
startup, so the value is only parsed once.
"""
device = (self.mm_processor_kwargs or {}).get("device")
if device is None:
return None
try:
return torch.device(device).type # type: ignore[arg-type]
except (RuntimeError, TypeError, ValueError):
raise ValueError(
f'Invalid "device" in mm_processor_kwargs: {device!r}. Expected a '
'torch device such as "cpu", "cuda" or "cuda:0".'
) from None
def validate_mm_processor_device(self, ec_config: ECTransferConfig | None) -> None:
"""Check `mm_processor_kwargs["device"]` for this deployment.
The only place the requested device is validated, so it runs even on a
CPU-only platform: the value is parsed before any early return.
Args:
ec_config: The deployment's EC config, or None when it is not an
encode/prefill/decode deployment. Passed in because it is not
reachable from here, and because a field assigned after
construction would not re-trigger this config's validators.
Raises:
ValueError: If the requested device is not a torch device, or if it
is the accelerator on an instance that also runs the language
model.
"""
from vllm.platforms import current_platform
device_type = self.get_mm_processor_device_type()
accelerator = current_platform.device_type
if device_type is None or accelerator in ("", "cpu"):
return
if device_type != accelerator:
return
if ec_config is None or not ec_config.is_encode_only:
raise ValueError(
f"Cannot run the multi-modal processor on {device_type!r}: this "
"instance also runs the language model. The processor would "
"share the device with the model's forward pass, so its "
"transform kernels contend with that compute, and because it "
"runs in the API-server process its allocations are outside the "
"memory the engine profiled for its KV cache -- risking OOM or "
"a silently shrunken cache.\n"
"Accelerator preprocessing is only supported on an encode-only "
"instance of an encode/prefill/decode deployment (an EC "
"producer that is not also a consumer), which runs no forward "
"pass and allocates no KV cache.\n"
'Use --mm-processor-device=cpu, or drop "device" from '
"--mm-processor-kwargs."
)
logger.info_once(
"Running the multi-modal processor on %s. Override with "
"--mm-processor-device=cpu.",
device_type,
)
def compute_hash(self) -> str:
"""
WARNING: Whenever a new field is added to this config,
+99
View File
@@ -1509,6 +1509,10 @@ class VllmConfig:
)
current_platform.check_and_update_config(self)
self._resolve_mm_embeds_from_ec_connector()
self._resolve_mm_processor_device()
self._validate_mm_processor_device()
if self.use_v2_model_runner:
self._validate_v2_model_runner()
elif self.parallel_config.prefill_context_parallel_size > 1:
@@ -2185,6 +2189,101 @@ class VllmConfig:
f"kernel_config={self.kernel_config!r}"
)
def _resolve_mm_embeds_from_ec_connector(self) -> None:
"""Allow `*_embeds` to be omitted only where the connector supplies them.
That is exactly an EC consumer: the encoder instance publishes the
embeddings through the EC connector, so the request only has to carry
the grid metadata that sizes the placeholder range. On every other
deployment a missing `*_embeds` is a client error and must keep failing
fast in the frontend.
"""
model_config = self.model_config
if model_config is None:
return
mm_config = model_config.multimodal_config
if mm_config is None:
return
ec_config = self.ec_transfer_config
# Derived, so overwrite unconditionally rather than honouring a value
# that was set by hand.
mm_config.mm_embeds_from_ec_connector = (
ec_config is not None and ec_config.is_ec_consumer
)
if mm_config.mm_embeds_from_ec_connector:
logger.info_once(
"EC consumer: pre-computed-embedding inputs may omit the "
"embedding tensor; embeddings are loaded from the EC connector."
)
def _resolve_mm_processor_device(self) -> None:
"""Settle `--mm-processor-device=auto` now that the EC role is known.
"auto" means "the accelerator, but only where the processor has it to
itself and its output can be handed over without a copy back to host":
an encode-only instance whose tensor transport carries device tensors.
Every other deployment keeps the processor on CPU.
An explicit device -- from `--mm-processor-device` or straight from
`mm_processor_kwargs` -- is already folded in by `MultiModalConfig`, so
it is left alone here and validated by `_validate_mm_processor_device`.
"""
model_config = self.model_config
if model_config is None:
return
mm_config = model_config.multimodal_config
if mm_config is None:
return
if mm_config.get_mm_processor_device_type() is not None:
return
from vllm.platforms import current_platform
device_type = current_platform.device_type
if device_type in ("", "cpu"):
return
ec_config = self.ec_transfer_config
# An EC producer that is not also a consumer runs no forward pass and
# allocates no KV cache, so frontend accelerator work has the device to
# itself.
if ec_config is None or not ec_config.is_encode_only:
return
if mm_config.mm_tensor_ipc != "torch_shm":
# Any other transport serializes host bytes, so the output would be
# copied back, and that copy costs more than running the transform
# on device saves.
logger.info_once(
"EPD encoder instance: keeping the multi-modal processor on CPU "
"because mm_tensor_ipc=%s cannot carry device tensors. Add "
"--mm-tensor-ipc=torch_shm to run it on the accelerator.",
mm_config.mm_tensor_ipc,
)
return
mm_config.mm_processor_kwargs = {
**(mm_config.mm_processor_kwargs or {}),
"device": device_type,
}
logger.info_once(
"EPD encoder instance: running the multi-modal processor on %s. "
"Override with --mm-processor-device=cpu.",
device_type,
)
def _validate_mm_processor_device(self) -> None:
"""Hand the EC config to `MultiModalConfig`, which owns the rule."""
model_config = self.model_config
if model_config is None:
return
mm_config = model_config.multimodal_config
if mm_config is None:
return
mm_config.validate_mm_processor_device(self.ec_transfer_config)
def _get_v2_model_runner_unsupported_features(self) -> list[str]:
"""Collect features not yet supported by the V2 model runner."""
unsupported: list[str] = []
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -50,6 +51,8 @@ class ECExampleConnector(ECConnectorBase):
super().__init__(vllm_config=vllm_config, role=role)
# req_id -> index
self._mm_datas_need_loads: dict[str, int] = {}
self._model_config = vllm_config.model_config
self._metadata_fields_cache: dict[str, set[str]] = {}
transfer_config = vllm_config.ec_transfer_config
if transfer_config is not None:
self._storage_path = transfer_config.get_from_extra_config(
@@ -165,6 +168,70 @@ class ECExampleConnector(ECConnectorBase):
self._mm_datas_need_loads.clear()
return meta
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Which processed keys this model needs published for `modality`.
Read from `MultiModalDataParser.embedding_fields`, the same declaration
the consumer's parser requires, so the two cannot drift. An empty set
means the modality cannot be delivered out of band, and the consumer
will process the media itself.
"""
if modality in self._metadata_fields_cache:
return self._metadata_fields_cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
# Reporting nothing is a safe degradation: the consumer falls back to
# processing the media itself.
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._metadata_fields_cache[modality] = fields
return fields
def request_finished(
self,
request: "Request",
) -> tuple[bool, dict[str, Any] | None]:
"""Report each item's cache key and grid so a consumer can skip the
image transform.
A consumer only needs the grid to size the prompt's placeholder range;
the embedding itself arrives through this connector. Reporting the grid
the producer actually computed keeps the two sides in agreement without
the caller re-deriving it from the raw media.
"""
if not self.is_producer:
return False, None
items = []
for feature in request.mm_features:
metadata = {}
# `data` is None for items served from the processor cache, in which
# case the metadata is unavailable here and the consumer has to fall
# back to processing the media itself.
if feature.data is not None:
wanted = self._placeholder_metadata_fields(feature.modality)
metadata = {
key: value.tolist()
for key, value in feature.data.get_data().items()
if key in wanted and isinstance(value, torch.Tensor)
}
items.append({"mm_hash": feature.identifier, **metadata})
if not items:
return False, None
return False, {"ec_items": items}
# ==============================
# Helper functions
# ==============================
+27
View File
@@ -90,6 +90,7 @@ from vllm.config.multimodal import (
MMCacheType,
MMEncoderTPMode,
MMHasherAlgorithm,
MMProcessorDevice,
MMTensorIPC,
)
from vllm.config.observability import DetailedTraceModules
@@ -596,6 +597,7 @@ class EngineArgs:
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
video_pruning_method: str = MultiModalConfig.video_pruning_method
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
mm_processor_device: MMProcessorDevice = "auto"
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
# LoRA fields
enable_lora: bool = False
@@ -1353,6 +1355,30 @@ class EngineArgs:
multimodal_group.add_argument(
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
)
multimodal_group.add_argument(
"--mm-processor-device",
choices=["auto", "cpu"]
+ (
[current_platform.device_type]
if current_platform.device_type not in ("", "cpu")
else []
),
default="auto",
help="Device the HF multi-modal processor runs the image/video "
"transform on. Convenience for `--mm-processor-kwargs "
"'{\"device\": ...}'`: the value is resolved here and stored there, "
"it is not kept as separate state. Only takes effect for HF "
'"fast" (torchvision-backed) processors, which accept a `device` '
"argument; the others ignore it and stay on CPU.\n\n"
'"auto" uses the accelerator on encoder instances of an '
"encode/prefill/decode deployment -- an EC producer that is not "
"also a consumer allocates no KV cache, so its accelerator is not "
"contended by the language model -- and then only when "
"`--mm-tensor-ipc=torch_shm` can carry device tensors, since every "
"other transport would copy the result back to the host and that "
'copy costs more than it saves. "auto" resolves to "cpu" '
"everywhere else.",
)
multimodal_group.add_argument(
"--mm-ipc-gpu-memory-gb",
**multimodal_kwargs["mm_ipc_gpu_memory_gb"],
@@ -1743,6 +1769,7 @@ class EngineArgs:
video_pruning_method=self.video_pruning_method,
mm_tensor_ipc=self.mm_tensor_ipc,
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
mm_processor_device=self.mm_processor_device,
io_processor_plugin=self.io_processor_plugin,
renderer_num_workers=self.renderer_num_workers,
)
+1
View File
@@ -104,6 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
+1
View File
@@ -101,6 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
+9 -1
View File
@@ -551,15 +551,22 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
class HunYuanVLMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_hunyuan_vl_field_config,
)
@@ -592,6 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return HunYuanVLMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+13 -8
View File
@@ -881,18 +881,23 @@ def _keye_field_config(
class KeyeMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={
"image_embeds",
"image_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={
"video_embeds",
"video_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return KeyeMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(
+13 -8
View File
@@ -320,18 +320,23 @@ def _keye_field_config(
class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={
"image_embeds",
"image_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -342,13 +347,12 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={
"video_embeds",
"video_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -359,6 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
def get_data_parser(self):
return KeyeVL1_5MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_max_frame_per_video(self) -> int:
@@ -1279,6 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
return LlavaOnevision2MultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_hf_processor(self, **kwargs: object):
@@ -1510,6 +1511,11 @@ class LlavaOnevision2DummyInputsBuilder(
class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(*args, **kwargs)
@@ -1519,10 +1525,12 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_field_factory(self._spatial_merge_size),
)
return super()._parse_image_data(data)
+22 -3
View File
@@ -26,7 +26,7 @@
import math
from collections import defaultdict
from collections.abc import Callable, Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
from functools import partial
from itertools import chain
from typing import Annotated, Any, Literal, TypeAlias
@@ -474,11 +474,14 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
required_fields: Set[str],
optional_fields: Set[str] = frozenset(),
) -> None:
super().__init__(
data,
modality="image",
required_fields={"image_embeds", "image_sizes"},
required_fields=required_fields,
optional_fields=optional_fields,
fields_factory=fields_factory,
)
@@ -495,11 +498,14 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
required_fields: Set[str],
optional_fields: Set[str] = frozenset(),
) -> None:
super().__init__(
data,
modality="video",
required_fields={"video_embeds", "video_image_sizes"},
required_fields=required_fields,
optional_fields=optional_fields,
fields_factory=fields_factory,
)
@@ -512,14 +518,23 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
# The per-item sizes are what size the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_sizes": "metadata"},
"video": {"video_embeds": "values", "video_image_sizes": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return MiniCPMVImageEmbeddingItems(
data,
fields_factory=_minicpmv_field_config,
required_fields=required,
optional_fields=optional,
)
return super()._parse_image_data(data)
@@ -529,9 +544,12 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return MiniCPMVVideoEmbeddingItems(
data,
fields_factory=_minicpmv_field_config,
required_fields=required,
optional_fields=optional,
)
return super()._parse_video_data(data)
@@ -573,6 +591,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return MiniCPMVMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_model_version(self):
+1
View File
@@ -57,6 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_hf_config(self):
@@ -372,6 +372,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_target_channels(self) -> int:
+13 -2
View File
@@ -795,6 +795,12 @@ def _create_qwen2vl_field_factory(
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(*args, **kwargs)
@@ -804,10 +810,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
)
@@ -818,10 +826,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={"video_embeds", "video_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
)
@@ -846,6 +856,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+1
View File
@@ -879,6 +879,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def _get_vision_info(
+13 -1
View File
@@ -23,7 +23,7 @@ from typing_extensions import TypeVar
from vllm.utils.collection_utils import is_list_of
from vllm.utils.import_utils import LazyLoader
from vllm.utils.jsontree import json_map_leaves
from vllm.utils.jsontree import json_iter_leaves, json_map_leaves
from .media import MediaWithBytes
@@ -286,6 +286,13 @@ def nested_tensors_equal(
return a == b
def _nested_tensors_are_cpu(tensors: NestedTensors) -> bool:
"""Whether every tensor in `tensors` lives in host memory."""
return not any(
isinstance(x, torch.Tensor) and not x.is_cpu for x in json_iter_leaves(tensors)
)
def _nested_tensors_h2d(
tensors: NestedTensors,
device: torch.types.Device,
@@ -478,6 +485,11 @@ class BaseMultiModalField(ABC):
pin_memory = False
batch = [elem.data for elem in elems]
if pin_memory and not _nested_tensors_are_cpu(batch):
# Data already sits on an accelerator (e.g. a device-side HF
# processor handed it over by IPC). Pinning is a host-memory
# concept, and `pin_memory()` rejects device tensors outright.
pin_memory = False
out = self._reduce_data(batch, pin_memory=pin_memory)
return _nested_tensors_h2d(out, device=device)
+89 -6
View File
@@ -40,8 +40,18 @@ from .media import MediaWithBytes
_T = TypeVar("_T")
_I = TypeVar("_I")
EmbeddingFieldRole: TypeAlias = Literal["values", "metadata"]
"""What a field of a pre-computed-embedding input carries.
`"values"` is the embedding tensor itself; `"metadata"` is a key that sizes the
prompt's placeholder range. The distinction is what lets one declaration serve
both an ordinary request (everything present) and an EC consumer (embeddings
arrive through the connector, metadata still in the request).
"""
if TYPE_CHECKING:
import PIL.Image as PILImage
else:
PILImage = LazyLoader("PILImage", globals(), "PIL.Image")
@@ -250,34 +260,62 @@ class DictEmbeddingItems(
self,
data: Mapping[str, torch.Tensor],
modality: str,
required_fields: set[str],
required_fields: Set[str],
fields_factory: Callable[
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
optional_fields: Set[str] = frozenset(),
) -> None:
"""
Args:
data: The dictionary of tensors for this modality.
modality: The modality these items belong to.
required_fields: Fields `data` must contain.
fields_factory: Builds the field config from the data.
optional_fields: Fields `data` may omit. Which fields these are is
the caller's decision -- see
`MultiModalDataParser.embedding_field_sets`, where a deployment
that receives embeddings through an EC connector makes the
embeddings optional. They still need a field config, since they
are used whenever they *are* supplied.
"""
from transformers.feature_extraction_utils import BatchFeature
super().__init__(data, modality)
missing_required_data_keys = required_fields - data.keys()
# Nothing required would leave nothing to size the placeholder range
# from, so the item would parse into zero entries and silently produce a
# wrong prompt instead of failing here.
if not required_fields:
raise ValueError(
f"Cannot parse {modality!r} embeddings: every declared field is "
f"optional ({sorted(optional_fields)}), so nothing is left to "
"size the placeholder range."
)
declared_fields = set(required_fields) | set(optional_fields)
missing_required_data_keys = set(required_fields) - data.keys()
if missing_required_data_keys:
data_keys = set(data.keys())
msg = (
f"The data should contain the fields: {required_fields}, "
f"The data should contain the fields: {set(required_fields)}, "
f"but only found the following keys: {data_keys}"
)
raise ValueError(msg)
fields_config = fields_factory(data)
missing_required_fields = required_fields - fields_config.keys()
# Check every declared field, not just the required ones: an optional
# field still needs a config for when it is supplied.
missing_required_fields = declared_fields - fields_config.keys()
if missing_required_fields:
fields = set(fields_config.keys())
msg = f"{required_fields=} should be a subset of {fields=}"
msg = f"{declared_fields=} should be a subset of {fields=}"
raise ValueError(msg)
self.fields_config = fields_config
self.required_fields = required_fields
self.required_fields = set(required_fields)
self._kwargs = MultiModalKwargsItems.from_hf_inputs(
BatchFeature(dict(data)),
@@ -516,8 +554,50 @@ class MultiModalDataParser:
embedding inputs. If provided, validates that user-supplied
embeddings have the correct hidden size to prevent crashes
during model inference.
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
absent from the request because an encode/prefill/decode encoder
instance publishes them through an EC connector instead. Derived by
`BaseProcessingInfo.embeds_from_ec_connector`.
"""
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
"""Per-modality field roles for pre-computed-embedding inputs.
Declared here rather than inside `_parse_*_data` so an EC producer can read
it too: on a producer the request carries real media, so the branch that
builds `DictEmbeddingItems` never runs, yet the producer still has to know
which processed keys to publish. One declaration, both sides, no drift.
A modality absent from this mapping can only be sent whole in the request.
"""
@classmethod
def placeholder_metadata_fields(cls, modality: str) -> set[str]:
"""The keys that size `modality`'s placeholder range.
What an EC producer publishes alongside the embedding, and what a
consumer keeps requiring once the embedding itself is gone.
"""
return {
field
for field, role in cls.embedding_fields.get(modality, {}).items()
if role == "metadata"
}
def embedding_field_sets(self, modality: str) -> tuple[set[str], set[str]]:
"""`modality`'s (required, optional) fields for this deployment.
Resolves the static roles in `embedding_fields` against where the
embeddings actually come from: on an EC consumer they arrive through the
connector, so the request may omit them; anywhere else a request that
claims to carry pre-computed embeddings has to actually carry them.
"""
metadata = self.placeholder_metadata_fields(modality)
values = set(self.embedding_fields.get(modality, {})) - metadata
if self.embeds_from_ec_connector:
return metadata, values
return metadata | values, set()
def __init__(
self,
*,
@@ -526,9 +606,12 @@ class MultiModalDataParser:
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
embeds_from_ec_connector: bool = False,
) -> None:
super().__init__()
self.embeds_from_ec_connector = embeds_from_ec_connector
self.audio_resampler = AudioResampler(
target_sr=target_sr,
method=audio_resample_method,
+35 -8
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from abc import abstractmethod
from collections.abc import Callable, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
@@ -227,13 +226,30 @@ class InputProcessingContext:
self,
output: JSONTree,
) -> JSONTree:
def _postprocess_one(x: object):
if isinstance(x, torch.Tensor): # noqa: SIM102
# This mimics the behavior of transformers.BatchFeature
if x.is_floating_point():
x = x.to(dtype=self.model_config.dtype)
# "torch_shm" puts tensors on a torch.multiprocessing queue, which
# shares device tensors by CUDA IPC handle, so a device-side processor
# can hand `pixel_values` straight to the worker. Every other transport
# serializes host bytes, so the result has to be copied back first.
keep_on_device = (
self.model_config.get_multimodal_config().mm_tensor_ipc == "torch_shm"
)
return x
def _postprocess_one(x: object):
if not isinstance(x, torch.Tensor):
return x
# Bind to a Tensor-typed local: reassigning the `object`-typed
# parameter would discard the isinstance narrowing.
tensor = x
# This mimics the behavior of transformers.BatchFeature
if tensor.is_floating_point():
tensor = tensor.to(dtype=self.model_config.dtype)
if not tensor.is_cpu and not keep_on_device:
tensor = tensor.cpu()
return tensor
return json_map_leaves(_postprocess_one, output)
@@ -350,6 +366,17 @@ class BaseProcessingInfo:
return None
@property
def embeds_from_ec_connector(self) -> bool:
"""Whether pre-computed embeddings may arrive outside the request.
True only on an EC consumer, where an encode/prefill/decode encoder
instance publishes them through the connector instead, so the request
carries only the metadata that sizes the placeholder range.
"""
mm_config = self.ctx.model_config.multimodal_config
return mm_config is not None and mm_config.mm_embeds_from_ec_connector
def get_data_parser(self) -> MultiModalDataParser:
"""
Constructs a parser to preprocess multi-modal data items
@@ -362,6 +389,7 @@ class BaseProcessingInfo:
"""
return MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
@cached_property
@@ -372,7 +400,6 @@ class BaseProcessingInfo:
def skip_prompt_length_check(self) -> bool:
return False
@abstractmethod
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
"""
Return the maximum supported number of items for each modality.