[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 15:59:54 -07:00
committed by GitHub
parent 38ebd97bca
commit b30291cf0c
22 changed files with 904 additions and 45 deletions
@@ -2,9 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
import safetensors
import torch
from vllm.config import VllmConfig
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -50,6 +51,8 @@ class ECExampleConnector(ECConnectorBase):
super().__init__(vllm_config=vllm_config, role=role)
# req_id -> index
self._mm_datas_need_loads: dict[str, int] = {}
self._model_config = vllm_config.model_config
self._metadata_fields_cache: dict[str, set[str]] = {}
transfer_config = vllm_config.ec_transfer_config
if transfer_config is not None:
self._storage_path = transfer_config.get_from_extra_config(
@@ -165,6 +168,70 @@ class ECExampleConnector(ECConnectorBase):
self._mm_datas_need_loads.clear()
return meta
def _placeholder_metadata_fields(self, modality: str) -> set[str]:
"""Which processed keys this model needs published for `modality`.
Read from `MultiModalDataParser.embedding_fields`, the same declaration
the consumer's parser requires, so the two cannot drift. An empty set
means the modality cannot be delivered out of band, and the consumer
will process the media itself.
"""
if modality in self._metadata_fields_cache:
return self._metadata_fields_cache[modality]
fields: set[str] = set()
try:
from vllm.multimodal import MULTIMODAL_REGISTRY
info = MULTIMODAL_REGISTRY.create_processor(self._model_config).info
fields = info.data_parser.placeholder_metadata_fields(modality)
except Exception:
# Reporting nothing is a safe degradation: the consumer falls back to
# processing the media itself.
logger.warning(
"Could not determine the placeholder metadata fields for "
"modality %s; the consumer will preprocess the media itself.",
modality,
exc_info=True,
)
self._metadata_fields_cache[modality] = fields
return fields
def request_finished(
self,
request: "Request",
) -> tuple[bool, dict[str, Any] | None]:
"""Report each item's cache key and grid so a consumer can skip the
image transform.
A consumer only needs the grid to size the prompt's placeholder range;
the embedding itself arrives through this connector. Reporting the grid
the producer actually computed keeps the two sides in agreement without
the caller re-deriving it from the raw media.
"""
if not self.is_producer:
return False, None
items = []
for feature in request.mm_features:
metadata = {}
# `data` is None for items served from the processor cache, in which
# case the metadata is unavailable here and the consumer has to fall
# back to processing the media itself.
if feature.data is not None:
wanted = self._placeholder_metadata_fields(feature.modality)
metadata = {
key: value.tolist()
for key, value in feature.data.get_data().items()
if key in wanted and isinstance(value, torch.Tensor)
}
items.append({"mm_hash": feature.identifier, **metadata})
if not items:
return False, None
return False, {"ec_items": items}
# ==============================
# Helper functions
# ==============================