[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 15:59:54 -07:00
committed by GitHub
parent 38ebd97bca
commit b30291cf0c
22 changed files with 904 additions and 45 deletions
+89 -6
View File
@@ -40,8 +40,18 @@ from .media import MediaWithBytes
_T = TypeVar("_T")
_I = TypeVar("_I")
EmbeddingFieldRole: TypeAlias = Literal["values", "metadata"]
"""What a field of a pre-computed-embedding input carries.
`"values"` is the embedding tensor itself; `"metadata"` is a key that sizes the
prompt's placeholder range. The distinction is what lets one declaration serve
both an ordinary request (everything present) and an EC consumer (embeddings
arrive through the connector, metadata still in the request).
"""
if TYPE_CHECKING:
import PIL.Image as PILImage
else:
PILImage = LazyLoader("PILImage", globals(), "PIL.Image")
@@ -250,34 +260,62 @@ class DictEmbeddingItems(
self,
data: Mapping[str, torch.Tensor],
modality: str,
required_fields: set[str],
required_fields: Set[str],
fields_factory: Callable[
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
optional_fields: Set[str] = frozenset(),
) -> None:
"""
Args:
data: The dictionary of tensors for this modality.
modality: The modality these items belong to.
required_fields: Fields `data` must contain.
fields_factory: Builds the field config from the data.
optional_fields: Fields `data` may omit. Which fields these are is
the caller's decision -- see
`MultiModalDataParser.embedding_field_sets`, where a deployment
that receives embeddings through an EC connector makes the
embeddings optional. They still need a field config, since they
are used whenever they *are* supplied.
"""
from transformers.feature_extraction_utils import BatchFeature
super().__init__(data, modality)
missing_required_data_keys = required_fields - data.keys()
# Nothing required would leave nothing to size the placeholder range
# from, so the item would parse into zero entries and silently produce a
# wrong prompt instead of failing here.
if not required_fields:
raise ValueError(
f"Cannot parse {modality!r} embeddings: every declared field is "
f"optional ({sorted(optional_fields)}), so nothing is left to "
"size the placeholder range."
)
declared_fields = set(required_fields) | set(optional_fields)
missing_required_data_keys = set(required_fields) - data.keys()
if missing_required_data_keys:
data_keys = set(data.keys())
msg = (
f"The data should contain the fields: {required_fields}, "
f"The data should contain the fields: {set(required_fields)}, "
f"but only found the following keys: {data_keys}"
)
raise ValueError(msg)
fields_config = fields_factory(data)
missing_required_fields = required_fields - fields_config.keys()
# Check every declared field, not just the required ones: an optional
# field still needs a config for when it is supplied.
missing_required_fields = declared_fields - fields_config.keys()
if missing_required_fields:
fields = set(fields_config.keys())
msg = f"{required_fields=} should be a subset of {fields=}"
msg = f"{declared_fields=} should be a subset of {fields=}"
raise ValueError(msg)
self.fields_config = fields_config
self.required_fields = required_fields
self.required_fields = set(required_fields)
self._kwargs = MultiModalKwargsItems.from_hf_inputs(
BatchFeature(dict(data)),
@@ -516,8 +554,50 @@ class MultiModalDataParser:
embedding inputs. If provided, validates that user-supplied
embeddings have the correct hidden size to prevent crashes
during model inference.
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
absent from the request because an encode/prefill/decode encoder
instance publishes them through an EC connector instead. Derived by
`BaseProcessingInfo.embeds_from_ec_connector`.
"""
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
"""Per-modality field roles for pre-computed-embedding inputs.
Declared here rather than inside `_parse_*_data` so an EC producer can read
it too: on a producer the request carries real media, so the branch that
builds `DictEmbeddingItems` never runs, yet the producer still has to know
which processed keys to publish. One declaration, both sides, no drift.
A modality absent from this mapping can only be sent whole in the request.
"""
@classmethod
def placeholder_metadata_fields(cls, modality: str) -> set[str]:
"""The keys that size `modality`'s placeholder range.
What an EC producer publishes alongside the embedding, and what a
consumer keeps requiring once the embedding itself is gone.
"""
return {
field
for field, role in cls.embedding_fields.get(modality, {}).items()
if role == "metadata"
}
def embedding_field_sets(self, modality: str) -> tuple[set[str], set[str]]:
"""`modality`'s (required, optional) fields for this deployment.
Resolves the static roles in `embedding_fields` against where the
embeddings actually come from: on an EC consumer they arrive through the
connector, so the request may omit them; anywhere else a request that
claims to carry pre-computed embeddings has to actually carry them.
"""
metadata = self.placeholder_metadata_fields(modality)
values = set(self.embedding_fields.get(modality, {})) - metadata
if self.embeds_from_ec_connector:
return metadata, values
return metadata | values, set()
def __init__(
self,
*,
@@ -526,9 +606,12 @@ class MultiModalDataParser:
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
video_needs_metadata: bool = False,
expected_hidden_size: int | None = None,
embeds_from_ec_connector: bool = False,
) -> None:
super().__init__()
self.embeds_from_ec_connector = embeds_from_ec_connector
self.audio_resampler = AudioResampler(
target_sr=target_sr,
method=audio_resample_method,