[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -40,8 +40,18 @@ from .media import MediaWithBytes
|
||||
_T = TypeVar("_T")
|
||||
_I = TypeVar("_I")
|
||||
|
||||
EmbeddingFieldRole: TypeAlias = Literal["values", "metadata"]
|
||||
"""What a field of a pre-computed-embedding input carries.
|
||||
|
||||
`"values"` is the embedding tensor itself; `"metadata"` is a key that sizes the
|
||||
prompt's placeholder range. The distinction is what lets one declaration serve
|
||||
both an ordinary request (everything present) and an EC consumer (embeddings
|
||||
arrive through the connector, metadata still in the request).
|
||||
"""
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import PIL.Image as PILImage
|
||||
|
||||
else:
|
||||
PILImage = LazyLoader("PILImage", globals(), "PIL.Image")
|
||||
|
||||
@@ -250,34 +260,62 @@ class DictEmbeddingItems(
|
||||
self,
|
||||
data: Mapping[str, torch.Tensor],
|
||||
modality: str,
|
||||
required_fields: set[str],
|
||||
required_fields: Set[str],
|
||||
fields_factory: Callable[
|
||||
[Mapping[str, torch.Tensor]],
|
||||
Mapping[str, MultiModalFieldConfig],
|
||||
],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
data: The dictionary of tensors for this modality.
|
||||
modality: The modality these items belong to.
|
||||
required_fields: Fields `data` must contain.
|
||||
fields_factory: Builds the field config from the data.
|
||||
optional_fields: Fields `data` may omit. Which fields these are is
|
||||
the caller's decision -- see
|
||||
`MultiModalDataParser.embedding_field_sets`, where a deployment
|
||||
that receives embeddings through an EC connector makes the
|
||||
embeddings optional. They still need a field config, since they
|
||||
are used whenever they *are* supplied.
|
||||
"""
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
super().__init__(data, modality)
|
||||
|
||||
missing_required_data_keys = required_fields - data.keys()
|
||||
# Nothing required would leave nothing to size the placeholder range
|
||||
# from, so the item would parse into zero entries and silently produce a
|
||||
# wrong prompt instead of failing here.
|
||||
if not required_fields:
|
||||
raise ValueError(
|
||||
f"Cannot parse {modality!r} embeddings: every declared field is "
|
||||
f"optional ({sorted(optional_fields)}), so nothing is left to "
|
||||
"size the placeholder range."
|
||||
)
|
||||
|
||||
declared_fields = set(required_fields) | set(optional_fields)
|
||||
|
||||
missing_required_data_keys = set(required_fields) - data.keys()
|
||||
if missing_required_data_keys:
|
||||
data_keys = set(data.keys())
|
||||
msg = (
|
||||
f"The data should contain the fields: {required_fields}, "
|
||||
f"The data should contain the fields: {set(required_fields)}, "
|
||||
f"but only found the following keys: {data_keys}"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
fields_config = fields_factory(data)
|
||||
missing_required_fields = required_fields - fields_config.keys()
|
||||
# Check every declared field, not just the required ones: an optional
|
||||
# field still needs a config for when it is supplied.
|
||||
missing_required_fields = declared_fields - fields_config.keys()
|
||||
if missing_required_fields:
|
||||
fields = set(fields_config.keys())
|
||||
msg = f"{required_fields=} should be a subset of {fields=}"
|
||||
msg = f"{declared_fields=} should be a subset of {fields=}"
|
||||
raise ValueError(msg)
|
||||
|
||||
self.fields_config = fields_config
|
||||
self.required_fields = required_fields
|
||||
self.required_fields = set(required_fields)
|
||||
|
||||
self._kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
BatchFeature(dict(data)),
|
||||
@@ -516,8 +554,50 @@ class MultiModalDataParser:
|
||||
embedding inputs. If provided, validates that user-supplied
|
||||
embeddings have the correct hidden size to prevent crashes
|
||||
during model inference.
|
||||
embeds_from_ec_connector (bool): Whether pre-computed embeddings may be
|
||||
absent from the request because an encode/prefill/decode encoder
|
||||
instance publishes them through an EC connector instead. Derived by
|
||||
`BaseProcessingInfo.embeds_from_ec_connector`.
|
||||
"""
|
||||
|
||||
embedding_fields: Mapping[str, Mapping[str, EmbeddingFieldRole]] = {}
|
||||
"""Per-modality field roles for pre-computed-embedding inputs.
|
||||
|
||||
Declared here rather than inside `_parse_*_data` so an EC producer can read
|
||||
it too: on a producer the request carries real media, so the branch that
|
||||
builds `DictEmbeddingItems` never runs, yet the producer still has to know
|
||||
which processed keys to publish. One declaration, both sides, no drift.
|
||||
|
||||
A modality absent from this mapping can only be sent whole in the request.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def placeholder_metadata_fields(cls, modality: str) -> set[str]:
|
||||
"""The keys that size `modality`'s placeholder range.
|
||||
|
||||
What an EC producer publishes alongside the embedding, and what a
|
||||
consumer keeps requiring once the embedding itself is gone.
|
||||
"""
|
||||
return {
|
||||
field
|
||||
for field, role in cls.embedding_fields.get(modality, {}).items()
|
||||
if role == "metadata"
|
||||
}
|
||||
|
||||
def embedding_field_sets(self, modality: str) -> tuple[set[str], set[str]]:
|
||||
"""`modality`'s (required, optional) fields for this deployment.
|
||||
|
||||
Resolves the static roles in `embedding_fields` against where the
|
||||
embeddings actually come from: on an EC consumer they arrive through the
|
||||
connector, so the request may omit them; anywhere else a request that
|
||||
claims to carry pre-computed embeddings has to actually carry them.
|
||||
"""
|
||||
metadata = self.placeholder_metadata_fields(modality)
|
||||
values = set(self.embedding_fields.get(modality, {})) - metadata
|
||||
if self.embeds_from_ec_connector:
|
||||
return metadata, values
|
||||
return metadata | values, set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -526,9 +606,12 @@ class MultiModalDataParser:
|
||||
audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
|
||||
video_needs_metadata: bool = False,
|
||||
expected_hidden_size: int | None = None,
|
||||
embeds_from_ec_connector: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.embeds_from_ec_connector = embeds_from_ec_connector
|
||||
|
||||
self.audio_resampler = AudioResampler(
|
||||
target_sr=target_sr,
|
||||
method=audio_resample_method,
|
||||
|
||||
Reference in New Issue
Block a user