[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -881,18 +881,23 @@ def _keye_field_config(
|
||||
|
||||
|
||||
class KeyeMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={
|
||||
"image_embeds",
|
||||
"image_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={
|
||||
"video_embeds",
|
||||
"video_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(
|
||||
|
||||
Reference in New Issue
Block a user