[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-08-05 15:59:54 -07:00
committed by GitHub
parent 38ebd97bca
commit b30291cf0c
22 changed files with 904 additions and 45 deletions
+1
View File
@@ -104,6 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
+1
View File
@@ -101,6 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
spatial_merge_size,
video_needs_metadata=self._supports_video,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
+9 -1
View File
@@ -551,15 +551,22 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
class HunYuanVLMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_hunyuan_vl_field_config,
)
@@ -592,6 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return HunYuanVLMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+13 -8
View File
@@ -881,18 +881,23 @@ def _keye_field_config(
class KeyeMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={
"image_embeds",
"image_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={
"video_embeds",
"video_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return KeyeMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(
+13 -8
View File
@@ -320,18 +320,23 @@ def _keye_field_config(
class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={
"image_embeds",
"image_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -342,13 +347,12 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={
"video_embeds",
"video_grid_thw",
},
required_fields=required,
optional_fields=optional,
fields_factory=_keye_field_config,
)
@@ -359,6 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
def get_data_parser(self):
return KeyeVL1_5MultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_max_frame_per_video(self) -> int:
@@ -1279,6 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
return LlavaOnevision2MultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_hf_processor(self, **kwargs: object):
@@ -1510,6 +1511,11 @@ class LlavaOnevision2DummyInputsBuilder(
class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(*args, **kwargs)
@@ -1519,10 +1525,12 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_field_factory(self._spatial_merge_size),
)
return super()._parse_image_data(data)
+22 -3
View File
@@ -26,7 +26,7 @@
import math
from collections import defaultdict
from collections.abc import Callable, Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
from functools import partial
from itertools import chain
from typing import Annotated, Any, Literal, TypeAlias
@@ -474,11 +474,14 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
required_fields: Set[str],
optional_fields: Set[str] = frozenset(),
) -> None:
super().__init__(
data,
modality="image",
required_fields={"image_embeds", "image_sizes"},
required_fields=required_fields,
optional_fields=optional_fields,
fields_factory=fields_factory,
)
@@ -495,11 +498,14 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
[Mapping[str, torch.Tensor]],
Mapping[str, MultiModalFieldConfig],
],
required_fields: Set[str],
optional_fields: Set[str] = frozenset(),
) -> None:
super().__init__(
data,
modality="video",
required_fields={"video_embeds", "video_image_sizes"},
required_fields=required_fields,
optional_fields=optional_fields,
fields_factory=fields_factory,
)
@@ -512,14 +518,23 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
# The per-item sizes are what size the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_sizes": "metadata"},
"video": {"video_embeds": "values", "video_image_sizes": "metadata"},
}
def _parse_image_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return MiniCPMVImageEmbeddingItems(
data,
fields_factory=_minicpmv_field_config,
required_fields=required,
optional_fields=optional,
)
return super()._parse_image_data(data)
@@ -529,9 +544,12 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return MiniCPMVVideoEmbeddingItems(
data,
fields_factory=_minicpmv_field_config,
required_fields=required,
optional_fields=optional,
)
return super()._parse_video_data(data)
@@ -573,6 +591,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
def get_data_parser(self):
return MiniCPMVMultiModalDataParser(
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_model_version(self):
+1
View File
@@ -57,6 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_hf_config(self):
@@ -372,6 +372,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
target_sr=feature_extractor.sampling_rate,
target_channels=self.get_target_channels(),
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_target_channels(self) -> int:
+13 -2
View File
@@ -795,6 +795,12 @@ def _create_qwen2vl_field_factory(
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
# The patch grid is what sizes the placeholder range.
embedding_fields = {
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
}
def __init__(self, spatial_merge_size: int, *args, **kwargs):
self._spatial_merge_size = spatial_merge_size
super().__init__(*args, **kwargs)
@@ -804,10 +810,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("image")
return DictEmbeddingItems(
data,
modality="image",
required_fields={"image_embeds", "image_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
)
@@ -818,10 +826,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("video")
return DictEmbeddingItems(
data,
modality="video",
required_fields={"video_embeds", "video_grid_thw"},
required_fields=required,
optional_fields=optional,
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
)
@@ -846,6 +856,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+1
View File
@@ -879,6 +879,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
embeds_from_ec_connector=self.embeds_from_ec_connector,
)
def _get_vision_info(