[EPD] Remove duplicate image preprocessing in EPD and enable preprocess on GPU (#50390)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -104,6 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -101,6 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -551,15 +551,22 @@ def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
|
||||
|
||||
|
||||
class HunYuanVLMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_hunyuan_vl_field_config,
|
||||
)
|
||||
|
||||
@@ -592,6 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return HunYuanVLMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -881,18 +881,23 @@ def _keye_field_config(
|
||||
|
||||
|
||||
class KeyeMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={
|
||||
"image_embeds",
|
||||
"image_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -903,13 +908,12 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={
|
||||
"video_embeds",
|
||||
"video_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -929,6 +933,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(
|
||||
|
||||
@@ -320,18 +320,23 @@ def _keye_field_config(
|
||||
|
||||
|
||||
class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={
|
||||
"image_embeds",
|
||||
"image_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -342,13 +347,12 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={
|
||||
"video_embeds",
|
||||
"video_grid_thw",
|
||||
},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_keye_field_config,
|
||||
)
|
||||
|
||||
@@ -359,6 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeVL1_5MultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_max_frame_per_video(self) -> int:
|
||||
|
||||
@@ -1279,6 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
||||
return LlavaOnevision2MultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
@@ -1510,6 +1511,11 @@ class LlavaOnevision2DummyInputsBuilder(
|
||||
|
||||
|
||||
class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def __init__(self, spatial_merge_size: int, *args, **kwargs):
|
||||
self._spatial_merge_size = spatial_merge_size
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -1519,10 +1525,12 @@ class LlavaOnevision2MultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_create_field_factory(self._spatial_merge_size),
|
||||
)
|
||||
return super()._parse_image_data(data)
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence, Set
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
@@ -474,11 +474,14 @@ class MiniCPMVImageEmbeddingItems(DictEmbeddingItems):
|
||||
[Mapping[str, torch.Tensor]],
|
||||
Mapping[str, MultiModalFieldConfig],
|
||||
],
|
||||
required_fields: Set[str],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
super().__init__(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_sizes"},
|
||||
required_fields=required_fields,
|
||||
optional_fields=optional_fields,
|
||||
fields_factory=fields_factory,
|
||||
)
|
||||
|
||||
@@ -495,11 +498,14 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
|
||||
[Mapping[str, torch.Tensor]],
|
||||
Mapping[str, MultiModalFieldConfig],
|
||||
],
|
||||
required_fields: Set[str],
|
||||
optional_fields: Set[str] = frozenset(),
|
||||
) -> None:
|
||||
super().__init__(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={"video_embeds", "video_image_sizes"},
|
||||
required_fields=required_fields,
|
||||
optional_fields=optional_fields,
|
||||
fields_factory=fields_factory,
|
||||
)
|
||||
|
||||
@@ -512,14 +518,23 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
|
||||
|
||||
|
||||
class MiniCPMVMultiModalDataParser(MultiModalDataParser):
|
||||
# The per-item sizes are what size the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_sizes": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_image_sizes": "metadata"},
|
||||
}
|
||||
|
||||
def _parse_image_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return MiniCPMVImageEmbeddingItems(
|
||||
data,
|
||||
fields_factory=_minicpmv_field_config,
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
)
|
||||
|
||||
return super()._parse_image_data(data)
|
||||
@@ -529,9 +544,12 @@ class MiniCPMVMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return MiniCPMVVideoEmbeddingItems(
|
||||
data,
|
||||
fields_factory=_minicpmv_field_config,
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
)
|
||||
|
||||
return super()._parse_video_data(data)
|
||||
@@ -573,6 +591,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return MiniCPMVMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_model_version(self):
|
||||
|
||||
@@ -57,6 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_hf_config(self):
|
||||
|
||||
@@ -372,6 +372,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
|
||||
@@ -795,6 +795,12 @@ def _create_qwen2vl_field_factory(
|
||||
|
||||
|
||||
class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
||||
# The patch grid is what sizes the placeholder range.
|
||||
embedding_fields = {
|
||||
"image": {"image_embeds": "values", "image_grid_thw": "metadata"},
|
||||
"video": {"video_embeds": "values", "video_grid_thw": "metadata"},
|
||||
}
|
||||
|
||||
def __init__(self, spatial_merge_size: int, *args, **kwargs):
|
||||
self._spatial_merge_size = spatial_merge_size
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -804,10 +810,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("image")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="image",
|
||||
required_fields={"image_embeds", "image_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
|
||||
)
|
||||
|
||||
@@ -818,10 +826,12 @@ class Qwen2VLMultiModalDataParser(MultiModalDataParser):
|
||||
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("video")
|
||||
return DictEmbeddingItems(
|
||||
data,
|
||||
modality="video",
|
||||
required_fields={"video_embeds", "video_grid_thw"},
|
||||
required_fields=required,
|
||||
optional_fields=optional,
|
||||
fields_factory=_create_qwen2vl_field_factory(self._spatial_merge_size),
|
||||
)
|
||||
|
||||
@@ -846,6 +856,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -879,6 +879,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
)
|
||||
|
||||
def _get_vision_info(
|
||||
|
||||
Reference in New Issue
Block a user