[EPD] Allow KV consumers to omit MM embeddings (#52697)
Signed-off-by: zhenwei-intel <zhenwei.liu@intel.com>
This commit is contained in:
@@ -104,7 +104,7 @@ class ColQwen3ProcessingInfo(Qwen3VLProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -101,7 +101,7 @@ class ColQwen3_5ProcessingInfo(Qwen3_5ProcessingInfo):
|
||||
spatial_merge_size,
|
||||
video_needs_metadata=self._supports_video,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -599,7 +599,7 @@ class HunYuanVLProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return HunYuanVLMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -936,7 +936,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(
|
||||
|
||||
@@ -363,7 +363,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return KeyeVL1_5MultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_max_frame_per_video(self) -> int:
|
||||
|
||||
@@ -1279,7 +1279,7 @@ class LlavaOnevision2ProcessingInfo(BaseProcessingInfo):
|
||||
return LlavaOnevision2MultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_hf_processor(self, **kwargs: object):
|
||||
|
||||
@@ -653,7 +653,7 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
|
||||
def get_data_parser(self):
|
||||
return MiniCPMVMultiModalDataParser(
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_model_version(self):
|
||||
|
||||
@@ -57,7 +57,7 @@ class OpenCUAProcessingInfo(Qwen2VLProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_hf_config(self):
|
||||
|
||||
@@ -373,7 +373,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
target_sr=feature_extractor.sampling_rate,
|
||||
target_channels=self.get_target_channels(),
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_target_channels(self) -> int:
|
||||
|
||||
@@ -857,7 +857,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
|
||||
@@ -910,7 +910,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
embeds_from_ec_connector=self.embeds_from_ec_connector,
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
def _get_vision_info(
|
||||
|
||||
Reference in New Issue
Block a user