[Bugfix][MiMo] Declare embedding_fields so an EPD pair can serve images (#58938)

Signed-off-by: Giulio De Pasquale <git@depasquale.giugl.io>
This commit is contained in:
Giulio De Pasquale
2026-09-29 23:33:01 +08:00
committed by GitHub
parent ac7f3e11ea
commit 8aaeef343a
+20 -4
View File
@@ -69,7 +69,10 @@ from .qwen2_5_vl import (
Qwen2_5_VLVideoInputs,
Qwen2_5_VLVideoPixelInputs,
)
from .qwen2_vl import _create_qwen2vl_field_factory
from .qwen2_vl import (
Qwen2VLMultiModalDataParser,
_create_qwen2vl_field_factory,
)
from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix
@@ -670,6 +673,14 @@ class MiMoVisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiMoV2OmniMultiModalDataParser(Qwen2VLMultiModalDataParser):
# Video stays raw: get_video_replacement also reads second_per_grid_ts and
# video_start_times, which a metadata-only rewrite would not carry.
embedding_fields = {
"image": Qwen2VLMultiModalDataParser.embedding_fields["image"],
}
class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None, "image": None, "video": None}
@@ -689,9 +700,14 @@ class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
return self.get_hf_processor(**kwargs).image_processor
def get_data_parser(self):
from vllm.multimodal.parse import MultiModalDataParser
return MultiModalDataParser(target_sr=24000.0)
# Without embedding_fields the EC producer publishes no metadata and
# the EPD proxy never rewrites the media item.
return MiMoV2OmniMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
target_sr=24000.0,
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_mm_max_tokens_per_item(
self,