[Bugfix][Multimodal] Honor modality-scoped mm_processor_kwargs (#53808)

Signed-off-by: Prudhvivuda <prudhvi12042001@gmail.com>
This commit is contained in:
Prudhvi Vuda
2026-08-31 06:37:12 +00:00
committed by GitHub
parent 4ae172231c
commit 7292ee2791
5 changed files with 172 additions and 9 deletions
+5 -2
View File
@@ -881,6 +881,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
) -> tuple[ImageSize, int]:
hf_config = self.get_hf_config()
vision_config = hf_config.vision_config
@@ -888,7 +889,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
@@ -936,6 +937,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
num_frames=1,
image_processor=image_processor,
mm_kwargs=mm_kwargs,
modality="image",
)
return num_image_tokens
@@ -954,6 +956,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
num_frames=num_frames,
image_processor=image_processor,
mm_kwargs=mm_kwargs,
modality="video",
)
return num_video_tokens
@@ -981,7 +984,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
if max_pixels is None:
image_processor = self.get_image_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
+9 -4
View File
@@ -941,6 +941,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
) -> tuple[ImageSize, int]:
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
@@ -950,7 +951,9 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs)
if modality is None:
modality = "video" if is_video else "image"
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
@@ -1014,7 +1017,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
) -> int:
video_processor = self.get_video_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({})
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
@@ -1150,7 +1153,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
video_processor = self.info.get_video_processor()
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({})
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
@@ -1365,7 +1368,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**hf_processor_mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_processor_mm_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(
hf_processor_mm_kwargs, modality="video"
)
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")