[Bugfix][Multimodal] Fix flat/scoped mm_processor_kwargs merge and resolution (#56372)

Signed-off-by: Daniel M. García-Ocaña Hernández <danielgarciaocana@gmail.com>
This commit is contained in:
Daniel M. García-Ocaña Hernández
2026-09-29 03:58:39 +00:00
committed by GitHub
parent 491f44adfa
commit 295a0b8d64
11 changed files with 1606 additions and 192 deletions
+9 -3
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
@@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
) -> PixtralHFEncoderInfo:
processor = self.get_hf_processor()
size = processor.image_processor.size
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
if override_size := merged_kwargs.get("size"):
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(
mm_processor_kwargs or {}
)
image_mm_kwargs = cast(
Mapping[str, Any],
merged_mm_kwargs.get("images_kwargs", {}),
)
if override_size := image_mm_kwargs.get("size"):
size = size | override_size
image_size = size["longest_edge"]
+130 -18
View File
@@ -28,7 +28,7 @@
import math
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias, cast
import numpy as np
import torch
@@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
@staticmethod
def _complete_mm_processor_size_aliases(
mm_kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Complete missing Qwen-VL size aliases.
Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and
``size.longest_edge`` / ``max_pixels`` as equivalent controls. When
only one representation is provided with a non-``None`` value, add the
missing equivalent representation. If both are already present, leave
them unchanged even when their values conflict.
Apply this independently to the flat kwargs and to each existing
``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated
between these mappings.
"""
def add_missing_size_aliases(
kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Add missing size aliases within a kwargs mapping."""
out_kwargs = dict(kwargs)
# Copy `size` separately because `out_kwargs` is a shallow copy and
# size edges may be added below.
raw_size = cast(
Mapping[str, object] | None,
out_kwargs.get("size"),
)
size_kwargs = {} if raw_size is None else dict(raw_size)
for edge_key, pixel_key in (
("shortest_edge", "min_pixels"),
("longest_edge", "max_pixels"),
):
# Add the pixel alias from `size` when only the size edge is
# present.
if pixel_key not in out_kwargs:
if edge_key in size_kwargs and size_kwargs[edge_key] is not None:
out_kwargs[pixel_key] = size_kwargs[edge_key]
continue
pixel_value = out_kwargs[pixel_key]
# HF treats `min_pixels` / `max_pixels` set to None as no size
# override, so do not propagate them to the equivalent size edge.
if pixel_value is None:
continue
# Add the size edge when only the pixel alias is present.
if edge_key not in size_kwargs:
size_kwargs[edge_key] = pixel_value
out_kwargs["size"] = size_kwargs
return out_kwargs
prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs)
# Complete aliases independently inside existing image/video kwargs;
# flat values are not copied into the nested mappings.
for nested_kwargs_key in ("images_kwargs", "videos_kwargs"):
nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key)
if isinstance(nested_kwargs, Mapping):
prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases(
nested_kwargs
)
return prepared_mm_kwargs
def _merge_and_resolve_mm_processor_kwargs(
self,
mm_kwargs: Mapping[str, object],
) -> dict[str, Any]:
"""Merge configured and request Qwen-VL ``mm_processor_kwargs``.
Request-side ``size.shortest_edge`` / ``min_pixels`` and
``size.longest_edge`` / ``max_pixels`` aliases are completed before the
merge so request precedence is preserved across equivalent
representations.
"""
# Complete request-side Qwen-VL size aliases before merging with
# configured kwargs.
prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs)
return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs)
@staticmethod
def _get_vision_size(
mm_kwargs: Mapping[str, object],
default_size: Mapping[str, int] | None = None,
) -> dict[str, int]:
"""Resolve Qwen-VL vision size kwargs with optional processor defaults.
``size`` overrides any processor defaults, then ``min_pixels`` and
``max_pixels`` override the corresponding size edges.
"""
size = dict(default_size) if default_size is not None else {}
override_size = cast(
Mapping[str, int] | None,
mm_kwargs.get("size"),
)
if override_size is not None:
size.update(override_size)
min_pixels = cast(int | None, mm_kwargs.get("min_pixels"))
if min_pixels is not None:
size["shortest_edge"] = min_pixels
max_pixels = cast(int | None, mm_kwargs.get("max_pixels"))
if max_pixels is not None:
size["longest_edge"] = max_pixels
return size
def get_mm_max_tokens_per_item(
self,
seq_len: int,
@@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
hf_config = self.get_hf_config()
vision_config = hf_config.vision_config
@@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
if modality is None:
modality = "image"
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "images_kwargs" if modality == "image" else "videos_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
resized_height, resized_width = smart_resize(
@@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
if max_pixels is None:
image_processor = self.get_image_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {})
size = self._get_vision_size(
image_mm_kwargs,
default_size=image_processor.size,
)
max_pixels = size["longest_edge"]
+107 -52
View File
@@ -27,7 +27,7 @@
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import lru_cache, partial
from itertools import islice
from typing import Any
from typing import Any, Literal
import numpy as np
import torch
@@ -944,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
supported = super().get_supported_mm_processor_kwargs()
# vLLM has historically accepted `min_pixels` / `max_pixels` for
# Qwen3-VL video. Include them in the routing schema even though
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
# folded into `size` before calling HF.
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
return supported
def _get_vision_info(
self,
*,
@@ -953,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
@@ -965,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
if modality is None:
modality = "video" if is_video else "image"
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
if is_video:
@@ -1030,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
) -> int:
video_processor = self.get_video_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
@@ -1162,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
video_processor = self.info.get_video_processor()
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self.info._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
@@ -1180,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
# over enough frames that the cap is not binding for the dummy.
# This takes precedence over a shrinking num_frames override, which
# would otherwise reintroduce the under-profiling.
if mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = mm_kwargs.get(
if video_mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = video_mm_kwargs.get(
"max_video_tokens",
getattr(video_processor, "max_video_tokens", 768),
)
patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
target_num_frames = max(
target_num_frames,
@@ -1348,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
# Merge and resolve the multimodal processor kwargs once so every HF
# call below starts from the same engine/request state and scoped form.
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
if "images_kwargs" in merged_mm_kwargs:
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_image_size = self.info.get_image_processor().size
image_scoped_kwargs["size"] = self.info._get_vision_size(
image_scoped_kwargs,
default_size=default_image_size,
)
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
if "videos_kwargs" in merged_mm_kwargs:
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_video_size = self.info.get_video_processor().size
video_scoped_kwargs["size"] = self.info._get_vision_size(
video_scoped_kwargs,
default_size=default_video_size,
)
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
# compatibility kwargs. They have already been folded into `size`,
# so remove them before any HF call, including calls that contain
# only text/images.
video_scoped_kwargs.pop("min_pixels", None)
video_scoped_kwargs.pop("max_pixels", None)
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
@@ -1373,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos.
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
video_mm_kwargs["size"] = video_size
sampled_fps = video_mm_kwargs.get("fps")
# The root copy is shallow, so give each item its own video
# scope before modifying per-video values below.
video_mm_kwargs = dict(merged_mm_kwargs)
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
sampled_fps = video_scoped_kwargs.get("fps")
if is_list_of(sampled_fps, float):
video_mm_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_mm_kwargs.get("num_frames")
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_scoped_kwargs.get("num_frames")
if is_list_of(sampled_num_frames, int):
video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_mm_kwargs:
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_scoped_kwargs:
# qwen_vl_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
video_scoped_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -1409,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# Compute timestamps here where we have access to metadata
timestamps = self.info._get_video_second_idx(
metadata=metadata,
do_sample_frames=video_mm_kwargs["do_sample_frames"],
sampled_fps=video_mm_kwargs.get("fps"),
sampled_num_frames=video_mm_kwargs.get("num_frames"),
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
sampled_fps=video_scoped_kwargs.get("fps"),
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
)
timestamps_per_video.append(timestamps)
@@ -1423,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# to prevent HF's BaseVideoProcessor.preprocess() from
# filling in the class default (fps=2) via setdefault(),
# which would conflict with num_frames (mutually exclusive).
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
video_mm_kwargs["fps"] = None
if (
"num_frames" in video_scoped_kwargs
and "fps" not in video_scoped_kwargs
):
video_scoped_kwargs["fps"] = None
video_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
@@ -1485,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
else:
video_outputs = dict()
# fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = {
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
}
# The root copy is shallow, so copy the video scope before removing
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
non_video_mm_kwargs = dict(merged_mm_kwargs)
if "videos_kwargs" in non_video_mm_kwargs:
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
non_video_scoped_kwargs.pop("fps", None)
non_video_scoped_kwargs.pop("num_frames", None)
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
hf_data,