diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index f047f832b9..b55dad266e 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -48,18 +48,6 @@ VideoInput: TypeAlias = ( AudioInput = list[tuple[np.ndarray, int]] -MM_OPTIONS_OVERRIDES = { - # Qwen3-VL's default profiling video size (64x64) can cause trouble - # after resizing, so we override it here for testing. - "qwen3_vl": dict( - video=VideoDummyOptions(num_frames=128, width=256, height=256), - ), - "qwen3_vl_moe": dict( - video=VideoDummyOptions(num_frames=128, width=256, height=256), - ), -} - - def _resize_data( _data: Image.Image | np.ndarray, size_factor: float ) -> Image.Image | np.ndarray: @@ -73,12 +61,12 @@ def _resize_data( elif is_list_of(_data, Image.Image): W, H = next(iter(_data)).width, next(iter(_data)).height T = len(_data) - T, W, H = map(lambda x: max(int(x * size_factor), 1), (T, W, H)) + T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H)) return [d.resize((W, H)) for d in _data[:T]] # Video input with numpy arrays elif isinstance(_data, np.ndarray) and _data.ndim >= 4: T, H, W, C = _data.shape[-4:] - T, H, W = map(lambda x: max(int(x * size_factor), 1), (T, H, W)) + T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W)) return _data[..., :T, :H, :W, :C] # Audio input elif isinstance(_data, np.ndarray) and _data.ndim == 1: @@ -103,8 +91,6 @@ def create_batched_mm_kwargs( processor: BaseMultiModalProcessor, size_factors: tuple[float, ...] = (1.0, 0.5, 0.25), ) -> Iterable[tuple[str, int, BatchedTensorInputs]]: - model_type = model_config.hf_config.model_type - processing_info = processor.info dummy_inputs = processor.dummy_inputs supported_mm_limits = processing_info.get_supported_mm_limits() @@ -115,7 +101,6 @@ def create_batched_mm_kwargs( processor_inputs = dummy_inputs.get_dummy_processor_inputs( seq_len=model_config.max_model_len, mm_counts=mm_counts, - mm_options=MM_OPTIONS_OVERRIDES.get(model_type), ) mm_data = processor_inputs.mm_data resized_mm_data = { diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index ef5f6d1e45..0af303f0af 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -892,7 +892,9 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): ) return num_video_tokens - def get_image_size_with_most_features(self) -> ImageSize: + def get_image_size_with_most_features( + self, max_pixels: int | None = None + ) -> ImageSize: # NOTE: Simply processing a huge size with _get_vision_info might not give a # size that maximizes the number of featrues, i.e., the number of (merged) # patches. This is because the number of patches limits the allowed aspect @@ -910,8 +912,11 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo): vision_config = hf_config.vision_config patch_size = vision_config.patch_size merge_size = vision_config.spatial_merge_size - image_processor = self.get_image_processor() - max_pixels = image_processor.max_pixels or image_processor.size["longest_edge"] + if max_pixels is None: + image_processor = self.get_image_processor() + max_pixels = ( + image_processor.max_pixels or image_processor.size["longest_edge"] + ) unit = patch_size * merge_size max_seq_len = max_pixels // (unit * unit) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index ee9944fdfa..ac114e3c5b 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -91,6 +91,7 @@ from vllm.multimodal.processing import ( ) from vllm.sequence import IntermediateTensors from vllm.utils.collection_utils import is_list_of +from vllm.utils.math_utils import round_up from vllm.v1.attention.backends.registry import AttentionBackendEnum from .interfaces import ( @@ -129,8 +130,9 @@ from .vision import ( logger = init_logger(__name__) -# Official recommended max frames is 2048 -_MAX_FRAMES_PER_VIDEO = 2048 +# We use 2048 dummy video frames that would generate vision embeddings +# of the maximum size. +DUMMY_VIDEO_NUM_FRAMES = 2048 class Qwen3_VisionPatchEmbed(nn.Module): @@ -662,7 +664,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): else: preprocessed_size = ImageSize(width=image_width, height=image_height) - padded_num_frames = num_frames + num_frames % temporal_patch_size + padded_num_frames = round_up(num_frames, temporal_patch_size) grid_t = max(padded_num_frames // temporal_patch_size, 1) grid_h = preprocessed_size.height // patch_size @@ -684,7 +686,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): mm_counts: Mapping[str, int], ) -> int: return super().get_num_frames_with_most_features( - seq_len, mm_counts, max_frames_per_video=_MAX_FRAMES_PER_VIDEO + seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES ) def get_max_video_tokens( @@ -692,11 +694,17 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): seq_len: int, mm_counts: Mapping[str, int], ) -> int: - target_width, target_height = self.get_image_size_with_most_features() + video_processor = self.get_video_processor() + video_max_pixels = video_processor.size["longest_edge"] + # video_max_pixels contains the temporal compression factor, + # so we divide by 2 to get the maximum number of image pixels. + target_width, target_height = self.get_image_size_with_most_features( + max_pixels=video_max_pixels // video_processor.temporal_patch_size + ) num_video_soft_tokens = self.get_num_video_tokens( image_width=target_width, image_height=target_height, - num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts), + num_frames=2, image_processor=None, ) return num_video_soft_tokens @@ -779,11 +787,12 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): image_overrides = mm_options.get("image") if mm_options else None video_overrides = mm_options.get("video") if mm_options else None - target_width, target_height = self.info.get_image_size_with_most_features() - target_num_frames = self.info.get_num_frames_with_most_features( - seq_len, mm_counts + target_image_width, target_image_height = ( + self.info.get_image_size_with_most_features() ) + # treat videos as special images + target_num_frames = 2 if video_overrides: assert isinstance(video_overrides, VideoDummyOptions) num_frames_override = video_overrides.num_frames @@ -804,48 +813,60 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): target_num_frames = min(target_num_frames, num_frames_override) target_num_frames = max(target_num_frames, 2) + video_processor = self.info.get_video_processor() + video_max_pixels = video_processor.size["longest_edge"] + # video_max_pixels contains the temporal compression factor, + # so we divide by 2 to get the maximum number of image pixels. + target_video_width, target_video_height = ( + self.info.get_image_size_with_most_features( + max_pixels=video_max_pixels // video_processor.temporal_patch_size + ) + ) target_video_size, _ = self.info._get_vision_info( - image_width=target_width, - image_height=target_height, + image_width=target_video_width, + image_height=target_video_height, num_frames=target_num_frames, - image_processor=self.info.get_video_processor(), + image_processor=video_processor, ) # NOTE: we need to do this check here since Qwen3-VL resizes video # frames depending on how many frames there are. - width, height = target_video_size.width, target_video_size.height + target_video_width, target_video_height = ( + target_video_size.width, + target_video_size.height, + ) if video_overrides: assert isinstance(video_overrides, VideoDummyOptions) width_override = video_overrides.width if width_override: - if width_override > width: + if width_override > target_video_width: logger.warning( "video.width override (%d) exceeds model's " "maximum width (%d), will be ignored", width_override, - width, + target_video_width, ) - width = min(width, width_override) + target_video_width = min(target_video_width, width_override) height_override = video_overrides.height if height_override: - if height_override > height: + if height_override > target_video_height: logger.warning( "video.height override (%d) exceeds model's " "maximum height (%d), will be ignored", height_override, - height, + target_video_height, ) - height = min(height, height_override) + target_video_height = min(target_video_height, height_override) return { "image": self._get_dummy_images( - width=target_width, - height=target_height, + width=target_image_width, + height=target_image_height, num_images=num_images, overrides=image_overrides, ), "video": self._get_dummy_videos( - width=width, - height=height, + width=target_video_width, + height=target_video_height, num_frames=target_num_frames, num_videos=num_videos, ),