diff --git a/tests/models/multimodal/processing/test_deepseek_v4_vl.py b/tests/models/multimodal/processing/test_deepseek_v4_vl.py new file mode 100644 index 0000000000..5358fae970 --- /dev/null +++ b/tests/models/multimodal/processing/test_deepseek_v4_vl.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek-V4 VL processing helpers.""" + +from PIL import Image + +from vllm.models.deepseek_v4.common.mm_preprocess import ( + DeepseekV4VLProcessingInfo, + DeepseekV4VLProcessor, + build_image_block_pad_free, +) +from vllm.transformers_utils.configs.deepseek_v4 import DeepseekV4Config + + +class _ConfigCtx: + def __init__(self, config: DeepseekV4Config) -> None: + self._config = config + + def get_hf_config(self, *args, **kwargs): + return self._config + + +def _feature_counts( + processor: DeepseekV4VLProcessor, width: int, height: int +) -> tuple[int, int]: + out = processor(images=[Image.new("RGB", (width, height))]) + n_llm_h, n_llm_w = out["llm_grid"][0].tolist() + types, _ = build_image_block_pad_free(n_llm_h, n_llm_w) + return len(types), out["patches"].shape[0] + + +def test_get_image_size_with_most_features_beats_wide_real_image(): + """Dummy size must maximize tokens/patches within the budget (#59271). + + A square under-counts because each aligner row pays an IMAGE_NEW_LINE. + """ + config = DeepseekV4Config(vision_n_layers=24) + info = DeepseekV4VLProcessingInfo(_ConfigCtx(config)) + processor = DeepseekV4VLProcessor(config) + + dummy = info.get_image_size_with_most_features() + dummy_features = _feature_counts(processor, dummy.width, dummy.height) + wide_features = _feature_counts(processor, 1932, 336) + + assert dummy == (1932, 336) + assert wide_features <= dummy_features + + +def test_get_image_size_with_most_features_respects_max_wh_ratio(): + config = DeepseekV4Config(vision_n_layers=24) + info = DeepseekV4VLProcessingInfo(_ConfigCtx(config)) + dummy = info.get_image_size_with_most_features() + assert dummy.width / dummy.height <= config.vision_max_wh_ratio diff --git a/vllm/models/deepseek_v4/common/mm_preprocess.py b/vllm/models/deepseek_v4/common/mm_preprocess.py index d28848bb8f..62a5c2070f 100644 --- a/vllm/models/deepseek_v4/common/mm_preprocess.py +++ b/vllm/models/deepseek_v4/common/mm_preprocess.py @@ -354,15 +354,34 @@ class DeepseekV4VLProcessingInfo(BaseProcessingInfo): hf_config = self.get_hf_config() patch_size = hf_config.vision_patch_size downsample_ratio = hf_config.vision_downsample_ratio - # A square maximizes the ViT patch count (area) within the token - # budget; solve the budget-derived size directly to keep the dummy - # image small. + max_wh_ratio = hf_config.vision_max_wh_ratio + # Every aligner row costs an extra IMAGE_NEW_LINE token, so height is + # more expensive than width and a square is *not* optimal: search the + # LLM grids that fit the budget for the one with the most ViT patches + # (ties broken by the longer sentinel block). budget = hf_config.vision_max_n_token - (COMPRESS_PAD_TO - 1) - side = budget * patch_size * downsample_ratio - _, _, best_h, best_w, _ = solve_resize_ratio( - side, side, patch_size, downsample_ratio, budget - ) - return ImageSize(width=best_w, height=best_h) + step = patch_size * downsample_ratio + best_grid = (1, 1) + best_key = (0, 0) + for n_llm_h in range(1, budget): + if ( + grid_tokens(n_llm_h * step, step, patch_size, downsample_ratio)[2] + > budget + ): + break + for n_llm_w in range(1, budget): + if max_wh_ratio is not None and n_llm_w > n_llm_h * max_wh_ratio: + break + num_tokens = grid_tokens( + n_llm_h * step, n_llm_w * step, patch_size, downsample_ratio + )[2] + if num_tokens > budget: + break + key = (n_llm_h * n_llm_w, num_tokens) + if key > best_key: + best_key, best_grid = key, (n_llm_h, n_llm_w) + n_llm_h, n_llm_w = best_grid + return ImageSize(width=n_llm_w * step, height=n_llm_h * step) class DeepseekV4VLDummyInputsBuilder(