diff --git a/docs/design/mm_processing.md b/docs/design/mm_processing.md index 8e49a4099c..d350836ca8 100644 --- a/docs/design/mm_processing.md +++ b/docs/design/mm_processing.md @@ -78,10 +78,17 @@ GPU-side fusion is controlled by `multimodal_config.mm_device_do_normalize`: - The flag is **enabled by default** for all models that support it. - Currently, it’s on by default for these architectures: -| name | Architecture | Example HF Models | -|--------------|--------------------------------------|-------------------------------------| -| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | -| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | +| name | Architecture | Example HF Models | +|--------------------|------------------------------------------|----------------------------------------| +| `intern-s1-pro` | `InternS1ProForConditionalGeneration` | `internlm/Intern-S1-Pro` | +| `intern-s2-mobius` | `InternS2MobiusForConditionalGeneration` | `internlm/Intern-S2-Mobius` | +| `qwen2-vl` | `Qwen2VLForConditionalGeneration` | `Qwen/Qwen2-VL-2B-Instruct`, etc. | +| `qwen2.5-vl` | `Qwen2_5_VLForConditionalGeneration` | `Qwen/Qwen2.5-VL-3B-Instruct`, etc. | +| `qwen3-vl` | `Qwen3VLForConditionalGeneration` | `Qwen/Qwen3-VL-4B-Instruct`, etc. | +| `qwen3-vl-moe` | `Qwen3VLMoeForConditionalGeneration` | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | +| `qwen3.5` | `Qwen3_5ForConditionalGeneration` | `Qwen/Qwen3.5-0.8B`, etc. | +| `qwen3.5-moe` | `Qwen3_5MoeForConditionalGeneration` | `Qwen/Qwen3.5-35B-A3B`, etc. | +| `qwen4-exp` | `Qwen4ExpForConditionalGeneration` | `Qwen/Qwen3.8-Flash-Next` | #### Key Properties and Gains diff --git a/tests/models/multimodal/generation_ppl_test/ppl_utils.py b/tests/models/multimodal/generation_ppl_test/ppl_utils.py index 317c583204..c612eb7141 100644 --- a/tests/models/multimodal/generation_ppl_test/ppl_utils.py +++ b/tests/models/multimodal/generation_ppl_test/ppl_utils.py @@ -34,6 +34,7 @@ def vqa_ppl_test( mm_processor_kwargs = {} processor = AutoProcessor.from_pretrained(model_info.name) + image_token_id = processor.image_token_id images = [] prompts = [] @@ -99,8 +100,10 @@ def vqa_ppl_test( for token_data in token_datas[1:]: assert token_data is not None assert len(token_data) == 1 - token_log_prob = list(token_data.values())[0].logprob - token_log_probs.append(token_log_prob) + token_id, token_log_prob = next(iter(token_data.items())) + if token_id == image_token_id: + continue + token_log_probs.append(token_log_prob.logprob) neg_log_likelihood = -torch.tensor( token_log_probs, dtype=torch.float32, device="cpu" @@ -132,11 +135,12 @@ def vqa_ppl_test( inputs = inputs.to("cuda") input_ids = inputs["input_ids"] - outputs = hf_model.model(**inputs, labels=input_ids) + labels = input_ids.masked_fill(input_ids == image_token_id, -100) + outputs = hf_model.model(**inputs, labels=labels) neg_log_likelihood = outputs.loss neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu() - num_loss_tokens = input_ids.shape[1] - 1 + num_loss_tokens = int((labels[:, 1:] != -100).sum()) nll_sum += neg_log_likelihood * num_loss_tokens n_tokens += num_loss_tokens diff --git a/tests/models/multimodal/generation_ppl_test/test_qwen.py b/tests/models/multimodal/generation_ppl_test/test_qwen.py index 39f6877cf6..63742b600f 100644 --- a/tests/models/multimodal/generation_ppl_test/test_qwen.py +++ b/tests/models/multimodal/generation_ppl_test/test_qwen.py @@ -8,26 +8,51 @@ from tests.models.utils import GenerateModelInfo from .ppl_utils import vqa_ppl_test MODELS = [ - GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=41081356.0), - GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=18330016.0), + ( + GenerateModelInfo("Qwen/Qwen2-VL-2B-Instruct", hf_ppl=9.810060501098633), + 28, + {}, + ), + ( + GenerateModelInfo("Qwen/Qwen2.5-VL-3B-Instruct", hf_ppl=22.17719078063965), + 28, + {}, + ), + ( + GenerateModelInfo("Qwen/Qwen3-VL-4B-Instruct", hf_ppl=11.916606903076172), + 32, + {}, + ), + ( + GenerateModelInfo("Qwen/Qwen3.5-4B", hf_ppl=6.478141784667969), + 32, + {"enable_prefix_caching": False}, + ), ] -mm_processor_kwargs = { - "min_pixels": 28 * 28, - "max_pixels": 1280 * 28 * 28, -} +def _pixel_factor(factor: int): + """`factor = patch_size * merge_size` for Qwen models""" + return {"min_pixels": factor**2, "max_pixels": 1280 * factor**2} -@pytest.mark.parametrize("model_info", MODELS) +@pytest.mark.parametrize("model_info,factor,extra_kwargs", MODELS) @pytest.mark.parametrize("mm_device_do_normalize", [True, False]) def test_ppl( - hf_runner, vllm_runner, model_info: GenerateModelInfo, mm_device_do_normalize: bool + hf_runner, + vllm_runner, + model_info: GenerateModelInfo, + factor: int, + extra_kwargs: dict, + mm_device_do_normalize: bool, ): vqa_ppl_test( hf_runner, vllm_runner, model_info, - vllm_extra_kwargs={"mm_device_do_normalize": mm_device_do_normalize}, - mm_processor_kwargs=mm_processor_kwargs, + vllm_extra_kwargs={ + "mm_device_do_normalize": mm_device_do_normalize, + **extra_kwargs, + }, + mm_processor_kwargs=_pixel_factor(factor), ) diff --git a/vllm/model_executor/models/colqwen3.py b/vllm/model_executor/models/colqwen3.py index 0e92363001..e5cdcc773c 100644 --- a/vllm/model_executor/models/colqwen3.py +++ b/vllm/model_executor/models/colqwen3.py @@ -134,6 +134,8 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction): # Mark this as a pooling model so vLLM routes to pooler path is_pooling_model = True + # Mark as false to avoid collision with the checkpoint's custom code path + supports_mm_device_do_normalize = False # Override hf_to_vllm_mapper to handle ColQwen3 weight naming. # NOTE: WeightsMapper applies ALL matching prefix rules sequentially diff --git a/vllm/model_executor/models/interns1_pro.py b/vllm/model_executor/models/interns1_pro.py index e757241e94..629d0de3dc 100644 --- a/vllm/model_executor/models/interns1_pro.py +++ b/vllm/model_executor/models/interns1_pro.py @@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -575,6 +576,7 @@ class InternS1ProForConditionalGeneration( self.visual = Qwen3_VisionTransformer( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/interns2_mobius.py b/vllm/model_executor/models/interns2_mobius.py index c3fece062f..e186dc575b 100644 --- a/vllm/model_executor/models/interns2_mobius.py +++ b/vllm/model_executor/models/interns2_mobius.py @@ -21,6 +21,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, GateLinear, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( @@ -496,6 +497,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index ac27296461..2f5e01f7ae 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -39,6 +39,7 @@ from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( @@ -517,6 +518,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -739,6 +741,7 @@ class Qwen3_5MoeForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index c9aa0d2066..d534d408ca 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -67,6 +67,10 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import ( MMEncoderAttention, ) from vllm.model_executor.layers.conv import Conv3dLayer +from vllm.model_executor.layers.fusion.mm_input_norm import ( + IdentityInputNorm, + build_mm_input_norm, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, RowParallelLinear, @@ -566,6 +570,7 @@ class Qwen3_VisionTransformer(nn.Module): vision_config: Qwen3VLVisionConfig, norm_eps: float = 1e-6, quant_config: QuantizationConfig | None = None, + input_norm: nn.Module | None = None, prefix: str = "", ) -> None: super().__init__() @@ -605,6 +610,7 @@ class Qwen3_VisionTransformer(nn.Module): in_channels=vision_config.in_channels, hidden_size=self.hidden_size, ) + self.input_norm = input_norm if input_norm is not None else IdentityInputNorm() self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size) @@ -858,7 +864,9 @@ class Qwen3_VisionTransformer(nn.Module): *, encoder_metadata: dict[str, torch.Tensor] | None = None, ) -> torch.Tensor: - hidden_states = x.to(device=self.device, dtype=self.dtype, non_blocking=True) + hidden_states = self.input_norm( + x.to(device=self.device, non_blocking=True), self.dtype + ) hidden_states = self.patch_embed(hidden_states) if encoder_metadata is None: @@ -1813,6 +1821,7 @@ class Qwen3VLForConditionalGeneration( supports_encoder_tp_data = True supports_tower_connector_lora = True + supports_mm_device_do_normalize = True supported_video_pruning_methods = ("evs", "vidcom2") @@ -1863,6 +1872,7 @@ class Qwen3VLForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) @@ -2172,8 +2182,11 @@ class Qwen3VLForConditionalGeneration( flattened_patch_size = ( in_channels * temporal_patch_size * patch_size * patch_size ) - dummy_pixel_values = torch.randn( - total_patches, flattened_patch_size, device=device, dtype=dtype + dummy_pixel_values = torch.zeros( + total_patches, + flattened_patch_size, + device=device, + dtype=self.visual.input_norm.input_dtype or dtype, ) # Override max_seqlen with a safe upper bound for capture. @@ -2307,7 +2320,7 @@ class Qwen3VLForConditionalGeneration( if image_input["type"] == "image_embeds": image_embeds = image_input["image_embeds"].type(self.visual.dtype) else: - pixel_values = image_input["pixel_values"].type(self.visual.dtype) + pixel_values = image_input["pixel_values"] if self.use_data_parallel: return run_dp_sharded_mrope_vision_model( self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d" @@ -2329,9 +2342,7 @@ class Qwen3VLForConditionalGeneration( if video_input["type"] == "video_embeds": video_embeds = video_input["video_embeds"].type(self.visual.dtype) else: - pixel_values_videos = video_input["pixel_values_videos"].type( - self.visual.dtype - ) + pixel_values_videos = video_input["pixel_values_videos"] if self.use_data_parallel: grid_thw_list = grid_thw.tolist() return run_dp_sharded_mrope_vision_model( diff --git a/vllm/model_executor/models/qwen3_vl_moe.py b/vllm/model_executor/models/qwen3_vl_moe.py index a345f577b1..4d0c7788a6 100644 --- a/vllm/model_executor/models/qwen3_vl_moe.py +++ b/vllm/model_executor/models/qwen3_vl_moe.py @@ -36,6 +36,7 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed import get_pp_group from vllm.logger import init_logger +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.multimodal import MULTIMODAL_REGISTRY @@ -254,6 +255,7 @@ class Qwen3VLMoeForConditionalGeneration( config.vision_config, norm_eps=getattr(config, "rms_norm_eps", 1e-6), quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/qwen4_exp/amd/model.py b/vllm/models/qwen4_exp/amd/model.py index b320967892..db94a2a7d6 100644 --- a/vllm/models/qwen4_exp/amd/model.py +++ b/vllm/models/qwen4_exp/amd/model.py @@ -14,6 +14,7 @@ from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( QwenGatedDeltaNetAttention, @@ -897,6 +898,7 @@ class Qwen4ExpForConditionalGeneration( config.vision_config, norm_eps=config.text_config.rms_norm_eps, quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), ) diff --git a/vllm/models/qwen4_exp/nvidia/model.py b/vllm/models/qwen4_exp/nvidia/model.py index a7e24bcd2a..6f35bea130 100644 --- a/vllm/models/qwen4_exp/nvidia/model.py +++ b/vllm/models/qwen4_exp/nvidia/model.py @@ -13,6 +13,7 @@ from vllm.distributed import get_pp_group from vllm.model_executor.layers.fused_moe.utils import ( is_model_fused_shared_expert_compatible, ) +from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( QwenGatedDeltaNetAttention, @@ -939,6 +940,7 @@ class Qwen4ExpForConditionalGeneration( config.vision_config, norm_eps=config.text_config.rms_norm_eps, quant_config=quant_config, + input_norm=build_mm_input_norm(self.model_config), prefix=maybe_prefix(prefix, "visual"), )