[Model] Extend device-side mm normalization to Qwen3VL/Qwen3.5/Qwen4Next (#51289)
Signed-off-by: cjackal <44624812+cjackal@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
07f1ba2ec6
commit
491f44adfa
@@ -134,6 +134,8 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction):
|
||||
|
||||
# Mark this as a pooling model so vLLM routes to pooler path
|
||||
is_pooling_model = True
|
||||
# Mark as false to avoid collision with the checkpoint's custom code path
|
||||
supports_mm_device_do_normalize = False
|
||||
|
||||
# Override hf_to_vllm_mapper to handle ColQwen3 weight naming.
|
||||
# NOTE: WeightsMapper applies ALL matching prefix rules sequentially
|
||||
|
||||
@@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
@@ -575,6 +576,7 @@ class InternS1ProForConditionalGeneration(
|
||||
self.visual = Qwen3_VisionTransformer(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
GateLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
@@ -496,6 +497,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
is_model_fused_shared_expert_compatible,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
@@ -517,6 +518,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
@@ -739,6 +741,7 @@ class Qwen3_5MoeForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
@@ -67,6 +67,10 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
MMEncoderAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
IdentityInputNorm,
|
||||
build_mm_input_norm,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
@@ -566,6 +570,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
vision_config: Qwen3VLVisionConfig,
|
||||
norm_eps: float = 1e-6,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
@@ -605,6 +610,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
in_channels=vision_config.in_channels,
|
||||
hidden_size=self.hidden_size,
|
||||
)
|
||||
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
||||
|
||||
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
|
||||
|
||||
@@ -858,7 +864,9 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
*,
|
||||
encoder_metadata: dict[str, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = x.to(device=self.device, dtype=self.dtype, non_blocking=True)
|
||||
hidden_states = self.input_norm(
|
||||
x.to(device=self.device, non_blocking=True), self.dtype
|
||||
)
|
||||
hidden_states = self.patch_embed(hidden_states)
|
||||
|
||||
if encoder_metadata is None:
|
||||
@@ -1813,6 +1821,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
supports_encoder_tp_data = True
|
||||
supports_tower_connector_lora = True
|
||||
supports_mm_device_do_normalize = True
|
||||
|
||||
supported_video_pruning_methods = ("evs", "vidcom2")
|
||||
|
||||
@@ -1863,6 +1872,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
@@ -2172,8 +2182,11 @@ class Qwen3VLForConditionalGeneration(
|
||||
flattened_patch_size = (
|
||||
in_channels * temporal_patch_size * patch_size * patch_size
|
||||
)
|
||||
dummy_pixel_values = torch.randn(
|
||||
total_patches, flattened_patch_size, device=device, dtype=dtype
|
||||
dummy_pixel_values = torch.zeros(
|
||||
total_patches,
|
||||
flattened_patch_size,
|
||||
device=device,
|
||||
dtype=self.visual.input_norm.input_dtype or dtype,
|
||||
)
|
||||
|
||||
# Override max_seqlen with a safe upper bound for capture.
|
||||
@@ -2307,7 +2320,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
if image_input["type"] == "image_embeds":
|
||||
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
|
||||
else:
|
||||
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
|
||||
pixel_values = image_input["pixel_values"]
|
||||
if self.use_data_parallel:
|
||||
return run_dp_sharded_mrope_vision_model(
|
||||
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
|
||||
@@ -2329,9 +2342,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
if video_input["type"] == "video_embeds":
|
||||
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
|
||||
else:
|
||||
pixel_values_videos = video_input["pixel_values_videos"].type(
|
||||
self.visual.dtype
|
||||
)
|
||||
pixel_values_videos = video_input["pixel_values_videos"]
|
||||
if self.use_data_parallel:
|
||||
grid_thw_list = grid_thw.tolist()
|
||||
return run_dp_sharded_mrope_vision_model(
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_pp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
@@ -254,6 +255,7 @@ class Qwen3VLMoeForConditionalGeneration(
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user