Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-09-29 12:54:18 +07:00
264 changed files with 13095 additions and 1541 deletions
+5 -2
View File
@@ -526,6 +526,9 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
def load_weights(
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(self)
loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+2
View File
@@ -134,6 +134,8 @@ class ColQwen3Model(Qwen3VLForConditionalGeneration, SupportsLateInteraction):
# Mark this as a pooling model so vLLM routes to pooler path
is_pooling_model = True
# Mark as false to avoid collision with the checkpoint's custom code path
supports_mm_device_do_normalize = False
# Override hf_to_vllm_mapper to handle ColQwen3 weight naming.
# NOTE: WeightsMapper applies ALL matching prefix rules sequentially
@@ -1493,7 +1493,7 @@ class DiffusionSampler:
# [tile * W, vocab] copies, so a tile is also bounded by free memory.
widths_np = states.canvas_width_np[decode_slots_np]
order = np.argsort(widths_np, kind="stable")
free = current_platform.mem_get_info()[0] if num_decode > 0 else 0
free = torch.accelerator.get_memory_info()[0] if num_decode > 0 else 0
run_start = 0
while run_start < num_decode:
W = int(widths_np[order[run_start]])
+1 -1
View File
@@ -529,7 +529,7 @@ class FunAudioChatProcessingInfo(BaseProcessingInfo):
return TokenizersBackend.from_pretrained(
self.model_id,
subfolder="speech_tokenizer",
revision=self.ctx.model_config.tokenizer_revision,
revision=self.ctx.model_config.revision,
)
def get_feature_extractor(self) -> WhisperFeatureExtractor:
+13 -4
View File
@@ -846,13 +846,22 @@ class Gemma4SelfDecoderLayers(nn.Module):
# Shared references to modules owned by Gemma4Model — must be
# inside this nn.Module so torch.compile captures them.
self.embed_tokens = embed_tokens
self.normalizer = normalizer
self.embed_tokens_per_layer = embed_tokens_per_layer
self.embed_scale_per_layer = embed_scale_per_layer
self.per_layer_model_projection = per_layer_model_projection
self.per_layer_projection_norm = per_layer_projection_norm
self.per_layer_input_scale = per_layer_input_scale
self.per_layer_projection_scale = per_layer_projection_scale
# Register the shared scalars as buffers so they follow the model on
# .to(device); plain attributes stay on CPU.
self.register_buffer("normalizer", normalizer, persistent=False)
self.register_buffer(
"embed_scale_per_layer", embed_scale_per_layer, persistent=False
)
self.register_buffer(
"per_layer_input_scale", per_layer_input_scale, persistent=False
)
self.register_buffer(
"per_layer_projection_scale", per_layer_projection_scale, persistent=False
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids) * self.normalizer
@@ -45,6 +45,7 @@ from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -575,6 +576,7 @@ class InternS1ProForConditionalGeneration(
self.visual = Qwen3_VisionTransformer(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -21,6 +21,7 @@ from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
GateLinear,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
@@ -496,6 +497,7 @@ class InternS2MobiusForConditionalGeneration(InternS2PreviewForConditionalGenera
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+9 -3
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal
from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
@@ -212,8 +212,14 @@ class Mistral3ProcessingInfo(BaseProcessingInfo):
) -> PixtralHFEncoderInfo:
processor = self.get_hf_processor()
size = processor.image_processor.size
merged_kwargs = self.ctx.get_merged_mm_kwargs(mm_processor_kwargs or {})
if override_size := merged_kwargs.get("size"):
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(
mm_processor_kwargs or {}
)
image_mm_kwargs = cast(
Mapping[str, Any],
merged_mm_kwargs.get("images_kwargs", {}),
)
if override_size := image_mm_kwargs.get("size"):
size = size | override_size
image_size = size["longest_edge"]
+7 -3
View File
@@ -1976,7 +1976,8 @@ class Qwen2_5_VLForConditionalGeneration(
for _ in range(max_batch_size)
]
# Create dummy pixel_values
# Create dummy pixel_values; uint8 when normalization is fused
# on-device. Contents are overwritten before every replay.
patch_embed = self.visual.patch_embed
in_channels = patch_embed.proj.in_channels
patch_size = patch_embed.patch_size
@@ -1985,8 +1986,11 @@ class Qwen2_5_VLForConditionalGeneration(
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# Override max_seqlen with a safe upper bound for capture.
+137 -21
View File
@@ -28,7 +28,7 @@
import math
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Literal, TypeAlias
from typing import Annotated, Any, Literal, TypeAlias, cast
import numpy as np
import torch
@@ -865,6 +865,118 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
@staticmethod
def _complete_mm_processor_size_aliases(
mm_kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Complete missing Qwen-VL size aliases.
Qwen-VL accepts ``size.shortest_edge`` / ``min_pixels`` and
``size.longest_edge`` / ``max_pixels`` as equivalent controls. When
only one representation is provided with a non-``None`` value, add the
missing equivalent representation. If both are already present, leave
them unchanged even when their values conflict.
Apply this independently to the flat kwargs and to each existing
``images_kwargs`` / ``videos_kwargs`` mapping. Values are not propagated
between these mappings.
"""
def add_missing_size_aliases(
kwargs: Mapping[str, object],
) -> dict[str, object]:
"""Add missing size aliases within a kwargs mapping."""
out_kwargs = dict(kwargs)
# Copy `size` separately because `out_kwargs` is a shallow copy and
# size edges may be added below.
raw_size = cast(
Mapping[str, object] | None,
out_kwargs.get("size"),
)
size_kwargs = {} if raw_size is None else dict(raw_size)
for edge_key, pixel_key in (
("shortest_edge", "min_pixels"),
("longest_edge", "max_pixels"),
):
# Add the pixel alias from `size` when only the size edge is
# present.
if pixel_key not in out_kwargs:
if edge_key in size_kwargs and size_kwargs[edge_key] is not None:
out_kwargs[pixel_key] = size_kwargs[edge_key]
continue
pixel_value = out_kwargs[pixel_key]
# HF treats `min_pixels` / `max_pixels` set to None as no size
# override, so do not propagate them to the equivalent size edge.
if pixel_value is None:
continue
# Add the size edge when only the pixel alias is present.
if edge_key not in size_kwargs:
size_kwargs[edge_key] = pixel_value
out_kwargs["size"] = size_kwargs
return out_kwargs
prepared_mm_kwargs = add_missing_size_aliases(mm_kwargs)
# Complete aliases independently inside existing image/video kwargs;
# flat values are not copied into the nested mappings.
for nested_kwargs_key in ("images_kwargs", "videos_kwargs"):
nested_kwargs = prepared_mm_kwargs.get(nested_kwargs_key)
if isinstance(nested_kwargs, Mapping):
prepared_mm_kwargs[nested_kwargs_key] = add_missing_size_aliases(
nested_kwargs
)
return prepared_mm_kwargs
def _merge_and_resolve_mm_processor_kwargs(
self,
mm_kwargs: Mapping[str, object],
) -> dict[str, Any]:
"""Merge configured and request Qwen-VL ``mm_processor_kwargs``.
Request-side ``size.shortest_edge`` / ``min_pixels`` and
``size.longest_edge`` / ``max_pixels`` aliases are completed before the
merge so request precedence is preserved across equivalent
representations.
"""
# Complete request-side Qwen-VL size aliases before merging with
# configured kwargs.
prepared_mm_kwargs = self._complete_mm_processor_size_aliases(mm_kwargs)
return super()._merge_and_resolve_mm_processor_kwargs(prepared_mm_kwargs)
@staticmethod
def _get_vision_size(
mm_kwargs: Mapping[str, object],
default_size: Mapping[str, int] | None = None,
) -> dict[str, int]:
"""Resolve Qwen-VL vision size kwargs with optional processor defaults.
``size`` overrides any processor defaults, then ``min_pixels`` and
``max_pixels`` override the corresponding size edges.
"""
size = dict(default_size) if default_size is not None else {}
override_size = cast(
Mapping[str, int] | None,
mm_kwargs.get("size"),
)
if override_size is not None:
size.update(override_size)
min_pixels = cast(int | None, mm_kwargs.get("min_pixels"))
if min_pixels is not None:
size["shortest_edge"] = min_pixels
max_pixels = cast(int | None, mm_kwargs.get("max_pixels"))
if max_pixels is not None:
size["longest_edge"] = max_pixels
return size
def get_mm_max_tokens_per_item(
self,
seq_len: int,
@@ -883,7 +995,7 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
hf_config = self.get_hf_config()
vision_config = hf_config.vision_config
@@ -891,14 +1003,16 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
if modality is None:
modality = "image"
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "images_kwargs" if modality == "image" else "videos_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
resized_height, resized_width = smart_resize(
@@ -986,14 +1100,12 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
if max_pixels is None:
image_processor = self.get_image_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="image")
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
image_mm_kwargs = merged_mm_kwargs.get("images_kwargs", {})
size = self._get_vision_size(
image_mm_kwargs,
default_size=image_processor.size,
)
max_pixels = size["longest_edge"]
@@ -1616,7 +1728,8 @@ class Qwen2VLForConditionalGeneration(
for _ in range(max_batch_size)
]
# Create dummy pixel_values.
# Create dummy pixel_values; uint8 when normalization is fused
# on-device. Contents are overwritten before every replay.
patch_embed = self.visual.patch_embed
in_channels = patch_embed.proj.in_channels
patch_size = patch_embed.patch_size
@@ -1625,8 +1738,11 @@ class Qwen2VLForConditionalGeneration(
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# max_seqlen.item() gets baked into the CUDA graph at capture time.
+3
View File
@@ -39,6 +39,7 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.utils import (
is_model_fused_shared_expert_compatible,
)
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
@@ -519,6 +520,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -741,6 +743,7 @@ class Qwen3_5MoeForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -34,7 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.multimodal.inputs import NestedTensors
from vllm.platforms import current_platform
from vllm.transformers_utils.config import set_default_rope_theta
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
from vllm.v1.attention.backend import AttentionType
@@ -622,15 +621,6 @@ class DFlashQwen3Model(nn.Module):
# --- Grouped RMSNorm K across all layers ([L, num_ctx, nkv, hd]) ---
# The weight is selected per layer by the outermost (layer) index.
all_k_normed = torch.empty_like(all_k)
if current_platform.is_xpu():
for layer_idx in range(all_k.shape[0]):
ops.rms_norm(
all_k_normed[layer_idx],
all_k[layer_idx],
self._k_norm_weights[layer_idx],
self._rms_norm_eps,
)
return all_k_normed
ops.rms_norm(
all_k_normed,
all_k,
+125 -59
View File
@@ -27,7 +27,7 @@
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import lru_cache, partial
from itertools import islice
from typing import Any
from typing import Any, Literal
import numpy as np
import torch
@@ -67,6 +67,10 @@ from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
@@ -566,6 +570,7 @@ class Qwen3_VisionTransformer(nn.Module):
vision_config: Qwen3VLVisionConfig,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -605,6 +610,7 @@ class Qwen3_VisionTransformer(nn.Module):
in_channels=vision_config.in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
@@ -858,7 +864,9 @@ class Qwen3_VisionTransformer(nn.Module):
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
hidden_states = x.to(device=self.device, dtype=self.dtype, non_blocking=True)
hidden_states = self.input_norm(
x.to(device=self.device, non_blocking=True), self.dtype
)
hidden_states = self.patch_embed(hidden_states)
if encoder_metadata is None:
@@ -936,6 +944,16 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
supported = super().get_supported_mm_processor_kwargs()
# vLLM has historically accepted `min_pixels` / `max_pixels` for
# Qwen3-VL video. Include them in the routing schema even though
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
# folded into `size` before calling HF.
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
return supported
def _get_vision_info(
self,
*,
@@ -945,7 +963,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
mm_kwargs: Mapping[str, object],
modality: str | None = None,
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
@@ -957,14 +975,14 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
if modality is None:
modality = "video" if is_video else "image"
mm_kwargs = self.ctx.get_merged_mm_kwargs(mm_kwargs, modality=modality)
size = image_processor.size
if override_size := mm_kwargs.get("size"):
size = size | override_size
if (override_min_pixels := mm_kwargs.get("min_pixels")) is not None:
size = size | {"shortest_edge": override_min_pixels}
if (override_max_pixels := mm_kwargs.get("max_pixels")) is not None:
size = size | {"longest_edge": override_max_pixels}
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
if is_video:
@@ -1022,9 +1040,13 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
) -> int:
video_processor = self.get_video_processor()
mm_kwargs = self.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
@@ -1154,9 +1176,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
video_processor = self.info.get_video_processor()
mm_kwargs = self.info.ctx.get_merged_mm_kwargs({}, modality="video")
video_size = mm_kwargs.get("size", video_processor.size)
temporal_patch_size = mm_kwargs.get(
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self.info._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
@@ -1172,13 +1198,13 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
# over enough frames that the cap is not binding for the dummy.
# This takes precedence over a shrinking num_frames override, which
# would otherwise reintroduce the under-profiling.
if mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = mm_kwargs.get(
if video_mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = video_mm_kwargs.get(
"max_video_tokens",
getattr(video_processor, "max_video_tokens", 768),
)
patch_size = mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = mm_kwargs.get("merge_size", video_processor.merge_size)
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
target_num_frames = max(
target_num_frames,
@@ -1340,6 +1366,47 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
# Merge and resolve the multimodal processor kwargs once so every HF
# call below starts from the same engine/request state and scoped form.
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
if "images_kwargs" in merged_mm_kwargs:
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_image_size = self.info.get_image_processor().size
image_scoped_kwargs["size"] = self.info._get_vision_size(
image_scoped_kwargs,
default_size=default_image_size,
)
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
if "videos_kwargs" in merged_mm_kwargs:
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_video_size = self.info.get_video_processor().size
video_scoped_kwargs["size"] = self.info._get_vision_size(
video_scoped_kwargs,
default_size=default_video_size,
)
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
# compatibility kwargs. They have already been folded into `size`,
# so remove them before any HF call, including calls that contain
# only text/images.
video_scoped_kwargs.pop("min_pixels", None)
video_scoped_kwargs.pop("max_pixels", None)
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
@@ -1365,32 +1432,22 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos.
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
size_override = merged.get("size")
if size_override is not None:
video_size = video_size | size_override
min_pixels = merged.get("min_pixels")
if min_pixels is not None:
video_size["shortest_edge"] = min_pixels
max_pixels = merged.get("max_pixels")
if max_pixels is not None:
video_size["longest_edge"] = max_pixels
video_mm_kwargs["size"] = video_size
sampled_fps = video_mm_kwargs.get("fps")
# The root copy is shallow, so give each item its own video
# scope before modifying per-video values below.
video_mm_kwargs = dict(merged_mm_kwargs)
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
sampled_fps = video_scoped_kwargs.get("fps")
if is_list_of(sampled_fps, float):
video_mm_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_mm_kwargs.get("num_frames")
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_scoped_kwargs.get("num_frames")
if is_list_of(sampled_num_frames, int):
video_mm_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_mm_kwargs:
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_scoped_kwargs:
# qwen_vl_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it.
video_mm_kwargs["do_sample_frames"] = metadata.get(
video_scoped_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
@@ -1401,9 +1458,9 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# Compute timestamps here where we have access to metadata
timestamps = self.info._get_video_second_idx(
metadata=metadata,
do_sample_frames=video_mm_kwargs["do_sample_frames"],
sampled_fps=video_mm_kwargs.get("fps"),
sampled_num_frames=video_mm_kwargs.get("num_frames"),
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
sampled_fps=video_scoped_kwargs.get("fps"),
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
)
timestamps_per_video.append(timestamps)
@@ -1415,8 +1472,11 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# to prevent HF's BaseVideoProcessor.preprocess() from
# filling in the class default (fps=2) via setdefault(),
# which would conflict with num_frames (mutually exclusive).
if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
video_mm_kwargs["fps"] = None
if (
"num_frames" in video_scoped_kwargs
and "fps" not in video_scoped_kwargs
):
video_scoped_kwargs["fps"] = None
video_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
@@ -1477,11 +1537,14 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
else:
video_outputs = dict()
# fps/num_frames are video-only kwargs already consumed by the loop;
# exclude them so the text/image processor call below never gets a list.
non_video_mm_kwargs = {
k: v for k, v in hf_kwargs.items() if k not in ("fps", "num_frames")
}
# The root copy is shallow, so copy the video scope before removing
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
non_video_mm_kwargs = dict(merged_mm_kwargs)
if "videos_kwargs" in non_video_mm_kwargs:
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
non_video_scoped_kwargs.pop("fps", None)
non_video_scoped_kwargs.pop("num_frames", None)
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
hf_data,
@@ -1813,6 +1876,7 @@ class Qwen3VLForConditionalGeneration(
supports_encoder_tp_data = True
supports_tower_connector_lora = True
supports_mm_device_do_normalize = True
supported_video_pruning_methods = ("evs", "vidcom2")
@@ -1863,6 +1927,7 @@ class Qwen3VLForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
@@ -2172,8 +2237,11 @@ class Qwen3VLForConditionalGeneration(
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.randn(
total_patches, flattened_patch_size, device=device, dtype=dtype
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# Override max_seqlen with a safe upper bound for capture.
@@ -2307,7 +2375,7 @@ class Qwen3VLForConditionalGeneration(
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"].type(self.visual.dtype)
pixel_values = image_input["pixel_values"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
@@ -2329,9 +2397,7 @@ class Qwen3VLForConditionalGeneration(
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"].type(
self.visual.dtype
)
pixel_values_videos = video_input["pixel_values_videos"]
if self.use_data_parallel:
grid_thw_list = grid_thw.tolist()
return run_dp_sharded_mrope_vision_model(
@@ -36,6 +36,7 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -254,6 +255,7 @@ class Qwen3VLMoeForConditionalGeneration(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
+7 -4
View File
@@ -5,7 +5,7 @@ import itertools
import math
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import Final, Generic, Literal, Protocol, TypeAlias, TypeVar
from typing import Generic, Literal, Protocol, TypeAlias, TypeVar
import torch
from transformers import PreTrainedConfig
@@ -24,10 +24,12 @@ from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
_C = TypeVar("_C", bound=PreTrainedConfig)
_C_co = TypeVar("_C_co", bound=PreTrainedConfig, covariant=True)
class _RootConfig(Protocol[_C]):
vision_config: _C
class _RootConfig(Protocol[_C_co]):
@property
def vision_config(self) -> _C_co: ...
class VisionEncoderInfo(ABC, Generic[_C]):
@@ -60,7 +62,8 @@ class VisionEncoderInfo(ABC, Generic[_C]):
class VisionLanguageConfig(Protocol):
vision_config: Final[PreTrainedConfig]
@property
def vision_config(self) -> PreTrainedConfig: ...
def get_vision_encoder_info(hf_config: VisionLanguageConfig) -> VisionEncoderInfo:
+4 -3
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from functools import partial
from typing import cast
@@ -317,7 +317,7 @@ class VoxtralForConditionalGeneration(
# update quant config to so that ignored module and target module names
# match the vLLM model names
if hasattr(vllm_config, "quant_config"):
if vllm_config.quant_config is not None:
vllm_config.quant_config = self.maybe_update_quant_config(
vllm_config.quant_config
)
@@ -724,6 +724,7 @@ class VoxtralEncoderModel(nn.Module):
self.config = cast(WhisperConfig, vllm_config.model_config.hf_config)
self.dtype: torch.dtype = vllm_config.model_config.dtype
self.is_causal = getattr(self.config, "is_causal", False)
WhisperEncoderCls: Callable[..., WhisperEncoder | WhisperCausalEncoder]
if self.is_causal:
WhisperEncoderCls = WhisperCausalEncoder
else:
@@ -830,7 +831,7 @@ class VoxtralEncoderModel(nn.Module):
return results
def load_weight(self, weight: tuple[str, torch.Tensor]) -> str:
stacked_params_mapping = [
stacked_params_mapping: list[tuple[str, str, str | int]] = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
+12 -11
View File
@@ -16,7 +16,6 @@ from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.engine.protocol import StreamingInput
from vllm.inputs import PromptType, TokensPrompt
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime
@@ -26,6 +25,7 @@ from vllm.model_executor.models.voxtral import (
VoxtralMultiModalProcessor,
VoxtralProcessingInfo,
)
from vllm.model_executor.models.whisper_causal import WhisperCausalEncoder
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import ProcessorInputs, TimingContext
@@ -56,7 +56,7 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
mm_items: MultiModalDataItems,
mm_res: MultiModalProcessingResult,
) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]:
mm_kwargs = mm_res.kwargs
mm_kwargs = mm_res.kwargs.require_data()
# there are no placeholder audio tokens for streaming
# so we need to build the place placeholder positions manually
@@ -69,7 +69,9 @@ class VoxtralRealtimeMultiModalProcessor(VoxtralMultiModalProcessor):
tokenizer = self.info.get_tokenizer()
audio_config = tokenizer.instruct.audio_encoder.audio_config
num_audio_samples = audios[0]["audio_arrays"].data.shape[0]
audio_array = audios[0]["audio_arrays"].data
assert isinstance(audio_array, (torch.Tensor, np.ndarray))
num_audio_samples = audio_array.shape[0]
length = audio_config.num_audio_tokens(num_audio_samples)
features_info = PlaceholderFeaturesInfo(
@@ -162,7 +164,7 @@ class VoxtralRealtimeBuffer:
for token in tokens:
await self._token_queue.put(token)
async def get_input_stream(self) -> AsyncGenerator[StreamingInput]:
async def get_input_stream(self) -> AsyncGenerator[TokensPrompt]:
for frame_size, num_tokens in self._generate_frame_size_and_num_tokens():
next_tokens = [await self._token_queue.get() for _ in range(num_tokens)]
@@ -188,11 +190,9 @@ class VoxtralRealtimeBuffer:
self._leftover = audio_array[stride:]
yield StreamingInput(
TokensPrompt(
prompt_token_ids=next_tokens,
multi_modal_data={"audio": (frame, None)},
)
yield TokensPrompt(
prompt_token_ids=next_tokens,
multi_modal_data={"audio": (frame, None)},
)
@@ -274,8 +274,8 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
token_task = asyncio.create_task(feed_tokens())
try:
async for streaming_input in buffer.get_input_stream():
yield streaming_input.prompt
async for prompt in buffer.get_input_stream():
yield prompt
finally:
audio_task.cancel()
token_task.cancel()
@@ -420,6 +420,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
seq_lens = [mel.shape[1] for mel in mel_features]
# [total_num_20ms_frames, hidden_size]
assert isinstance(self.whisper_encoder.whisper_encoder, WhisperCausalEncoder)
audio_embeddings = self.whisper_encoder.whisper_encoder.forward_conv(
mel_features
)