Files
MOVZXandHinata 8dc54f35dd [Offloader] Stream every multimodal tower parameter to host RAM
Qwen3_VisionTransformer derived its compute device from
patch_embed.proj.weight and fed pos_embed.weight straight into the
position-embedding interpolation kernel, so the last two anchor weights
had to stay in VRAM for a tower to be streamable at all.

Record the device the tower was built on before the offloader moves
anything, and fall back to it when no weight is on the accelerator.
Move the interpolation table to that device per encode, which costs one
8.4 MiB host-to-device copy for a 456 MiB tower. Towers now leave no
parameters in VRAM between encodes; only small buffers such as the fused
input norm and rotary inv_freq remain.

Also let a tower restrict streaming through tower_offload_children, and
warn when a tower resolves its device property to another device type,
so a tower that reads an anchor weight reports the cause instead of a
Triton pointer error.

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
2026-10-16 16:07:53 +07:00

3278 lines
124 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The vLLM team.
# Copyright 2025 The Qwen Team.
# Copyright 2025 The HuggingFace Inc. team.
# All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only Qwen3VL model compatible with HuggingFace weights."""
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
from functools import lru_cache, partial
from itertools import islice
from typing import Any, Literal
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import BatchFeature, ProcessorMixin
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
smart_resize as image_smart_resize,
)
from transformers.models.qwen3_vl import Qwen3VLProcessor, Qwen3VLVideoProcessor
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
Qwen3VLConfig,
Qwen3VLVisionConfig,
)
from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
smart_resize as video_smart_resize,
)
from transformers.video_utils import VideoMetadata
from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
support_torch_compile,
)
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.config.multimodal import (
MultiModalConfig,
MultiModalDummyOptions,
VideoDummyOptions,
VideoPruningMethod,
)
from vllm.distributed import get_pp_group, parallel_state
from vllm.inputs import MultiModalDataDict, MultiModalHashes
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
from vllm.model_executor.layers.attention.mm_encoder_attention import (
MMEncoderAttention,
)
from vllm.model_executor.layers.conv import Conv3dLayer
from vllm.model_executor.layers.fusion.mm_input_norm import (
IdentityInputNorm,
build_mm_input_norm,
)
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalKwargsItems,
PlaceholderRange,
VideoItem,
)
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.video_prune.evs import (
compute_mrope_for_media,
compute_retained_tokens_count,
compute_retention_mask,
recompute_mrope_positions,
)
from vllm.multimodal.video_prune.vidcom2 import (
compute_retained_tokens_count as vidcom2_compute_retained_tokens_count,
)
from vllm.multimodal.video_prune.vidcom2 import (
compute_retention_mask as vidcom2_compute_retention_mask,
)
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.protocol import TokenizerLike
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.triton_utils import HAS_TRITON, tl, triton
from vllm.utils.cache import LRUCache
from vllm.utils.collection_utils import is_list_of
from vllm.utils.math_utils import cdiv, round_up
from vllm.utils.torch_utils import PIN_MEMORY
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
from ...utils.gpu_sync_debug import gpu_sync_allowed
from ...utils.torch_utils import async_tensor_h2d
from .interfaces import (
MultiModalEmbeddings,
SupportsEagle,
SupportsEagle3,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsMRoPE,
SupportsMultiModal,
SupportsMultiModalPruning,
SupportsPP,
_require_is_multimodal,
)
from .qwen2_5_vl import (
Qwen2_5_VisionAttention,
Qwen2_5_VLImageEmbeddingInputs,
Qwen2_5_VLImageInputs,
Qwen2_5_VLImagePixelInputs,
Qwen2_5_VLVideoEmbeddingInputs,
Qwen2_5_VLVideoInputs,
Qwen2_5_VLVideoPixelInputs,
)
from .qwen2_vl import (
Qwen2VLMultiModalDataParser,
Qwen2VLProcessingInfo,
_create_qwen2vl_field_factory,
)
from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
StageMissingLayer,
WeightsMapper,
_merge_multimodal_embeddings,
maybe_prefix,
)
from .vision import (
get_fp8_padded_hidden_size,
get_vit_attn_backend,
is_vit_use_data_parallel,
run_dp_sharded_mrope_vision_model,
)
logger = init_logger(__name__)
# We use 2048 dummy video frames that would generate vision embeddings
# of the maximum size.
DUMMY_VIDEO_NUM_FRAMES = 2048
# ---------------------------------------------------------------------------
# Triton kernel: fused bilinear position-embedding interpolation
# ---------------------------------------------------------------------------
# Replaces many small eager-mode CUDA kernels with a single launch.
# The spatial-merge reorder is baked into the index math so the output
# is ready to be added to the patch embeddings directly.
# ---------------------------------------------------------------------------
if HAS_TRITON:
@triton.jit
def _bilinear_pos_embed_kernel(
embed_ptr,
output_ptr,
H,
W,
h_scale,
w_scale,
NUM_GRID: tl.constexpr,
M_SIZE: tl.constexpr,
HIDDEN_DIM: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""Fused bilinear pos-embed interpolation with spatial-merge reorder."""
pid = tl.program_id(0)
total_spatial = H * W
spatial_idx = pid % total_spatial
num_blocks_w = W // M_SIZE
block_idx = spatial_idx // (M_SIZE * M_SIZE)
local_idx = spatial_idx % (M_SIZE * M_SIZE)
br = block_idx // num_blocks_w
bc = block_idx % num_blocks_w
lr = local_idx // M_SIZE
lc = local_idx % M_SIZE
row = br * M_SIZE + lr
col = bc * M_SIZE + lc
h_frac = row.to(tl.float32) * h_scale
w_frac = col.to(tl.float32) * w_scale
hf = tl.math.floor(h_frac).to(tl.int32)
wf = tl.math.floor(w_frac).to(tl.int32)
hc = tl.minimum(hf + 1, NUM_GRID - 1)
wc = tl.minimum(wf + 1, NUM_GRID - 1)
dh = h_frac - hf.to(tl.float32)
dw = w_frac - wf.to(tl.float32)
w11 = dh * dw
w10 = dh - w11
w01 = dw - w11
w00 = 1.0 - dh - w01
off00 = (hf * NUM_GRID + wf) * HIDDEN_DIM
off01 = (hf * NUM_GRID + wc) * HIDDEN_DIM
off10 = (hc * NUM_GRID + wf) * HIDDEN_DIM
off11 = (hc * NUM_GRID + wc) * HIDDEN_DIM
out_off = pid * HIDDEN_DIM
# Cast weights to output dtype so the multiply-accumulate stays
# in the same precision as the native PyTorch implementation.
out_dtype = output_ptr.dtype.element_ty
w00_c = w00.to(out_dtype)
w01_c = w01.to(out_dtype)
w10_c = w10.to(out_dtype)
w11_c = w11.to(out_dtype)
for d in tl.range(0, HIDDEN_DIM, BLOCK_D):
cols = d + tl.arange(0, BLOCK_D)
mask = cols < HIDDEN_DIM
e00 = tl.load(embed_ptr + off00 + cols, mask=mask)
e01 = tl.load(embed_ptr + off01 + cols, mask=mask)
e10 = tl.load(embed_ptr + off10 + cols, mask=mask)
e11 = tl.load(embed_ptr + off11 + cols, mask=mask)
val = w00_c * e00 + w01_c * e01 + w10_c * e10 + w11_c * e11
tl.store(output_ptr + out_off + cols, val, mask=mask)
def triton_pos_embed_interpolate(
embed_weight: torch.Tensor,
t: int,
h: int,
w: int,
num_grid_per_side: int,
m_size: int,
dtype: torch.dtype,
) -> torch.Tensor:
"""Launch the fused Triton kernel for one (t,h,w) grid.
Returns a tensor of shape ``(t * h * w, hidden_dim)`` with the
bilinearly-interpolated position embeddings in spatial-merge order.
"""
assert h % m_size == 0 and w % m_size == 0, (
f"h={h} and w={w} must be divisible by m_size={m_size}"
)
hidden_dim = embed_weight.shape[1]
total_out = t * h * w
output = torch.empty(
total_out,
hidden_dim,
device=embed_weight.device,
dtype=dtype,
)
h_scale = float(num_grid_per_side - 1) / float(h - 1) if h > 1 else 0.0
w_scale = float(num_grid_per_side - 1) / float(w - 1) if w > 1 else 0.0
BLOCK_D = triton.next_power_of_2(hidden_dim)
_bilinear_pos_embed_kernel[(total_out,)](
embed_weight,
output,
h,
w,
h_scale,
w_scale,
num_grid_per_side,
m_size,
hidden_dim,
BLOCK_D,
)
return output
def pos_embed_interpolate_native(
embed_weight: torch.Tensor,
t: int,
h: int,
w: int,
num_grid_per_side: int,
m_size: int,
dtype: torch.dtype,
) -> torch.Tensor:
"""Eager PyTorch bilinear position-embedding interpolation.
Returns a tensor of shape ``(t * h * w, hidden_dim)`` with the
bilinearly-interpolated position embeddings in spatial-merge order.
"""
assert h % m_size == 0 and w % m_size == 0, (
f"h={h} and w={w} must be divisible by m_size={m_size}"
)
hidden_dim = embed_weight.shape[1]
device = embed_weight.device
h_idxs = torch.linspace(
0,
num_grid_per_side - 1,
h,
dtype=torch.float32,
device=device,
)
w_idxs = torch.linspace(
0,
num_grid_per_side - 1,
w,
dtype=torch.float32,
device=device,
)
h_floor = h_idxs.to(torch.long)
w_floor = w_idxs.to(torch.long)
h_ceil = torch.clamp(h_floor + 1, max=num_grid_per_side - 1)
w_ceil = torch.clamp(w_floor + 1, max=num_grid_per_side - 1)
dh = h_idxs - h_floor
dw = w_idxs - w_floor
dh_grid, dw_grid = torch.meshgrid(dh, dw, indexing="ij")
h_floor_grid, w_floor_grid = torch.meshgrid(h_floor, w_floor, indexing="ij")
h_ceil_grid, w_ceil_grid = torch.meshgrid(h_ceil, w_ceil, indexing="ij")
w11 = dh_grid * dw_grid
w10 = dh_grid - w11
w01 = dw_grid - w11
w00 = 1 - dh_grid - w01
h_grid = torch.stack([h_floor_grid, h_floor_grid, h_ceil_grid, h_ceil_grid])
w_grid = torch.stack([w_floor_grid, w_ceil_grid, w_floor_grid, w_ceil_grid])
h_grid_idx = h_grid * num_grid_per_side
indices = (h_grid_idx + w_grid).reshape(4, -1)
weights = torch.stack([w00, w01, w10, w11], dim=0).reshape(4, -1, 1)
weights = weights.to(dtype=dtype)
embeds = embed_weight[indices]
embeds *= weights
combined = embeds.sum(dim=0)
combined = combined.reshape(h // m_size, m_size, w // m_size, m_size, hidden_dim)
combined = combined.permute(0, 2, 1, 3, 4).reshape(1, -1, hidden_dim)
repeated = combined.expand(t, -1, -1).reshape(-1, hidden_dim)
return repeated.to(dtype=dtype)
@support_torch_compile(
dynamic_arg_dims={
"x": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class Qwen3_VisionPatchEmbed(nn.Module):
def __init__(
self,
patch_size: int = 14,
temporal_patch_size: int = 2,
in_channels: int = 3,
hidden_size: int = 1152,
) -> None:
super().__init__()
self.patch_size = patch_size
self.temporal_patch_size = temporal_patch_size
self.hidden_size = hidden_size
kernel_size = (temporal_patch_size, patch_size, patch_size)
self.proj = Conv3dLayer(
in_channels,
hidden_size,
kernel_size=kernel_size,
stride=kernel_size,
bias=True,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
L, C = x.shape
x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
x = self.proj(x).view(L, self.hidden_size)
return x
class Qwen3_VisionMLP(nn.Module):
def __init__(
self,
in_features: int,
hidden_features: int,
bias: bool = False,
act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.linear_fc1 = ColumnParallelLinear(
in_features,
hidden_features,
bias=bias,
quant_config=quant_config,
return_bias=False,
prefix=f"{prefix}.linear_fc1",
disable_tp=use_data_parallel,
)
self.linear_fc2 = RowParallelLinear(
hidden_features,
in_features,
bias=bias,
quant_config=quant_config,
return_bias=False,
prefix=f"{prefix}.linear_fc2",
disable_tp=use_data_parallel,
)
self.act_fn = act_fn
def forward(self, x: torch.Tensor):
mlp_output = self.linear_fc2(self.act_fn(self.linear_fc1(x)))
return mlp_output
@support_torch_compile(
dynamic_arg_dims={
"x": 0,
"cu_seqlens": 0,
"rotary_pos_emb_cos": 0,
"rotary_pos_emb_sin": 0,
"sequence_lengths": 0,
},
enable_if=should_torch_compile_mm_encoder,
is_encoder=True,
)
class Qwen3_VisionBlock(nn.Module):
def __init__(
self,
dim: int,
num_heads: int,
mlp_hidden_dim: int,
act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
norm_layer: Callable[[int], nn.Module] | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
if norm_layer is None:
norm_layer = partial(nn.LayerNorm, eps=1e-6)
self.norm1 = norm_layer(dim)
self.norm2 = norm_layer(dim)
self.attn = Qwen2_5_VisionAttention(
embed_dim=dim,
num_heads=num_heads,
projection_size=dim,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
self.mlp = Qwen3_VisionMLP(
dim,
mlp_hidden_dim,
act_fn=act_fn,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
def forward(
self,
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb_cos: torch.Tensor,
rotary_pos_emb_sin: torch.Tensor,
max_seqlen: torch.Tensor, # Only used for Flash Attention
sequence_lengths: torch.Tensor, # Only used for FlashInfer CuDNN backend
) -> torch.Tensor:
x = x + self.attn(
self.norm1(x),
cu_seqlens=cu_seqlens,
rotary_pos_emb_cos=rotary_pos_emb_cos,
rotary_pos_emb_sin=rotary_pos_emb_sin,
max_seqlen=max_seqlen,
sequence_lengths=sequence_lengths,
)
x = x + self.mlp(self.norm2(x))
return x
class Qwen3_VisionPatchMerger(nn.Module):
def __init__(
self,
d_model: int,
context_dim: int,
norm_layer: Callable[[int], nn.Module] | None = None,
spatial_merge_size: int = 2,
use_postshuffle_norm: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
use_data_parallel = is_vit_use_data_parallel()
self.hidden_size = context_dim * (spatial_merge_size**2)
self.use_postshuffle_norm = use_postshuffle_norm
if self.use_postshuffle_norm:
context_dim = self.hidden_size
if norm_layer is None:
norm_layer = partial(nn.LayerNorm, eps=1e-6)
self.norm = norm_layer(context_dim)
self.linear_fc1 = ColumnParallelLinear(
self.hidden_size,
self.hidden_size,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.linear_fc1",
disable_tp=use_data_parallel,
)
self.act_fn = nn.GELU()
self.linear_fc2 = RowParallelLinear(
self.hidden_size,
d_model,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.linear_fc2",
disable_tp=use_data_parallel,
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.use_postshuffle_norm:
x = self.norm(x.view(-1, self.hidden_size))
else:
x = self.norm(x).view(-1, self.hidden_size)
x_parallel, _ = self.linear_fc1(x)
x_parallel = self.act_fn(x_parallel)
out, _ = self.linear_fc2(x_parallel)
return out
class Qwen3_VisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
"attn.q.": ("attn.qkv.", "q"),
"attn.k.": ("attn.qkv.", "k"),
"attn.v.": ("attn.qkv.", "v"),
}
)
def __init__(
self,
vision_config: Qwen3VLVisionConfig,
norm_eps: float = 1e-6,
quant_config: QuantizationConfig | None = None,
input_norm: nn.Module | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = vision_config.hidden_size
self.num_heads = vision_config.num_heads
self.num_position_embeddings = vision_config.num_position_embeddings
self.patch_size = vision_config.patch_size
self.spatial_merge_size = vision_config.spatial_merge_size
self.spatial_merge_unit = self.spatial_merge_size**2
self.temporal_patch_size = vision_config.temporal_patch_size
self.deepstack_visual_indexes = (
vision_config.deepstack_visual_indexes
if hasattr(vision_config, "deepstack_visual_indexes")
else []
)
self.num_grid_per_side = int(self.num_position_embeddings**0.5)
self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = (
LRUCache(capacity=1024)
)
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
1
if use_data_parallel
else parallel_state.get_tensor_model_parallel_world_size()
)
# NOTE: This is used for creating empty tensor for all_gather for
# DP ViT. Here out_hidden_size is enlarged due to deepstack
self.out_hidden_size = vision_config.out_hidden_size * (
1 + len(self.deepstack_visual_indexes)
)
self.patch_embed = Qwen3_VisionPatchEmbed(
patch_size=self.patch_size,
temporal_patch_size=self.temporal_patch_size,
in_channels=vision_config.in_channels,
hidden_size=self.hidden_size,
)
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
head_dim = self.hidden_size // self.num_heads
# FP8 attention: Q/K/V become independent contiguous tensors
# after quantization, so cu_seqlens uses uniform stride (no 3x V).
self.fp8_padded_hidden_size = get_fp8_padded_hidden_size(
self.num_heads, head_dim
)
self.rotary_pos_emb = get_rope(
head_size=head_dim,
max_position=8192,
is_neox_style=True,
rope_parameters={"partial_rotary_factor": 0.5},
)
self.merger = Qwen3_VisionPatchMerger(
d_model=vision_config.out_hidden_size,
context_dim=self.hidden_size,
norm_layer=norm_layer,
spatial_merge_size=self.spatial_merge_size,
quant_config=quant_config,
prefix=f"{prefix}.merger",
)
self.deepstack_merger_list = nn.ModuleList(
[
Qwen3_VisionPatchMerger(
d_model=vision_config.out_hidden_size,
context_dim=self.hidden_size,
spatial_merge_size=self.spatial_merge_size,
use_postshuffle_norm=True,
norm_layer=norm_layer,
quant_config=quant_config,
prefix=f"{prefix}.deepstack_merger_list.{layer_idx}",
)
for layer_idx in range(len(self.deepstack_visual_indexes))
]
)
self.attn_backend = get_vit_attn_backend(
head_size=head_dim,
dtype=torch.get_default_dtype(),
)
self.blocks = nn.ModuleList(
[
Qwen3_VisionBlock(
dim=self.hidden_size,
num_heads=self.num_heads,
mlp_hidden_dim=vision_config.intermediate_size,
act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
norm_layer=norm_layer,
quant_config=quant_config,
prefix=f"{prefix}.blocks.{layer_idx}",
)
for layer_idx in range(vision_config.depth)
]
)
# An offloader moves tower parameters to host RAM after this
# constructor returns, and no weight is left on the accelerator to
# read the compute device from. Record it here while they are still
# placed, so `device` keeps working once every weight is streamed.
first_param = next(self.parameters(), None)
self._compute_device = (
torch.device("cpu") if first_param is None else first_param.device
)
@property
def dtype(self) -> torch.dtype:
return self.patch_embed.proj.weight.dtype
@property
def device(self) -> torch.device:
weight_device = self.patch_embed.proj.weight.device
if weight_device.type != "cpu":
return weight_device
return self._compute_device
def rot_pos_ids(self, h: int, w: int, spatial_merge_size: int) -> torch.Tensor:
cache_key = (h, w, spatial_merge_size)
if cache_key in self._rot_pos_ids_cache:
return self._rot_pos_ids_cache[cache_key]
hpos_ids = np.broadcast_to(np.arange(h).reshape(h, 1), (h, w))
h_div = h // spatial_merge_size
w_div = w // spatial_merge_size
hpos_ids = hpos_ids.reshape(
h_div,
spatial_merge_size,
w_div,
spatial_merge_size,
)
hpos_ids = hpos_ids.transpose(0, 2, 1, 3)
hpos_ids = hpos_ids.flatten()
wpos_ids = np.broadcast_to(np.arange(w).reshape(1, w), (h, w))
wpos_ids = wpos_ids.reshape(
h_div,
spatial_merge_size,
w_div,
spatial_merge_size,
)
wpos_ids = wpos_ids.transpose(0, 2, 1, 3)
wpos_ids = wpos_ids.flatten()
result = torch.from_numpy(np.stack([hpos_ids, wpos_ids], axis=-1))
self._rot_pos_ids_cache[cache_key] = result
return result
def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]):
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
pos_ids = [
self.rot_pos_ids(h, w, self.spatial_merge_size)
if t == 1
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
for t, h, w in grid_thw
]
num_pos = sum(p.shape[0] for p in pos_ids)
pinned = torch.empty(
(num_pos, pos_ids[0].shape[1]),
dtype=pos_ids[0].dtype,
pin_memory=PIN_MEMORY,
)
pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to(
self.device, non_blocking=True
)
# Use pre-computed cos_sin_cache from RotaryEmbedding
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
cos_combined = cos[pos_ids].flatten(1)
sin_combined = sin[pos_ids].flatten(1)
return cos_combined, sin_combined
def fast_pos_embed_interpolate(
self, grid_thw: Sequence[Sequence[int]]
) -> torch.Tensor:
interpolate_fn = (
triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native
)
outputs = []
for t, h, w in grid_thw:
outputs.append(
interpolate_fn(
self.pos_embed_table(),
t,
h,
w,
self.num_grid_per_side,
self.spatial_merge_size,
self.dtype,
)
)
return torch.cat(outputs, dim=0)
def pos_embed_table(self) -> torch.Tensor:
"""The position embedding table on the compute device. The whole tower
may be streamed from host RAM, but the interpolation kernel cannot read
host memory.
"""
table = self.pos_embed.weight
if table.device.type == "cpu":
table = table.to(self.device, non_blocking=True)
return table
def prepare_encoder_metadata(
self,
grid_thw_list: Sequence[Sequence[int]],
*,
max_batch_size: int | None = None,
max_frames_per_batch: int | None = None,
max_seqlen_override: int | None = None,
device: torch.device | None = None,
) -> dict[str, torch.Tensor | None]:
"""Compute encoder metadata from grid_thw_list.
Shared by the eager forward path, CUDA graph capture, and
CUDA graph replay to avoid duplicated implementation.
Args:
grid_thw_list: Grid configurations as list of [t, h, w].
max_batch_size: If set, pad cu_seqlens to this size
(needed for CUDA graph capture/replay).
max_frames_per_batch: If set, overrides max_batch_size for
cu_seqlens padding. For video inputs each item contributes
T attention sequences (frames); this sizes the buffer to
the total frame budget so video replays never overflow.
max_seqlen_override: If set, use this value for max_seqlen
instead of computing from cu_seqlens (needed for CUDA
graph capture to cover worst-case replay scenarios).
device: Device to place tensors on. Defaults to self.device.
"""
if device is None:
device = self.device
metadata: dict[str, torch.Tensor | None] = {}
# Positional embeddings
metadata["pos_embeds"] = self.fast_pos_embed_interpolate(grid_thw_list)
rotary_cos, rotary_sin = self.rot_pos_emb(grid_thw_list)
metadata["rotary_pos_emb_cos"] = rotary_cos
metadata["rotary_pos_emb_sin"] = rotary_sin
# cu_seqlens from grid_thw
grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
dtype=np.int32
)
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
# Pad cu_seqlens to the required number of sequences.
# For videos each item contributes T frames = T attention sequences,
# so the total can exceed max_batch_size. max_frames_per_batch
# overrides the pad target when set.
pad_to = (
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
)
if pad_to is not None:
num_seqs = len(cu_seqlens) - 1
if num_seqs < pad_to:
cu_seqlens = np.concatenate(
[
cu_seqlens,
np.full(
pad_to - num_seqs,
cu_seqlens[-1],
dtype=np.int32,
),
]
)
# sequence_lengths (backend-specific)
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
self.attn_backend, cu_seqlens, device
)
# max_seqlen
if max_seqlen_override is not None:
max_seqlen_val = max_seqlen_override
else:
max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
self.attn_backend, cu_seqlens
)
# Keep max_seqlen on CPU: attention wrappers call .item() on it,
# and having it on GPU would capture a wasteful D2H copy in CUDA
# graphs without changing behavior (the scalar is baked at capture).
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
# Recompute cu_seqlens (backend-specific transformation)
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
self.attn_backend,
cu_seqlens,
self.hidden_size,
self.tp_size,
device,
fp8_padded_hidden_size=self.fp8_padded_hidden_size,
)
return metadata
def forward(
self,
x: torch.Tensor,
grid_thw: torch.Tensor | list[list[int]],
*,
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
hidden_states = self.input_norm(
x.to(device=self.device, non_blocking=True), self.dtype
)
hidden_states = self.patch_embed(hidden_states)
if encoder_metadata is None:
if isinstance(grid_thw, list):
grid_thw_list = grid_thw
else:
grid_thw_list = grid_thw.tolist()
encoder_metadata = self.prepare_encoder_metadata(grid_thw_list)
pos_embeds = encoder_metadata["pos_embeds"]
hidden_states = hidden_states + pos_embeds
hidden_states = hidden_states.unsqueeze(1)
deepstack_feature_lists = []
for layer_num, blk in enumerate(self.blocks):
hidden_states = blk(
hidden_states,
cu_seqlens=encoder_metadata["cu_seqlens"],
rotary_pos_emb_cos=encoder_metadata["rotary_pos_emb_cos"],
rotary_pos_emb_sin=encoder_metadata["rotary_pos_emb_sin"],
max_seqlen=encoder_metadata["max_seqlen"],
sequence_lengths=encoder_metadata.get("sequence_lengths"),
)
if layer_num in self.deepstack_visual_indexes:
deepstack_merger_idx = self.deepstack_visual_indexes.index(layer_num)
deepstack_feature = self.deepstack_merger_list[deepstack_merger_idx](
hidden_states
)
deepstack_feature_lists.append(deepstack_feature)
hidden_states = self.merger(hidden_states)
hidden_states = torch.cat(
[hidden_states] + deepstack_feature_lists, dim=1
) # [seq_len, hidden_size * (1 + depth_of_deepstack)]
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):
# Timestamps are part of the prompt replacement (each frame group's
# "<X.X seconds>" text), so they size the placeholder range and must
# travel with the grid when embeddings are delivered out of band.
embedding_fields = {
**Qwen2VLMultiModalDataParser.embedding_fields,
"video": {
**Qwen2VLMultiModalDataParser.embedding_fields["video"],
"timestamps": "metadata",
},
}
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(Qwen3VLConfig)
def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
return self.ctx.get_hf_processor(
Qwen3VLProcessor,
**kwargs,
)
def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessor:
return self.get_hf_processor(**kwargs).image_processor
def get_video_processor(self, **kwargs: object) -> Qwen3VLVideoProcessor:
return self.get_hf_processor(**kwargs).video_processor
def get_data_parser(self):
return Qwen3VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
supported = super().get_supported_mm_processor_kwargs()
# vLLM has historically accepted `min_pixels` / `max_pixels` for
# Qwen3-VL video. Include them in the routing schema even though
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
# folded into `size` before calling HF.
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
return supported
def _get_vision_info(
self,
*,
image_width: int,
image_height: int,
num_frames: int = 2,
do_resize: bool = True,
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
mm_kwargs: Mapping[str, object],
modality: Literal["image", "video"] | None = None,
) -> tuple[ImageSize, int]:
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
hf_config = self.get_hf_config()
vision_config = hf_config.vision_config
patch_size = vision_config.patch_size
merge_size = vision_config.spatial_merge_size
temporal_patch_size = vision_config.temporal_patch_size
if modality is None:
modality = "video" if is_video else "image"
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
size = self._get_vision_size(
scoped_mm_kwargs,
default_size=image_processor.size,
)
if do_resize:
if is_video:
smart_resize = video_smart_resize
extra_kwargs = {
"num_frames": num_frames,
"temporal_factor": temporal_patch_size,
}
else:
smart_resize = image_smart_resize
extra_kwargs = {}
resized_height, resized_width = smart_resize(
height=image_height,
width=image_width,
factor=patch_size * merge_size,
min_pixels=size["shortest_edge"],
max_pixels=size["longest_edge"],
**extra_kwargs,
)
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
else:
preprocessed_size = ImageSize(width=image_width, height=image_height)
padded_num_frames = round_up(num_frames, temporal_patch_size)
grid_t = max(padded_num_frames // temporal_patch_size, 1)
grid_h = preprocessed_size.height // patch_size
grid_w = preprocessed_size.width // patch_size
num_patches = grid_t * grid_h * grid_w
num_vision_tokens = num_patches // (merge_size**2)
return preprocessed_size, num_vision_tokens
def _get_max_video_frames(self, max_tokens: int, start_num_frames: int = 2) -> int:
return super()._get_max_video_frames(
max_tokens, start_num_frames=start_num_frames
)
def get_num_frames_with_most_features(
self,
seq_len: int,
mm_counts: Mapping[str, int],
max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES,
) -> int:
return super().get_num_frames_with_most_features(
seq_len, mm_counts, max_frames_per_video=max_frames_per_video
)
def get_max_video_tokens(
self,
seq_len: int,
mm_counts: Mapping[str, int],
) -> int:
video_processor = self.get_video_processor()
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
# video_max_pixels contains the temporal compression factor,
# so we divide by 2 to get the maximum number of image pixels.
video_max_pixels = video_size["longest_edge"]
target_width, target_height = self.get_image_size_with_most_features(
max_pixels=video_max_pixels // temporal_patch_size
)
num_video_soft_tokens = self.get_num_video_tokens(
image_width=target_width,
image_height=target_height,
num_frames=2,
image_processor=video_processor,
mm_kwargs={},
)
return num_video_soft_tokens
def _calculate_timestamps(
self, indices: list[int] | torch.Tensor, video_fps: float, merge_size: int
):
if not isinstance(indices, list):
indices = indices.tolist()
if len(indices) % merge_size != 0:
# don't update metadata's frames_indices directly
indices = indices + [indices[-1]] * (merge_size - len(indices) % merge_size)
timestamps = [idx / video_fps for idx in indices]
timestamps = [
(timestamps[i] + timestamps[i + merge_size - 1]) / 2
for i in range(0, len(timestamps), merge_size)
]
return timestamps
def _get_video_second_idx(
self,
metadata: dict[str, Any],
do_sample_frames: bool | None = None,
sampled_fps: float | None = None,
sampled_num_frames: int | None = None,
) -> list[int]:
video_processor = self.get_video_processor()
temporal_patch_size = video_processor.temporal_patch_size
indices = metadata["frames_indices"]
# metadata["fps"] refers to the true fps of the input video.
video_fps = metadata["fps"]
if do_sample_frames is None:
do_sample_frames = metadata.get("do_sample_frames", False)
# If video frames are sampled in HF processor (instead of vLLM
# video loader), we need to re-calculate the indices from original
# metadata.
if do_sample_frames:
total_num_frames = metadata["total_num_frames"]
# When num_frames is explicitly provided, use it directly
# instead of computing from fps. This mirrors the behavior of
# HF's Qwen3VLVideoProcessor.sample_frames where num_frames
# and fps are mutually exclusive.
if sampled_num_frames is not None:
num_frames = sampled_num_frames
else:
# here video_fps is the fps of the sampled video, and
# metadata["fps"] refers to the fps of the original video.
sampled_fps = sampled_fps if sampled_fps else video_processor.fps
num_frames = int(total_num_frames / metadata["fps"] * sampled_fps)
num_frames = min(
min(
max(num_frames, video_processor.min_frames),
video_processor.max_frames,
),
total_num_frames,
)
indices = (
np.linspace(0, total_num_frames - 1, num_frames)
.round()
.astype(int)
.tolist()
)
timestamps = self._calculate_timestamps(indices, video_fps, temporal_patch_size)
return timestamps
class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
image_token = "<|vision_start|><|image_pad|><|vision_end|>"
video_token = "<|vision_start|><|video_pad|><|vision_end|>"
return image_token * num_images + video_token * num_videos
def get_dummy_mm_data(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
video_overrides = mm_options.get("video")
target_image_width, target_image_height = (
self.info.get_image_size_with_most_features()
)
# treat videos as special images
target_num_frames = 2
if video_overrides:
num_frames_override = video_overrides.num_frames
if num_frames_override:
if num_frames_override > target_num_frames:
logger.warning(
"video.num_frames override (%d) exceeds model's "
"maximum number of frames (%d), will be ignored",
num_frames_override,
target_num_frames,
)
if num_frames_override < 2:
logger.warning(
"video.num_frames override (%d) cannot be less "
"than 2, will be ignored",
num_frames_override,
)
target_num_frames = min(target_num_frames, num_frames_override)
target_num_frames = max(target_num_frames, 2)
video_processor = self.info.get_video_processor()
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
video_size = self.info._get_vision_size(
video_mm_kwargs,
default_size=video_processor.size,
)
temporal_patch_size = video_mm_kwargs.get(
"temporal_patch_size", video_processor.temporal_patch_size
)
# video_max_pixels contains the temporal compression factor,
# so we divide by 2 to get the maximum number of image pixels.
video_max_pixels = video_size["longest_edge"]
# With the HF processor's per-frame pixel cap enabled
# (cap_pixels_per_frame, huggingface/transformers#48071), a 2-frame
# dummy is processed at only 2 * cap pixels, so memory profiling
# underestimates the largest possible item: a fully sampled video
# still reaches the whole longest_edge budget. Spread the budget
# over enough frames that the cap is not binding for the dummy.
# This takes precedence over a shrinking num_frames override, which
# would otherwise reintroduce the under-profiling.
if video_mm_kwargs.get("cap_pixels_per_frame"):
max_video_tokens = video_mm_kwargs.get(
"max_video_tokens",
getattr(video_processor, "max_video_tokens", 768),
)
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
target_num_frames = max(
target_num_frames,
min(video_processor.max_frames, cdiv(video_max_pixels, per_frame_cap)),
)
target_video_width, target_video_height = (
self.info.get_image_size_with_most_features(
max_pixels=video_max_pixels // temporal_patch_size
)
)
target_video_size, _ = self.info._get_vision_info(
image_width=target_video_width,
image_height=target_video_height,
num_frames=target_num_frames,
image_processor=video_processor,
mm_kwargs={},
)
# NOTE: we need to do this check here since Qwen3-VL resizes video
# frames depending on how many frames there are.
target_video_width, target_video_height = (
target_video_size.width,
target_video_size.height,
)
if video_overrides:
width_override = video_overrides.width
if width_override:
if width_override > target_video_width:
logger.warning(
"video.width override (%d) exceeds model's "
"maximum width (%d), will be ignored",
width_override,
target_video_width,
)
target_video_width = min(target_video_width, width_override)
height_override = video_overrides.height
if height_override:
if height_override > target_video_height:
logger.warning(
"video.height override (%d) exceeds model's "
"maximum height (%d), will be ignored",
height_override,
target_video_height,
)
target_video_height = min(target_video_height, height_override)
return {
"image": self._get_dummy_images(
width=target_image_width,
height=target_image_height,
num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"),
),
"video": self._get_dummy_videos(
width=target_video_width,
height=target_video_height,
num_frames=target_num_frames,
num_videos=mm_counts.get("video", 0),
),
}
def _get_dummy_videos(
self,
*,
width: int,
height: int,
num_frames: int,
num_videos: int,
overrides: VideoDummyOptions | None = None,
) -> list[VideoItem]:
videos = super()._get_dummy_videos(
width=width,
height=height,
num_frames=num_frames,
num_videos=num_videos,
overrides=overrides,
)
videos = [v.copy() for v in videos]
video_items: list[VideoItem] = []
for video in videos:
video_num_frames = video.shape[0]
video_metadata = {
"fps": 2.0,
"duration": video_num_frames / 2.0,
"total_num_frames": video_num_frames,
"frames_indices": list(range(video_num_frames)),
"video_backend": "opencv",
"do_sample_frames": False,
}
video_items.append((video, video_metadata))
return video_items
def _replace_video_token_placeholders(
prompt_ids: list[int],
target: list[int],
replacements: list[list[int]],
) -> list[int]:
"""Replace each video placeholder with its expanded sequence.
Args:
prompt_ids: Token IDs of the original (unexpanded) prompt.
target: Token-ID sequence to search for. New-style
(transformers>=5.10) processors expand only the bare
`[video_pad_id]`; older ones expand the full
`[vision_start_id, video_pad_id, vision_end_id]` triplet.
replacements: Per-video expanded token sequences, in prompt order.
Returns:
Token IDs with every placeholder replaced.
"""
result: list[int] = []
repl_idx = 0
i = 0
n = len(prompt_ids)
k = len(target)
num_repl = len(replacements)
while i < n:
if prompt_ids[i : i + k] == target:
result.extend(replacements[repl_idx])
repl_idx += 1
i += k
else:
result.append(prompt_ids[i])
i += 1
assert repl_idx == num_repl, (
f"Found {repl_idx} video placeholders but expected {num_repl}"
)
return result
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
def _get_per_video_kwargs(
self, kwargs: Mapping[str, object], num_videos: int | None = None
) -> dict[str, list[Any]]:
merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs)
per_video_kwargs = {
key: value
for key, value in merged.get("videos_kwargs", {}).items()
if key in ("fps", "num_frames") and isinstance(value, list)
}
if num_videos:
for key, values in per_video_kwargs.items():
if len(values) < num_videos:
raise ValueError(
f"{key} must have at least {num_videos} entries "
"(one per video); "
f"got {len(values)}."
)
return per_video_kwargs
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
hashes = dict(super()._get_mm_hashes(inputs))
per_video_kwargs = self._get_per_video_kwargs(
inputs.hf_processor_mm_kwargs, len(hashes.get("video", []))
)
if per_video_kwargs and "video" in hashes:
algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm
hashes["video"] = [
MultiModalHasher.hash_kwargs(
algorithm,
mm_hash=mm_hash,
**{key: values[i] for key, values in per_video_kwargs.items()},
)
for i, mm_hash in enumerate(hashes["video"])
]
return hashes
def _get_cache_missing_processor_kwargs(
self,
inputs: ProcessorInputs,
mm_is_cached: Mapping[str, list[bool]],
) -> Mapping[str, object]:
per_video_kwargs = self._get_per_video_kwargs(
inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", []))
)
if not per_video_kwargs or "video" not in mm_is_cached:
return inputs.hf_processor_mm_kwargs
kwargs = self.info._merge_and_resolve_mm_processor_kwargs(
inputs.hf_processor_mm_kwargs
)
kwargs["videos_kwargs"].update(
{
key: [
value
for value, hit in zip(values, mm_is_cached["video"])
if not hit
]
for key, values in per_video_kwargs.items()
}
)
return kwargs
@staticmethod
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
"""Transformers>=5.10 processors override `replace_video_token`
to expand only the bare video token, keeping the prompt's outer
`<|vision_start|>`/`<|vision_end|>` markers."""
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
return proc_impl is not None and proc_impl is not mixin_impl
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_kwargs
)
if not hf_data:
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
# Merge and resolve the multimodal processor kwargs once so every HF
# call below starts from the same engine/request state and scoped form.
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
if "images_kwargs" in merged_mm_kwargs:
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_image_size = self.info.get_image_processor().size
image_scoped_kwargs["size"] = self.info._get_vision_size(
image_scoped_kwargs,
default_size=default_image_size,
)
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
if "videos_kwargs" in merged_mm_kwargs:
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
# override is passed, fill missing edges from the processor defaults
# before any HF call.
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
default_video_size = self.info.get_video_processor().size
video_scoped_kwargs["size"] = self.info._get_vision_size(
video_scoped_kwargs,
default_size=default_video_size,
)
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
# compatibility kwargs. They have already been folded into `size`,
# so remove them before any HF call, including calls that contain
# only text/images.
video_scoped_kwargs.pop("min_pixels", None)
video_scoped_kwargs.pop("max_pixels", None)
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
if videos := hf_data.pop("videos", []):
assert isinstance(videos, list)
self._get_per_video_kwargs(hf_kwargs, len(videos))
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
hf_config = self.info.get_hf_config()
tokenizer = self.info.get_tokenizer()
merge_size = hf_config.vision_config.spatial_merge_size
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
vision_start_token_id = hf_config.vision_start_token_id
vision_end_token_id = hf_config.vision_end_token_id
video_token_id = hf_config.video_token_id
for item_idx, item in enumerate(videos):
video_array, metadata = item
# NOTE: @JJJYmmm new attr metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is
# used to calculate the timestamps. Make sure that
# do_sample_frames in mm_kwargs is false for presampled videos.
# The root copy is shallow, so give each item its own video
# scope before modifying per-video values below.
video_mm_kwargs = dict(merged_mm_kwargs)
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
sampled_fps = video_scoped_kwargs.get("fps")
if is_list_of(sampled_fps, float):
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
sampled_num_frames = video_scoped_kwargs.get("num_frames")
if is_list_of(sampled_num_frames, int):
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
if "do_sample_frames" not in video_scoped_kwargs:
# qwen_vl_utils already has "do_sample_frames" in
# mm_kwargs, don't overwrite it.
video_scoped_kwargs["do_sample_frames"] = metadata.get(
"do_sample_frames", False
)
metadata = VideoMetadata(
**{k: metadata[k] for k in metadata if k != "do_sample_frames"}
)
# Compute timestamps here where we have access to metadata
timestamps = self.info._get_video_second_idx(
metadata=metadata,
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
sampled_fps=video_scoped_kwargs.get("fps"),
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
)
timestamps_per_video.append(timestamps)
video_mm_data = dict()
video_mm_data["videos"] = [[video_array]]
video_mm_data["video_metadata"] = [[metadata]]
# When num_frames is specified, explicitly set fps=None
# to prevent HF's BaseVideoProcessor.preprocess() from
# filling in the class default (fps=2) via setdefault(),
# which would conflict with num_frames (mutually exclusive).
if (
"num_frames" in video_scoped_kwargs
and "fps" not in video_scoped_kwargs
):
video_scoped_kwargs["fps"] = None
video_outputs = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**video_mm_kwargs),
dict(
text="<|vision_start|><|video_pad|><|vision_end|>",
**video_mm_data,
),
video_mm_kwargs,
)
# Discard HF output input_ids — we use get_video_repl below
# to generate the correct (EVS-adjusted) token sequence.
video_outputs.pop("input_ids", None)
video_grid_thw = video_outputs["video_grid_thw"]
num_frames = int(video_grid_thw[0, 0])
tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // (
merge_size**2
)
# Apply video pruning (EVS or VidCom2) if enabled.
if pruning_spec is not None:
method, prune_q = pruning_spec
count_fn = (
vidcom2_compute_retained_tokens_count
if method == "vidcom2"
else compute_retained_tokens_count
)
num_tokens = count_fn(
tokens_per_frame=tokens_per_frame_base,
num_frames=num_frames,
q=prune_q,
)
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
select_token_id = False
else:
tokens_per_frame = [tokens_per_frame_base] * num_frames
select_token_id = True
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
tokens_per_frame=tokens_per_frame,
timestamps=timestamps,
tokenizer=tokenizer,
vision_start_token_id=vision_start_token_id,
vision_end_token_id=vision_end_token_id,
video_token_id=video_token_id,
select_token_id=select_token_id,
)
video_input_ids_lst.append(list(video_repl.full))
video_grid_thw_lst.append(video_outputs["video_grid_thw"])
pixel_values_videos_lst.append(video_outputs["pixel_values_videos"])
video_outputs = dict(
pixel_values_videos=torch.cat(pixel_values_videos_lst),
video_grid_thw=torch.cat(video_grid_thw_lst),
timestamps=timestamps_per_video,
)
else:
video_outputs = dict()
# The root copy is shallow, so copy the video scope before removing
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
non_video_mm_kwargs = dict(merged_mm_kwargs)
if "videos_kwargs" in non_video_mm_kwargs:
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
non_video_scoped_kwargs.pop("fps", None)
non_video_scoped_kwargs.pop("num_frames", None)
# The context merges model defaults again when calling HF.
for key in self._get_per_video_kwargs({}):
non_video_scoped_kwargs[key] = None
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**non_video_mm_kwargs),
hf_data,
non_video_mm_kwargs,
)
# Replace each placeholder with pre-computed video tokens.
if video_input_ids_lst:
hf_config = self.info.get_hf_config()
if self._expands_only_video_token(self.info.get_hf_processor()):
video_target = [hf_config.video_token_id]
else:
video_target = [
hf_config.vision_start_token_id,
hf_config.video_token_id,
hf_config.vision_end_token_id,
]
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
expanded_ids = _replace_video_token_placeholders(
prompt_ids, video_target, video_input_ids_lst
)
processed_data["input_ids"] = [expanded_ids]
processed_data.update(video_outputs)
return self._finalize_hf_mm_data(
hf_data, hf_kwargs, passthrough_data, processed_data
)
def _get_mm_fields_config(
self,
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return _create_qwen2vl_field_factory(
self.info.get_hf_config().vision_config.spatial_merge_size
)(hf_inputs)
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, Any],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
hf_config = self.info.get_hf_config()
video_token_id = hf_config.video_token_id
vision_start_token_id = hf_config.vision_start_token_id
vision_end_token_id = hf_config.vision_end_token_id
merge_length = image_processor.merge_size**2
def get_image_replacement_qwen3vl(item_idx: int):
out_item = out_mm_kwargs["image"][item_idx]
grid_thw = out_item["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
num_tokens = int(grid_thw.prod()) // merge_length
return [hf_processor.image_token_id] * num_tokens
def get_video_replacement_qwen3vl(item_idx: int):
out_item = out_mm_kwargs["video"][item_idx]
grid_thw = out_item["video_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
sampled_fps = hf_processor_mm_kwargs.get("fps")
if is_list_of(sampled_fps, float):
sampled_fps = sampled_fps[item_idx]
timestamps = out_item["timestamps"].data
assert isinstance(timestamps, (list, torch.Tensor))
assert len(timestamps) == grid_thw[0], (
f"The timestamps length({len(timestamps)}) should be equal "
f"video length ({grid_thw[0]})."
)
# Compute tokens per frame, with EVS / VidCom2 support
num_frames = int(grid_thw[0])
tokens_per_frame_base = int(grid_thw[1:].prod()) // merge_length
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
if pruning_spec is not None:
method, prune_q = pruning_spec
count_fn = (
vidcom2_compute_retained_tokens_count
if method == "vidcom2"
else compute_retained_tokens_count
)
num_tokens = count_fn(
tokens_per_frame=tokens_per_frame_base,
num_frames=num_frames,
q=prune_q,
)
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
select_token_id = False
else:
tokens_per_frame = [tokens_per_frame_base] * num_frames
select_token_id = True
return Qwen3VLMultiModalProcessor.get_video_repl(
tokens_per_frame=tokens_per_frame,
timestamps=timestamps,
tokenizer=tokenizer,
vision_start_token_id=vision_start_token_id,
vision_end_token_id=vision_end_token_id,
video_token_id=video_token_id,
select_token_id=select_token_id,
)
if self._expands_only_video_token(hf_processor):
# transformers>=5.10 expands only the bare video_token
video_target = [video_token_id]
else:
# Old-style processors expand the full placeholder
video_target = [
vision_start_token_id,
video_token_id,
vision_end_token_id,
]
return [
PromptReplacement(
modality="image",
target=[hf_processor.image_token_id],
replacement=get_image_replacement_qwen3vl,
),
PromptReplacement(
modality="video",
target=video_target,
replacement=get_video_replacement_qwen3vl,
),
]
@staticmethod
def get_video_repl(
*,
tokens_per_frame: list[int],
timestamps: Sequence[float] | torch.Tensor,
tokenizer: TokenizerLike,
vision_start_token_id: int,
vision_end_token_id: int,
video_token_id: int,
select_token_id: bool = False,
) -> PromptUpdateDetails:
"""Build prompt replacement for a video in Qwen3VL format.
The replacement structure for each frame is:
timestamp_tokens + vision_start_token + video_tokens + vision_end_token
Args:
tokens_per_frame: Number of video tokens per frame (can vary per frame for
EVS).
timestamps: List of timestamps in seconds for each frame
tokenizer: Tokenizer to encode timestamp strings
vision_start_token_id: Token ID for vision start marker
vision_end_token_id: Token ID for vision end marker
video_token_id: Token ID for video content
select_token_id: Whether to return token ids instead of text
Returns:
PromptUpdateDetails with full token sequence
"""
assert len(timestamps) == len(tokens_per_frame), (
"timestamps and tokens_per_frame must have the same length"
)
# Tokenize timestamp strings independently to avoid tokenizer merging
# tokens across boundaries.
timestamp_token_ids = [
tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
for timestamp in timestamps
]
# Build the full token sequence
all_token_ids = []
for frame_timestamp_ids, num_tokens in zip(
timestamp_token_ids, tokens_per_frame
):
# Add timestamp tokens
all_token_ids.extend(frame_timestamp_ids)
# Add vision tokens: vision_start + video_tokens + vision_end
all_token_ids.append(vision_start_token_id)
all_token_ids.extend([video_token_id] * num_tokens)
all_token_ids.append(vision_end_token_id)
if select_token_id:
return PromptUpdateDetails.select_token_id(all_token_ids, video_token_id)
# NOTE: we use `from_seq` instead of `select_token_id` because we want all
# tokens in the placeholder to be initially marked as candidates. Then
# in `get_input_embeddings``, we refine the mask to only replace
# `video_token_id` / `image_token_id`` positions with video/image embeddings,
# keeping text embeddings for timestamps and structural tokens.
return PromptUpdateDetails.from_seq(all_token_ids)
def mrope_positions_factor(vllm_config: VllmConfig) -> int:
# Video pruning keeps the positions of the tokens it drops. A drafter gets
# the target's positions, so it follows the target.
spec = vllm_config.speculative_config
model_config = (spec and spec.target_model_config) or vllm_config.model_config
mm_config = model_config.multimodal_config
pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled()
return 4 if pruned else 1
class Qwen3VLDecoderLayer(Qwen3DecoderLayer):
def __init__(self, **kwargs) -> None:
factor = mrope_positions_factor(get_current_vllm_config())
super().__init__(**kwargs, mrope_positions_factor=factor)
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
# positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
# otherwise (seq_len, ).
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
# the same shape as input_embeds
"deepstack_input_embeds": 0,
}
)
class Qwen3LLMModel(Qwen3Model):
decoder_layer_type = Qwen3VLDecoderLayer
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
# args for deepstack
deepstack_input_embeds: IntermediateTensors | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
for layer_idx, layer in islice(
enumerate(self.layers), self.start_layer, self.end_layer
):
hidden_states, residual = layer(
positions,
hidden_states,
residual,
)
if deepstack_input_embeds is not None and layer_idx in range(
0, len(deepstack_input_embeds)
):
hidden_states = (
hidden_states
+ deepstack_input_embeds[f"deepstack_input_embeds_{layer_idx}"]
)
self._maybe_add_hidden_state(
aux_hidden_states, layer_idx + 1, hidden_states, residual
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states, residual)
if len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
class Qwen3LLMForCausalLM(Qwen3ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super(Qwen3ForCausalLM, self).__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.model = Qwen3LLMModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix="lm_head",
)
if config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
@MULTIMODAL_REGISTRY.register_processor(
Qwen3VLMultiModalProcessor,
info=Qwen3VLProcessingInfo,
dummy_inputs=Qwen3VLDummyInputsBuilder,
)
class Qwen3VLForConditionalGeneration(
nn.Module,
SupportsMultiModal,
SupportsEncoderCudaGraph,
SupportsLoRA,
SupportsPP,
SupportsMRoPE,
SupportsEagle,
SupportsEagle3,
SupportsMultiModalPruning,
):
packed_modules_mapping: dict[str, list[str]] = {
"qkv_proj": [
"q_proj",
"k_proj",
"v_proj",
],
"gate_up_proj": [
"gate_proj",
"up_proj",
],
"qkv": ["qkv"], # For vision tower's already-packed QKV
}
supports_encoder_tp_data = True
supports_tower_connector_lora = True
supports_mm_device_do_normalize = True
supported_video_pruning_methods = ("evs", "vidcom2")
# To ensure correct weight loading and mapping.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"model.visual.": "visual.",
"lm_head.": "language_model.lm_head.",
"model.language_model.": "language_model.model.",
}
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|vision_start|><|image_pad|><|vision_end|>"
if modality.startswith("video"):
return "<|vision_start|><|video_pad|><|vision_end|>"
raise ValueError("Only image or video modality is supported")
def _init_video_pruning(self, multimodal_config: MultiModalConfig) -> None:
pruning_spec = multimodal_config.get_video_pruning_spec()
if pruning_spec is None:
self.video_pruning_method: VideoPruningMethod | None = None
self.video_pruning_rate = multimodal_config.video_pruning_rate
else:
self.video_pruning_method, self.video_pruning_rate = pruning_spec
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
super().__init__()
config: Qwen3VLConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config
self.model_config = vllm_config.model_config
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
self.multimodal_config = multimodal_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self._init_video_pruning(multimodal_config)
with self._mark_tower_model(vllm_config, {"image", "video"}):
self.visual = Qwen3_VisionTransformer(
config.vision_config,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
quant_config=quant_config,
input_norm=build_mm_input_norm(self.model_config),
prefix=maybe_prefix(prefix, "visual"),
)
self.use_deepstack = hasattr(
config.vision_config, "deepstack_visual_indexes"
) and not isinstance(self.visual, StageMissingLayer)
self.deepstack_num_level = (
len(config.vision_config.deepstack_visual_indexes)
if self.use_deepstack
else 0
)
self.visual_dim = config.vision_config.out_hidden_size
self.multiscale_dim = self.visual_dim * self.deepstack_num_level
if self.use_deepstack:
self.deepstack_input_embeds = [
torch.zeros(
vllm_config.scheduler_config.max_num_batched_tokens,
config.text_config.hidden_size,
)
for _ in range(self.deepstack_num_level)
]
# Tracks the valid token span currently stored in the buffer.
# Zero means there is no active deepstack payload to consume.
self.deepstack_input_embeds_num_tokens = 0
with self._mark_language_model(vllm_config):
self.language_model = Qwen3LLMForCausalLM(
vllm_config=vllm_config.with_hf_config(
config.text_config,
architectures=["Qwen3ForCausalLM"],
),
prefix=maybe_prefix(prefix, "language_model"),
)
if not get_pp_group().is_first_rank and hasattr(
config.vision_config, "deepstack_visual_indexes"
):
assert self.language_model.model.start_layer >= len(
config.vision_config.deepstack_visual_indexes
), (
"start_layer should be greater than or equal to "
"len(deepstack_visual_indexes)"
)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
def _get_deepstack_input_embeds(
self,
num_tokens: int,
) -> IntermediateTensors | None:
if not getattr(self, "deepstack_input_embeds", None):
return None # If vision tower is skipped
if num_tokens > self.deepstack_input_embeds[0].size(0):
self._resize_deepstack_input_embeds(num_tokens)
# get deepstack_input_embeds from buffer, and clear the buffer
return IntermediateTensors(
{
f"deepstack_input_embeds_{idx}": self.deepstack_input_embeds[idx][
:num_tokens
]
for idx in range(self.deepstack_num_level)
}
)
def _resize_deepstack_input_embeds(self, num_tokens: int) -> None:
self.deepstack_input_embeds = [
torch.zeros(
num_tokens,
self.config.text_config.hidden_size,
device=self.deepstack_input_embeds[0].device,
dtype=self.deepstack_input_embeds[0].dtype,
)
for _ in range(self.deepstack_num_level)
]
def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None:
if not getattr(self, "deepstack_input_embeds", None):
return
# set deepstack_input_embeds to buffer
num_tokens = deepstack_input_embeds.size(1)
if num_tokens > self.deepstack_input_embeds[0].size(0):
self._resize_deepstack_input_embeds(num_tokens)
for idx in range(self.deepstack_num_level):
self.deepstack_input_embeds[idx][:num_tokens].copy_(
deepstack_input_embeds[idx]
)
self.deepstack_input_embeds_num_tokens = num_tokens
def _clear_deepstack_input_embeds(self, num_tokens: int) -> None:
if not getattr(self, "deepstack_input_embeds", None):
return
if getattr(self, "deepstack_input_embeds_num_tokens", 0) == 0:
return
# clear deepstack_input_embeds in buffer
if num_tokens > 0:
for idx in range(self.deepstack_num_level):
self.deepstack_input_embeds[idx][:num_tokens].zero_()
self.deepstack_input_embeds_num_tokens = 0
# -- SupportsEncoderCudaGraph protocol methods --
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphConfig,
)
# When video pruning is enabled, embed_multimodal post-processes both
# image and video embeddings (mrope positions are appended for image,
# prune+append for video). The encoder CUDA graph path bypasses that
# post-process, producing inconsistent embedding formats vs eager. So
# disable CUDA graph for all modalities when pruning is on.
modalities = [] if self.is_multimodal_pruning_enabled else ["image", "video"]
# Compute max_frames_per_video for budget sizing.
max_frames = self.get_max_frames_per_video() if "video" in modalities else 1
return EncoderCudaGraphConfig(
modalities=modalities,
buffer_keys=[
"pixel_values",
"pos_embeds",
"rotary_pos_emb_cos",
"rotary_pos_emb_sin",
"cu_seqlens",
"max_seqlen",
"sequence_lengths",
],
out_hidden_size=self.visual.out_hidden_size,
max_frames_per_video=max_frames,
)
def get_input_modality(
self,
mm_kwargs: dict[str, Any],
) -> str:
if "image_grid_thw" in mm_kwargs:
return "image"
elif "video_grid_thw" in mm_kwargs:
return "video"
raise AssertionError("This line should be unreachable.")
def get_max_frames_per_video(self) -> int:
mm_registry = MULTIMODAL_REGISTRY
info = mm_registry.get_processing_info(self.model_config)
assert isinstance(info, Qwen3VLProcessingInfo)
max_frames_per_video = info.get_num_frames_with_most_features(
seq_len=self.model_config.max_model_len,
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
)
return max_frames_per_video
def get_encoder_cudagraph_budget_range(
self,
vllm_config,
) -> tuple[int, int]:
# Min: estimated smallest possible encoder input.
# 224x224 image → 16x16 patches (patch_size=14)
# spatial_merge_size=2 → 8x8 = 64 tokens
min_budget = 64
# Max: capped by max_num_batched_tokens
# TODO(shen-shanshan): the max_budget auto-infer needs to be optimized later.
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
self.model_config.max_model_len,
)
return (min_budget, max_budget)
def _get_pixel_values_by_modality(
self,
mm_kwargs: dict[str, Any],
) -> torch.Tensor:
modality = self.get_input_modality(mm_kwargs)
if modality == "image":
return mm_kwargs["pixel_values"]
elif modality == "video":
return mm_kwargs["pixel_values_videos"]
raise AssertionError("This line should be unreachable.")
def _get_grid_thw_by_modality(
self,
mm_kwargs: dict[str, Any],
) -> list[tuple[int, int, int]]:
grid_thw_key = f"{self.get_input_modality(mm_kwargs)}_grid_thw"
grid_thw = mm_kwargs[grid_thw_key]
if not isinstance(grid_thw, list):
grid_thw = grid_thw.tolist()
return grid_thw
def get_encoder_cudagraph_item_specs(
self,
mm_kwargs: dict[str, Any],
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
m = self.visual.spatial_merge_size
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return [
EncoderItemSpec(
input_size=t * h * w,
output_tokens=t * (h // m) * (w // m),
)
for t, h, w in grid_thw
]
def select_encoder_cudagraph_items(
self,
mm_kwargs: dict[str, Any],
indices: list[int],
) -> dict[str, Any]:
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
if len(indices) == 0:
if self.get_input_modality(mm_kwargs) == "image":
return {
"pixel_values": pixel_values[:0],
"image_grid_thw": [],
}
else:
return {
"pixel_values_videos": pixel_values[:0],
"video_grid_thw": [],
}
# Compute cumulative patch offsets for slicing pixel_values
patches_per_item = [t * h * w for t, h, w in grid_thw]
cum_patches = [0]
for p in patches_per_item:
cum_patches.append(cum_patches[-1] + p)
selected_pv = torch.cat(
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
)
selected_grid = [grid_thw[i] for i in indices]
if self.get_input_modality(mm_kwargs) == "image":
return {
"pixel_values": selected_pv,
"image_grid_thw": selected_grid,
}
else:
return {
"pixel_values_videos": selected_pv,
"video_grid_thw": selected_grid,
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
):
from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderCudaGraphCaptureInputs,
)
spatial_merge_size = self.visual.spatial_merge_size
# Ceil so the buffer fits the worst case of one item using the full
# budget. Floor under-allocates when budget is not a multiple of
# max_batch_size.
per_mm_item_output = (token_budget + max_batch_size - 1) // max_batch_size
frames_per_item = max_frames_per_batch // max_batch_size
if frames_per_item > 1:
# Build the capture grid using a video-format layout so that
# cu_seqlens is sized for video replays from the start.
# cu_seqlens has one entry per attention sequence (one per frame),
# so using T > 1 per item makes the buffer large enough without
# relying solely on padding.
# Ceiling ensures frames_per_item * tokens_per_frame >= per_mm_item_output
# so the pixel_values buffer covers any valid single-item replay.
tokens_per_frame = (
per_mm_item_output + frames_per_item - 1
) // frames_per_item
# Video-format grid_config (T=frames_per_item).
grid_config = [
[
frames_per_item,
spatial_merge_size,
tokens_per_frame * spatial_merge_size,
]
for _ in range(max_batch_size)
]
else:
# Image-format grid_config (T=1).
grid_config = [
[1, spatial_merge_size, per_mm_item_output * spatial_merge_size]
for _ in range(max_batch_size)
]
# Create dummy pixel_values
patch_embed = self.visual.patch_embed
in_channels = patch_embed.proj.in_channels
patch_size = patch_embed.patch_size
temporal_patch_size = patch_embed.temporal_patch_size
total_patches = sum(t * h * w for t, h, w in grid_config)
flattened_patch_size = (
in_channels * temporal_patch_size * patch_size * patch_size
)
dummy_pixel_values = torch.zeros(
total_patches,
flattened_patch_size,
device=device,
dtype=self.visual.input_norm.input_dtype or dtype,
)
# Override max_seqlen with a safe upper bound for capture.
# max_seqlen.item() gets baked into the CUDA graph (not replayed),
# so the capture value must cover any replay scenario.
# Worst case: 1 item consuming the full budget ->
# seq_len = token_budget * spatial_merge_size^2.
metadata = self.visual.prepare_encoder_metadata(
grid_config,
max_batch_size=max_batch_size,
max_frames_per_batch=max_frames_per_batch,
max_seqlen_override=token_budget * (spatial_merge_size**2),
device=device,
)
# Just use image-modality dummy input_buffer for capturing, since it's also
# compatible for video inputs (has the same shape: [num_patches, C*T*P*P]).
values = metadata | {
"pixel_values": dummy_pixel_values,
}
return EncoderCudaGraphCaptureInputs(
values=values,
)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
):
modality = self.get_input_modality(mm_kwargs)
grid_thw_list = self._get_grid_thw_by_modality(mm_kwargs)
if modality == "image":
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_batch_size=max_batch_size,
)
elif modality == "video":
metadata = self.visual.prepare_encoder_metadata(
grid_thw_list,
max_frames_per_batch=max_frames_per_batch,
)
else:
raise AssertionError("This line should be unreachable.")
values = metadata | {
"pixel_values": self._get_pixel_values_by_modality(mm_kwargs),
}
return EncoderCudaGraphReplayBuffers(values=values)
def encoder_cudagraph_forward(
self,
values: dict[str, torch.Tensor],
path: str = "default",
) -> torch.Tensor:
pixel_values = values.pop("pixel_values")
metadata = values
return self.visual(pixel_values, None, encoder_metadata=metadata)
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
return self.visual(pixel_values, grid_thw)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Qwen2_5_VLImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
if pixel_values is None and image_embeds is None:
return None
if pixel_values is not None:
return Qwen2_5_VLImagePixelInputs(
type="pixel_values",
pixel_values=pixel_values,
image_grid_thw=image_grid_thw,
)
else:
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Qwen2_5_VLVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
timestamps = kwargs.pop("timestamps", None)
if pixel_values_videos is None and video_embeds is None:
return None
if pixel_values_videos is not None:
return Qwen2_5_VLVideoPixelInputs(
type="pixel_values_videos",
pixel_values_videos=pixel_values_videos,
video_grid_thw=video_grid_thw,
second_per_grid_ts=second_per_grid_ts,
timestamps=timestamps,
)
else:
return Qwen2_5_VLVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
timestamps=timestamps,
)
def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs
) -> tuple[torch.Tensor, ...]:
grid_thw = image_input["image_grid_thw"]
assert grid_thw.ndim == 2
if image_input["type"] == "image_embeds":
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
else:
pixel_values = image_input["pixel_values"]
if self.use_data_parallel:
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
)
else:
image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
# Split concatenated embeddings for each image item.
merge_size = self.visual.spatial_merge_size
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return image_embeds.split(sizes)
def _process_video_input(
self, video_input: Qwen2_5_VLVideoInputs
) -> tuple[torch.Tensor, ...]:
grid_thw = video_input["video_grid_thw"]
assert grid_thw.ndim == 2
if video_input["type"] == "video_embeds":
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
else:
pixel_values_videos = video_input["pixel_values_videos"]
if self.use_data_parallel:
grid_thw_list = grid_thw.tolist()
return run_dp_sharded_mrope_vision_model(
self.visual, pixel_values_videos, grid_thw_list, rope_type="rope_3d"
)
else:
video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
# Split concatenated embeddings for each video item.
merge_size = self.visual.spatial_merge_size
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return video_embeds.split(sizes)
def _postprocess_image_embeds_evs(
self,
image_embeds_split: tuple[torch.Tensor, ...],
image_input: Qwen2_5_VLImageInputs,
) -> tuple[torch.Tensor, ...]:
"""Append mrope positions for each for images.
This is necessary to recover correct mrope
positions after video pruning
Args:
image_embeds_split: Tuple of image embeddings for
each image item.
image_input: Image input data.
Returns:
Tuple of image embeddings for each image item.
Resulting embeddings will have extra 5 channels for
computed mrope positions, consistent with video embeddings.
"""
if self.is_multimodal_pruning_enabled:
merge_size = self.visual.spatial_merge_size
grid_thw = image_input["image_grid_thw"]
grid_thw_list = grid_thw.tolist()
image_embeds_out = []
for emb, size in zip(image_embeds_split, grid_thw_list):
positions = async_tensor_h2d(
compute_mrope_for_media(size, merge_size), emb.device
)
positions = torch.cat(
[
positions,
torch.zeros_like(
positions[:, 0:1]
), # Dummy extra fifth channel
],
dim=1,
)
emb = torch.cat([emb, positions], dim=1)
image_embeds_out.append(emb)
image_embeds_split = tuple(image_embeds_out)
return image_embeds_split
def _postprocess_video_embeds_evs(
self,
video_embeds_split: tuple[torch.Tensor, ...],
video_input: Qwen2_5_VLVideoInputs,
) -> tuple[torch.Tensor, ...]:
"""Prunes video embeddings via Efficient Video Sampling (EVS)
and then appends mrope positions for each retained embeddings
Args:
video_embeds_split: Tuple of video embeddings for each video item.
video_input: Video input data.
Returns:
Tuple of video embeddings for each video item.
Resulting embeddings will have extra 5 channels for computed mrope
positions, and whether the index corresponds to a video embedding.
"""
grid_thw = video_input["video_grid_thw"]
assert grid_thw.ndim == 2
grid_thw_list = grid_thw.tolist()
merge_size = self.visual.spatial_merge_size
# Apply EVS to each video.
video_embeds_out = []
assert video_input.timestamps is not None
for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)):
# Compute positions.
timestamps = video_input.timestamps[video_idx]
num_frames = len(timestamps)
t, h, w = size
if self.is_multimodal_pruning_enabled:
assert self.video_pruning_rate is not None
# Compute the retention mask for each video (EVS or VidCom2).
if self.video_pruning_method == "vidcom2":
mask_fn = vidcom2_compute_retention_mask
else:
mask_fn = compute_retention_mask
retention_mask = mask_fn(
emb,
size,
spatial_merge_size=self.visual.spatial_merge_size,
q=self.video_pruning_rate,
)
with gpu_sync_allowed():
# Apply retention mask.
emb = emb[retention_mask]
# Calculate the actual number of retained tokens per frame.
num_frames, rows, cols = t, h // merge_size, w // merge_size
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
num_tokens_per_frame = (
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
)
else:
feature_size = emb.shape[0] // num_frames
num_tokens_per_frame = [feature_size] * num_frames
retention_mask = None
emb = self._create_final_video_embeddings(
video_embeddings=emb,
num_tokens_per_frame=num_tokens_per_frame,
timestamps=timestamps,
video_grid_thw=size,
retention_mask=retention_mask,
)
video_embeds_out.append(emb)
return tuple(video_embeds_out)
def _create_final_video_embeddings(
self,
video_embeddings: torch.Tensor,
num_tokens_per_frame: list[int],
timestamps: list[float],
video_grid_thw: list[int],
retention_mask: torch.Tensor,
) -> torch.Tensor:
"""Create final embeddings that combine video embeddings with
text embeddings of indicator tokens.
These final embeddings contain:
- Actual video embeddings in positions corresponding to video content
- Text embeddings for indicator tokens (<img>, </img>, and
frame separation text) in their respective positions
These embeddings will replace the placeholder embeddings to create
input_embeds for the LLM.
"""
device = video_embeddings.device
# Generate video replacement token IDs using get_video_repl
# This tokenizes each frame separator independently, then uses pre-tokenized
# special tokens to ensure consistent tokenization regardless of
# num_tokens_per_frame values.
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
tokens_per_frame=num_tokens_per_frame,
tokenizer=self._tokenizer,
timestamps=timestamps,
vision_start_token_id=self.config.vision_start_token_id,
vision_end_token_id=self.config.vision_end_token_id,
video_token_id=self.config.video_token_id,
select_token_id=self.is_multimodal_pruning_enabled,
)
repl_token_ids = torch.tensor(video_repl.full, device=device)
embed_token_id = _cached_tensor(self.config.video_token_id, device=device)
is_video_embed = torch.isin(repl_token_ids, embed_token_id)
# Get text embeddings for indicator tokens (has only `visual_dim``).
text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids)
if self.use_deepstack:
(
deepstack_input_embeds,
multimodal_embeddings,
) = self._compute_deepstack_embeds(
inputs_embeds=text_embeddings,
multimodal_embeddings=[video_embeddings],
is_multimodal=is_video_embed,
)
else:
deepstack_input_embeds = None
multimodal_embeddings = [video_embeddings]
merged_embeddings = _merge_multimodal_embeddings(
inputs_embeds=text_embeddings,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_video_embed,
)
to_concat = [merged_embeddings]
if deepstack_input_embeds is not None:
to_concat.append(
deepstack_input_embeds.permute(1, 0, 2).reshape(
deepstack_input_embeds.shape[1], -1
)
)
expanded_positions = None
if self.is_multimodal_pruning_enabled:
is_vision_start = repl_token_ids.eq(self.config.vision_start_token_id)
expanded_positions = self._get_expanded_positions(
device=merged_embeddings.device,
seq_len=merged_embeddings.shape[0],
video_grid_thw=video_grid_thw,
num_tokens_per_frame=num_tokens_per_frame,
timestamps=timestamps,
is_video_embed=is_video_embed,
is_vision_start=is_vision_start,
retention_mask=retention_mask,
)
to_concat.append(expanded_positions)
final_video_embeddings = torch.cat(to_concat, dim=-1)
return final_video_embeddings
def _get_expanded_positions(
self,
device,
seq_len,
video_grid_thw,
num_tokens_per_frame,
timestamps,
is_video_embed,
is_vision_start,
retention_mask,
):
embed_token_id = _cached_tensor(self.config.video_token_id, device=device)
# Expand positions to match the full sequence length
# (includes both video tokens and indicator tokens)
# Shape: [full_length, 5] where positions are filled for video tokens
# and zeros for indicator tokens.
# Channel 3 flags VISION_START tokens so that
# recompute_mrope_positions can reliably count timestamp tokens
# (even when early frames have all video tokens pruned).
# Channel 4 flags video-embedding tokens.
expanded_positions = torch.zeros(
seq_len,
5, # [t_index, h_index, w_index, is_vision_start, is_video]
device=device,
dtype=torch.long,
)
_, h, w = video_grid_thw
merge_size = self.visual.spatial_merge_size
num_frames = len(num_tokens_per_frame)
unpruned_token_ids = Qwen3VLMultiModalProcessor.get_video_repl(
tokens_per_frame=[(h // merge_size) * (w // merge_size)] * num_frames,
tokenizer=self._tokenizer,
timestamps=timestamps,
vision_start_token_id=self.config.vision_start_token_id,
vision_end_token_id=self.config.vision_end_token_id,
video_token_id=self.config.video_token_id,
).full
unpruned_token_ids_tensor = torch.tensor(unpruned_token_ids, device=device)
mm_feature = MultiModalFeatureSpec(
data=MultiModalKwargsItem(
{
"video_grid_thw": MultiModalFieldElem(
data=torch.tensor(video_grid_thw),
field=MultiModalFieldConfig.batched("video").field,
),
}
),
modality="video",
identifier="DUMMY",
mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),
)
original_mrope_cpu = self.get_mrope_input_positions(
input_tokens=unpruned_token_ids,
mm_features=[mm_feature],
)[0]
original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0)
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
with gpu_sync_allowed():
expanded_positions[is_video_embed, :3] = original_mrope[
full_is_video_embed
][retention_mask]
expanded_positions[~is_video_embed, :3] = original_mrope[
~full_is_video_embed
]
expanded_positions[..., 3] = is_vision_start
expanded_positions[..., 4] = is_video_embed
return expanded_positions
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality: dict[
str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
] = {}
for input_key in kwargs:
if (
input_key in ("pixel_values", "image_embeds")
and "image" not in mm_input_by_modality
):
mm_input_by_modality["image"] = self._parse_and_validate_image_input(
**kwargs
)
if (
input_key in ("pixel_values_videos", "video_embeds")
and "video" not in mm_input_by_modality
):
mm_input_by_modality["video"] = self._parse_and_validate_video_input(
**kwargs
)
return mm_input_by_modality
@staticmethod
def _iter_mm_grid_hw(
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
video_token_id: int,
vision_start_token_id: int,
vision_end_token_id: int,
spatial_merge_size: int,
) -> Iterator[tuple[int, int, int, int]]:
"""Iterate over multimodal features and yield position info.
Args:
input_tokens: List of token IDs in the input sequence.
mm_features: List of multimodal feature specifications containing
image/video data and position information.
video_token_id: Token ID used for video tokens.
vision_start_token_id: Token ID marking the start of a vision sequence.
vision_end_token_id: Token ID marking the end of a vision sequence.
spatial_merge_size: Size of the spatial merge operation used to
compute logical grid dimensions from the original feature grid.
Yields:
offset: Position of the first video/image token in the sequence.
llm_grid_h: Logical grid height (may not match actual token count with EVS).
llm_grid_w: Logical grid width (may not match actual token count with EVS).
actual_num_tokens: Actual number of video/image tokens in the placeholder.
"""
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset
data = mm_feature.data
assert data is not None
if mm_feature.modality == "image":
grid = data["image_grid_thw"].data
assert isinstance(grid, torch.Tensor)
t, h, w = grid.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
llm_grid_h = h // spatial_merge_size
llm_grid_w = w // spatial_merge_size
yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w
elif mm_feature.modality == "video":
grid = data["video_grid_thw"].data
assert isinstance(grid, torch.Tensor)
t, h, w = grid.tolist()
llm_grid_h = h // spatial_merge_size
llm_grid_w = w // spatial_merge_size
for _ in range(t):
# When EVS is enabled, some frames may have 0 video tokens in the
# placeholder. We use `vision_start_token_id` to locate each frame
# since it is always present for every frame.
# We then look for the first `video_token_id` after
# `vision_start_token_id` and before `vision_end_token_id`.
offset = input_tokens.index(vision_start_token_id, offset)
vision_end_offset = input_tokens.index(vision_end_token_id, offset)
try:
actual_num_tokens = 0
video_offset = input_tokens.index(
video_token_id, offset, vision_end_offset
)
# NOTE: looking at the
# `Qwen3VLMultiModalProcessor.get_video_repl` code, we can
# see that we can use the below formula to get the token
# count, since everything in between `video_offset` and
# `vision_end_offset` is populated as `video_token_id`.
# This saves us from manually counting the number tokens
# that match `video_token_id` in between.
actual_num_tokens += vision_end_offset - video_offset
except ValueError:
# No `video_token_id` in this frame (EVS with 0 tokens for
# this frame) -> use `offset + 1`` to move past
# `vision_start_token_id`.
video_offset = offset + 1
yield video_offset, llm_grid_h, llm_grid_w, actual_num_tokens
# Move offset past this frame for next iteration.
offset = vision_end_offset + 1
else:
raise ValueError(f"Unsupported modality: {mm_feature.modality}")
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
return self._get_mrope_input_positions(
input_tokens=input_tokens,
mm_features=mm_features,
config=self.config,
)
@staticmethod
def _get_mrope_input_positions(
input_tokens: list[int],
mm_features: list[MultiModalFeatureSpec],
config: Qwen3VLConfig,
):
llm_pos_ids_list: list[np.ndarray] = []
st = 0
for (
offset,
llm_grid_h,
llm_grid_w,
actual_num_tokens,
) in Qwen3VLForConditionalGeneration._iter_mm_grid_hw(
input_tokens,
mm_features,
video_token_id=config.video_token_id,
vision_start_token_id=config.vision_start_token_id,
vision_end_token_id=config.vision_end_token_id,
spatial_merge_size=config.vision_config.spatial_merge_size,
):
# Skip frames with 0 tokens (EVS placeholder with tokens lumped elsewhere)
if actual_num_tokens == 0:
continue
text_len = offset - st
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
llm_pos_ids_list.append(
np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
)
# Check if this is a "lumped placeholder" (all tokens from multiple frames
# assigned to the 0-th frame - see
# `Qwen3VLMultiModalProcessor.get_video_repl`.
expected_tokens_per_frame = llm_grid_h * llm_grid_w
if actual_num_tokens > expected_tokens_per_frame:
# Lumped placeholder: create grid positions for all "logical" frames
# represented.
num_logical_frames = actual_num_tokens // expected_tokens_per_frame
remainder = actual_num_tokens % expected_tokens_per_frame
# Create positions for complete frames.
for _ in range(num_logical_frames):
grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(
3, -1
)
llm_pos_ids_list.append(grid_indices + text_len + st_idx)
st_idx = llm_pos_ids_list[-1].max() + 1
text_len = 0 # No text between frames within the lump
# Handle remainder tokens if any (partial frame).
# NOTE: this should never be the case. Should we have an assert?
if remainder > 0:
# Create a partial grid - take first 'remainder' positions
full_grid = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)
grid_indices = full_grid[:, :remainder]
llm_pos_ids_list.append(grid_indices + text_len + st_idx)
else:
# Normal case: frame has exactly the expected tokens (after actual EVS
# pruning).
grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)
llm_pos_ids_list.append(grid_indices + text_len + st_idx)
st = offset + actual_num_tokens
if st < len(input_tokens):
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
text_len = len(input_tokens) - st
llm_pos_ids_list.append(
np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
)
llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
return torch.from_numpy(llm_positions), mrope_position_delta
def recompute_mrope_positions(
self,
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
"""Update part of input mrope positions (starting with
num_computed_tokens index). Original mrope_positions are computed
for unpruned sequence and becomes incorrect once pruning occurs,
so once we prune media tokens we should reflect this in the
mrope_positions before we feed it to LLM.
Args:
input_ids: (N,) All input tokens of the prompt containing
entire sequence.
multimodal_embeddings: Sequence of multimodal embeddings that
fits into the prefill chunk that is being processed.
mrope_positions: Existing mrope positions (3, N) for entire
sequence
num_computed_tokens: A number of computed tokens so far.
Returns:
Tuple of (multimodal_embeddings, mrope_positions,
mrope_position_delta).
"""
return self._recompute_mrope_positions(
input_ids=input_ids,
multimodal_embeddings=multimodal_embeddings,
mrope_positions=mrope_positions,
num_computed_tokens=num_computed_tokens,
image_token_id=self.config.image_token_id,
video_token_id=self.config.video_token_id,
vision_start_token_id=self.config.vision_start_token_id,
)
@staticmethod
def _recompute_mrope_positions(
input_ids: list[int] | torch.Tensor,
multimodal_embeddings: Sequence[torch.Tensor],
mrope_positions: torch.LongTensor,
num_computed_tokens: int,
vision_start_token_id: int,
image_token_id: int,
video_token_id: int,
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
# Device
device = (
multimodal_embeddings[0].device
if len(multimodal_embeddings)
else mrope_positions.device
)
# Tensors. input_ids may already be a (device-side) tensor.
if isinstance(input_ids, torch.Tensor):
assert input_ids.device == device
input_ids_t = input_ids.to(dtype=torch.long)
else:
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
mm_embeddings_out = []
mm_embeddings_pos = []
# Strip position information from embeddings (last 5 channels)
# For Qwen3 VL, handle potentially empty frames (from unpacking)
for mm in multimodal_embeddings:
if mm.shape[0] > 0: # Only process non-empty frames
mm_embeddings_out.append(mm[:, :-5])
mm_embeddings_pos.append(mm[:, -5:].permute(1, 0).long())
else:
# Empty frame - keep as is
mm_embeddings_out.append(mm)
# Create empty position tensor with correct shape
mm_embeddings_pos.append(
torch.empty(5, 0, device=device, dtype=torch.long)
)
with gpu_sync_allowed():
positions, mrope_positions_delta = recompute_mrope_positions(
input_ids_t,
mm_embeddings_pos,
mrope_positions,
num_computed_tokens,
vision_start_token_id,
image_token_id,
video_token_id,
)
return mm_embeddings_out, positions, mrope_positions_delta
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return None
# The result multimodal_embeddings is tuple of tensors, with each
# tensor corresponding to a multimodal data item (image or video).
multimodal_embeddings: list[torch.Tensor] = []
# NOTE: It is important to iterate over the keys in this dictionary
# to preserve the order of the modalities.
for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image":
image_embeddings = self._process_image_input(multimodal_input)
image_embeddings = self._postprocess_image_embeds_evs(
image_embeddings, multimodal_input
)
multimodal_embeddings.extend(image_embeddings)
if modality == "video":
video_embeddings = self._process_video_input(multimodal_input)
if self.is_multimodal_pruning_enabled:
video_embeddings = self._postprocess_video_embeds_evs(
video_embeddings, multimodal_input
)
multimodal_embeddings.extend(video_embeddings)
embeddings_tuple = tuple(multimodal_embeddings)
return embeddings_tuple
def _compute_deepstack_embeds(
self,
inputs_embeds: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings,
is_multimodal: torch.Tensor,
) -> tuple[torch.Tensor, MultiModalEmbeddings]:
visual_lens = [len(x) for x in multimodal_embeddings]
multimodal_embeddings_cat = torch.cat(multimodal_embeddings, dim=0)
(
multimodal_embeddings_main,
multimodal_embeddings_multiscale,
) = torch.split(
multimodal_embeddings_cat,
[self.visual_dim, self.multiscale_dim],
dim=-1,
)
multimodal_embeddings = torch.split(
multimodal_embeddings_main, visual_lens, dim=0
)
multimodal_embeddings_multiscale = torch.split(
multimodal_embeddings_multiscale, visual_lens, dim=0
)
deepstack_input_embeds = inputs_embeds.new_zeros(
inputs_embeds.size(0), self.deepstack_num_level * inputs_embeds.size(1)
)
deepstack_input_embeds = _merge_multimodal_embeddings(
inputs_embeds=deepstack_input_embeds,
multimodal_embeddings=multimodal_embeddings_multiscale,
is_multimodal=is_multimodal,
)
deepstack_input_embeds = deepstack_input_embeds.view(
inputs_embeds.shape[0], self.deepstack_num_level, self.visual_dim
)
deepstack_input_embeds = deepstack_input_embeds.permute(1, 0, 2)
return deepstack_input_embeds, multimodal_embeddings
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
inputs_embeds = self._embed_text_input_ids(
input_ids,
self.language_model.embed_input_ids,
is_multimodal=is_multimodal,
)
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
return inputs_embeds
is_multimodal = _require_is_multimodal(is_multimodal)
if self.use_deepstack:
(
deepstack_input_embeds,
multimodal_embeddings,
) = self._compute_deepstack_embeds(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
else:
deepstack_input_embeds = None
inputs_embeds = _merge_multimodal_embeddings(
inputs_embeds=inputs_embeds,
multimodal_embeddings=multimodal_embeddings,
is_multimodal=is_multimodal,
)
if deepstack_input_embeds is not None:
self._set_deepstack_input_embeds(deepstack_input_embeds)
return inputs_embeds
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for Qwen3VL.
Args:
input_ids: Flattened (concatenated) input_ids corresponding to a
batch.
positions: Flattened (concatenated) position ids corresponding to a
batch.
**NOTE**: If mrope is enabled (default setting for Qwen3VL
opensource models), the shape will be `(3, seq_len)`,
otherwise it will be `(seq_len,).
intermediate_tensors: Intermediate tensors from previous pipeline
stages.
inputs_embeds: Pre-computed input embeddings.
**kwargs: Additional keyword arguments including:
- pixel_values: Pixel values to be fed to a model.
`None` if no images are passed.
- image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in
LLM. `None` if no images are passed.
- pixel_values_videos: Pixel values of videos to be fed to a
model. `None` if no videos are passed.
- video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in
LLM. `None` if no videos are passed.
"""
if intermediate_tensors is not None:
inputs_embeds = None
if inputs_embeds is not None and get_pp_group().is_first_rank:
deepstack_input_embeds = self._get_deepstack_input_embeds(
inputs_embeds.size(0)
)
else:
deepstack_input_embeds = None
hidden_states = self.language_model.model(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
# args for deepstack
deepstack_input_embeds=deepstack_input_embeds,
)
if inputs_embeds is not None and get_pp_group().is_first_rank:
self._clear_deepstack_input_embeds(inputs_embeds.size(0))
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_mm_mapping(self) -> MultiModelKeys:
"""Get the module prefix in multimodal models."""
return MultiModelKeys.from_string_field(
language_model="language_model",
connector=["visual.merger", "visual.deepstack_merger_list"],
tower_model="visual.",
)
def get_mm_lora_token_counts(
self,
*,
modality: str,
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
del modality, mm_kwargs
hf_config = self.config
vision_config = hf_config.vision_config
merge_size = vision_config.spatial_merge_size
return num_mm_embeds * merge_size**2, num_mm_embeds
@lru_cache
def _cached_tensor(x, device) -> torch.Tensor:
return torch.tensor(x, device=device)