Qwen3_VisionTransformer derived its compute device from patch_embed.proj.weight and fed pos_embed.weight straight into the position-embedding interpolation kernel, so the last two anchor weights had to stay in VRAM for a tower to be streamable at all. Record the device the tower was built on before the offloader moves anything, and fall back to it when no weight is on the accelerator. Move the interpolation table to that device per encode, which costs one 8.4 MiB host-to-device copy for a 456 MiB tower. Towers now leave no parameters in VRAM between encodes; only small buffers such as the fused input norm and rotary inv_freq remain. Also let a tower restrict streaming through tower_offload_children, and warn when a tower resolves its device property to another device type, so a tower that reads an anchor weight reports the cause instead of a Triton pointer error. Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
3278 lines
124 KiB
Python
3278 lines
124 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
# Copyright 2025 The vLLM team.
|
|
# Copyright 2025 The Qwen Team.
|
|
# Copyright 2025 The HuggingFace Inc. team.
|
|
# All rights reserved.
|
|
#
|
|
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
|
# and OPT implementations in this library. It has been modified from its
|
|
# original forms to accommodate minor architectural differences compared
|
|
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
"""Inference-only Qwen3VL model compatible with HuggingFace weights."""
|
|
|
|
from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
|
|
from functools import lru_cache, partial
|
|
from itertools import islice
|
|
from typing import Any, Literal
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
from transformers import BatchFeature, ProcessorMixin
|
|
from transformers.models.qwen2_vl import Qwen2VLImageProcessor
|
|
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
|
|
smart_resize as image_smart_resize,
|
|
)
|
|
from transformers.models.qwen3_vl import Qwen3VLProcessor, Qwen3VLVideoProcessor
|
|
from transformers.models.qwen3_vl.configuration_qwen3_vl import (
|
|
Qwen3VLConfig,
|
|
Qwen3VLVisionConfig,
|
|
)
|
|
from transformers.models.qwen3_vl.video_processing_qwen3_vl import (
|
|
smart_resize as video_smart_resize,
|
|
)
|
|
from transformers.video_utils import VideoMetadata
|
|
|
|
from vllm.compilation.decorators import (
|
|
should_torch_compile_mm_encoder,
|
|
support_torch_compile,
|
|
)
|
|
from vllm.config import VllmConfig, get_current_vllm_config
|
|
from vllm.config.multimodal import (
|
|
MultiModalConfig,
|
|
MultiModalDummyOptions,
|
|
VideoDummyOptions,
|
|
VideoPruningMethod,
|
|
)
|
|
from vllm.distributed import get_pp_group, parallel_state
|
|
from vllm.inputs import MultiModalDataDict, MultiModalHashes
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
|
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
|
MMEncoderAttention,
|
|
)
|
|
from vllm.model_executor.layers.conv import Conv3dLayer
|
|
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
|
IdentityInputNorm,
|
|
build_mm_input_norm,
|
|
)
|
|
from vllm.model_executor.layers.linear import (
|
|
ColumnParallelLinear,
|
|
RowParallelLinear,
|
|
)
|
|
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
|
from vllm.model_executor.layers.quantization import QuantizationConfig
|
|
from vllm.model_executor.layers.rotary_embedding import get_rope
|
|
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
|
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
|
from vllm.multimodal.hasher import MultiModalHasher
|
|
from vllm.multimodal.inputs import (
|
|
MultiModalFeatureSpec,
|
|
MultiModalFieldConfig,
|
|
MultiModalFieldElem,
|
|
MultiModalKwargsItem,
|
|
MultiModalKwargsItems,
|
|
PlaceholderRange,
|
|
VideoItem,
|
|
)
|
|
from vllm.multimodal.parse import ImageSize, MultiModalDataItems
|
|
from vllm.multimodal.processing import (
|
|
BaseDummyInputsBuilder,
|
|
BaseMultiModalProcessor,
|
|
ProcessorInputs,
|
|
PromptReplacement,
|
|
PromptUpdate,
|
|
PromptUpdateDetails,
|
|
)
|
|
from vllm.multimodal.video_prune.evs import (
|
|
compute_mrope_for_media,
|
|
compute_retained_tokens_count,
|
|
compute_retention_mask,
|
|
recompute_mrope_positions,
|
|
)
|
|
from vllm.multimodal.video_prune.vidcom2 import (
|
|
compute_retained_tokens_count as vidcom2_compute_retained_tokens_count,
|
|
)
|
|
from vllm.multimodal.video_prune.vidcom2 import (
|
|
compute_retention_mask as vidcom2_compute_retention_mask,
|
|
)
|
|
from vllm.sequence import IntermediateTensors
|
|
from vllm.tokenizers.protocol import TokenizerLike
|
|
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
|
from vllm.triton_utils import HAS_TRITON, tl, triton
|
|
from vllm.utils.cache import LRUCache
|
|
from vllm.utils.collection_utils import is_list_of
|
|
from vllm.utils.math_utils import cdiv, round_up
|
|
from vllm.utils.torch_utils import PIN_MEMORY
|
|
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
|
|
|
|
from ...utils.gpu_sync_debug import gpu_sync_allowed
|
|
from ...utils.torch_utils import async_tensor_h2d
|
|
from .interfaces import (
|
|
MultiModalEmbeddings,
|
|
SupportsEagle,
|
|
SupportsEagle3,
|
|
SupportsEncoderCudaGraph,
|
|
SupportsLoRA,
|
|
SupportsMRoPE,
|
|
SupportsMultiModal,
|
|
SupportsMultiModalPruning,
|
|
SupportsPP,
|
|
_require_is_multimodal,
|
|
)
|
|
from .qwen2_5_vl import (
|
|
Qwen2_5_VisionAttention,
|
|
Qwen2_5_VLImageEmbeddingInputs,
|
|
Qwen2_5_VLImageInputs,
|
|
Qwen2_5_VLImagePixelInputs,
|
|
Qwen2_5_VLVideoEmbeddingInputs,
|
|
Qwen2_5_VLVideoInputs,
|
|
Qwen2_5_VLVideoPixelInputs,
|
|
)
|
|
from .qwen2_vl import (
|
|
Qwen2VLMultiModalDataParser,
|
|
Qwen2VLProcessingInfo,
|
|
_create_qwen2vl_field_factory,
|
|
)
|
|
from .qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM, Qwen3Model
|
|
from .utils import (
|
|
AutoWeightsLoader,
|
|
PPMissingLayer,
|
|
StageMissingLayer,
|
|
WeightsMapper,
|
|
_merge_multimodal_embeddings,
|
|
maybe_prefix,
|
|
)
|
|
from .vision import (
|
|
get_fp8_padded_hidden_size,
|
|
get_vit_attn_backend,
|
|
is_vit_use_data_parallel,
|
|
run_dp_sharded_mrope_vision_model,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# We use 2048 dummy video frames that would generate vision embeddings
|
|
# of the maximum size.
|
|
DUMMY_VIDEO_NUM_FRAMES = 2048
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Triton kernel: fused bilinear position-embedding interpolation
|
|
# ---------------------------------------------------------------------------
|
|
# Replaces many small eager-mode CUDA kernels with a single launch.
|
|
# The spatial-merge reorder is baked into the index math so the output
|
|
# is ready to be added to the patch embeddings directly.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
if HAS_TRITON:
|
|
|
|
@triton.jit
|
|
def _bilinear_pos_embed_kernel(
|
|
embed_ptr,
|
|
output_ptr,
|
|
H,
|
|
W,
|
|
h_scale,
|
|
w_scale,
|
|
NUM_GRID: tl.constexpr,
|
|
M_SIZE: tl.constexpr,
|
|
HIDDEN_DIM: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
):
|
|
"""Fused bilinear pos-embed interpolation with spatial-merge reorder."""
|
|
pid = tl.program_id(0)
|
|
total_spatial = H * W
|
|
spatial_idx = pid % total_spatial
|
|
|
|
num_blocks_w = W // M_SIZE
|
|
block_idx = spatial_idx // (M_SIZE * M_SIZE)
|
|
local_idx = spatial_idx % (M_SIZE * M_SIZE)
|
|
br = block_idx // num_blocks_w
|
|
bc = block_idx % num_blocks_w
|
|
lr = local_idx // M_SIZE
|
|
lc = local_idx % M_SIZE
|
|
row = br * M_SIZE + lr
|
|
col = bc * M_SIZE + lc
|
|
|
|
h_frac = row.to(tl.float32) * h_scale
|
|
w_frac = col.to(tl.float32) * w_scale
|
|
|
|
hf = tl.math.floor(h_frac).to(tl.int32)
|
|
wf = tl.math.floor(w_frac).to(tl.int32)
|
|
hc = tl.minimum(hf + 1, NUM_GRID - 1)
|
|
wc = tl.minimum(wf + 1, NUM_GRID - 1)
|
|
|
|
dh = h_frac - hf.to(tl.float32)
|
|
dw = w_frac - wf.to(tl.float32)
|
|
w11 = dh * dw
|
|
w10 = dh - w11
|
|
w01 = dw - w11
|
|
w00 = 1.0 - dh - w01
|
|
|
|
off00 = (hf * NUM_GRID + wf) * HIDDEN_DIM
|
|
off01 = (hf * NUM_GRID + wc) * HIDDEN_DIM
|
|
off10 = (hc * NUM_GRID + wf) * HIDDEN_DIM
|
|
off11 = (hc * NUM_GRID + wc) * HIDDEN_DIM
|
|
out_off = pid * HIDDEN_DIM
|
|
|
|
# Cast weights to output dtype so the multiply-accumulate stays
|
|
# in the same precision as the native PyTorch implementation.
|
|
out_dtype = output_ptr.dtype.element_ty
|
|
w00_c = w00.to(out_dtype)
|
|
w01_c = w01.to(out_dtype)
|
|
w10_c = w10.to(out_dtype)
|
|
w11_c = w11.to(out_dtype)
|
|
|
|
for d in tl.range(0, HIDDEN_DIM, BLOCK_D):
|
|
cols = d + tl.arange(0, BLOCK_D)
|
|
mask = cols < HIDDEN_DIM
|
|
|
|
e00 = tl.load(embed_ptr + off00 + cols, mask=mask)
|
|
e01 = tl.load(embed_ptr + off01 + cols, mask=mask)
|
|
e10 = tl.load(embed_ptr + off10 + cols, mask=mask)
|
|
e11 = tl.load(embed_ptr + off11 + cols, mask=mask)
|
|
|
|
val = w00_c * e00 + w01_c * e01 + w10_c * e10 + w11_c * e11
|
|
|
|
tl.store(output_ptr + out_off + cols, val, mask=mask)
|
|
|
|
def triton_pos_embed_interpolate(
|
|
embed_weight: torch.Tensor,
|
|
t: int,
|
|
h: int,
|
|
w: int,
|
|
num_grid_per_side: int,
|
|
m_size: int,
|
|
dtype: torch.dtype,
|
|
) -> torch.Tensor:
|
|
"""Launch the fused Triton kernel for one (t,h,w) grid.
|
|
|
|
Returns a tensor of shape ``(t * h * w, hidden_dim)`` with the
|
|
bilinearly-interpolated position embeddings in spatial-merge order.
|
|
"""
|
|
assert h % m_size == 0 and w % m_size == 0, (
|
|
f"h={h} and w={w} must be divisible by m_size={m_size}"
|
|
)
|
|
hidden_dim = embed_weight.shape[1]
|
|
total_out = t * h * w
|
|
output = torch.empty(
|
|
total_out,
|
|
hidden_dim,
|
|
device=embed_weight.device,
|
|
dtype=dtype,
|
|
)
|
|
|
|
h_scale = float(num_grid_per_side - 1) / float(h - 1) if h > 1 else 0.0
|
|
w_scale = float(num_grid_per_side - 1) / float(w - 1) if w > 1 else 0.0
|
|
|
|
BLOCK_D = triton.next_power_of_2(hidden_dim)
|
|
|
|
_bilinear_pos_embed_kernel[(total_out,)](
|
|
embed_weight,
|
|
output,
|
|
h,
|
|
w,
|
|
h_scale,
|
|
w_scale,
|
|
num_grid_per_side,
|
|
m_size,
|
|
hidden_dim,
|
|
BLOCK_D,
|
|
)
|
|
return output
|
|
|
|
|
|
def pos_embed_interpolate_native(
|
|
embed_weight: torch.Tensor,
|
|
t: int,
|
|
h: int,
|
|
w: int,
|
|
num_grid_per_side: int,
|
|
m_size: int,
|
|
dtype: torch.dtype,
|
|
) -> torch.Tensor:
|
|
"""Eager PyTorch bilinear position-embedding interpolation.
|
|
|
|
Returns a tensor of shape ``(t * h * w, hidden_dim)`` with the
|
|
bilinearly-interpolated position embeddings in spatial-merge order.
|
|
"""
|
|
assert h % m_size == 0 and w % m_size == 0, (
|
|
f"h={h} and w={w} must be divisible by m_size={m_size}"
|
|
)
|
|
hidden_dim = embed_weight.shape[1]
|
|
device = embed_weight.device
|
|
|
|
h_idxs = torch.linspace(
|
|
0,
|
|
num_grid_per_side - 1,
|
|
h,
|
|
dtype=torch.float32,
|
|
device=device,
|
|
)
|
|
w_idxs = torch.linspace(
|
|
0,
|
|
num_grid_per_side - 1,
|
|
w,
|
|
dtype=torch.float32,
|
|
device=device,
|
|
)
|
|
|
|
h_floor = h_idxs.to(torch.long)
|
|
w_floor = w_idxs.to(torch.long)
|
|
h_ceil = torch.clamp(h_floor + 1, max=num_grid_per_side - 1)
|
|
w_ceil = torch.clamp(w_floor + 1, max=num_grid_per_side - 1)
|
|
|
|
dh = h_idxs - h_floor
|
|
dw = w_idxs - w_floor
|
|
|
|
dh_grid, dw_grid = torch.meshgrid(dh, dw, indexing="ij")
|
|
h_floor_grid, w_floor_grid = torch.meshgrid(h_floor, w_floor, indexing="ij")
|
|
h_ceil_grid, w_ceil_grid = torch.meshgrid(h_ceil, w_ceil, indexing="ij")
|
|
|
|
w11 = dh_grid * dw_grid
|
|
w10 = dh_grid - w11
|
|
w01 = dw_grid - w11
|
|
w00 = 1 - dh_grid - w01
|
|
|
|
h_grid = torch.stack([h_floor_grid, h_floor_grid, h_ceil_grid, h_ceil_grid])
|
|
w_grid = torch.stack([w_floor_grid, w_ceil_grid, w_floor_grid, w_ceil_grid])
|
|
h_grid_idx = h_grid * num_grid_per_side
|
|
|
|
indices = (h_grid_idx + w_grid).reshape(4, -1)
|
|
weights = torch.stack([w00, w01, w10, w11], dim=0).reshape(4, -1, 1)
|
|
weights = weights.to(dtype=dtype)
|
|
|
|
embeds = embed_weight[indices]
|
|
embeds *= weights
|
|
combined = embeds.sum(dim=0)
|
|
|
|
combined = combined.reshape(h // m_size, m_size, w // m_size, m_size, hidden_dim)
|
|
combined = combined.permute(0, 2, 1, 3, 4).reshape(1, -1, hidden_dim)
|
|
repeated = combined.expand(t, -1, -1).reshape(-1, hidden_dim)
|
|
return repeated.to(dtype=dtype)
|
|
|
|
|
|
@support_torch_compile(
|
|
dynamic_arg_dims={
|
|
"x": 0,
|
|
},
|
|
enable_if=should_torch_compile_mm_encoder,
|
|
is_encoder=True,
|
|
)
|
|
class Qwen3_VisionPatchEmbed(nn.Module):
|
|
def __init__(
|
|
self,
|
|
patch_size: int = 14,
|
|
temporal_patch_size: int = 2,
|
|
in_channels: int = 3,
|
|
hidden_size: int = 1152,
|
|
) -> None:
|
|
super().__init__()
|
|
self.patch_size = patch_size
|
|
self.temporal_patch_size = temporal_patch_size
|
|
self.hidden_size = hidden_size
|
|
|
|
kernel_size = (temporal_patch_size, patch_size, patch_size)
|
|
self.proj = Conv3dLayer(
|
|
in_channels,
|
|
hidden_size,
|
|
kernel_size=kernel_size,
|
|
stride=kernel_size,
|
|
bias=True,
|
|
)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
L, C = x.shape
|
|
x = x.view(L, -1, self.temporal_patch_size, self.patch_size, self.patch_size)
|
|
x = self.proj(x).view(L, self.hidden_size)
|
|
return x
|
|
|
|
|
|
class Qwen3_VisionMLP(nn.Module):
|
|
def __init__(
|
|
self,
|
|
in_features: int,
|
|
hidden_features: int,
|
|
bias: bool = False,
|
|
act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
):
|
|
super().__init__()
|
|
use_data_parallel = is_vit_use_data_parallel()
|
|
self.linear_fc1 = ColumnParallelLinear(
|
|
in_features,
|
|
hidden_features,
|
|
bias=bias,
|
|
quant_config=quant_config,
|
|
return_bias=False,
|
|
prefix=f"{prefix}.linear_fc1",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
self.linear_fc2 = RowParallelLinear(
|
|
hidden_features,
|
|
in_features,
|
|
bias=bias,
|
|
quant_config=quant_config,
|
|
return_bias=False,
|
|
prefix=f"{prefix}.linear_fc2",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
self.act_fn = act_fn
|
|
|
|
def forward(self, x: torch.Tensor):
|
|
mlp_output = self.linear_fc2(self.act_fn(self.linear_fc1(x)))
|
|
return mlp_output
|
|
|
|
|
|
@support_torch_compile(
|
|
dynamic_arg_dims={
|
|
"x": 0,
|
|
"cu_seqlens": 0,
|
|
"rotary_pos_emb_cos": 0,
|
|
"rotary_pos_emb_sin": 0,
|
|
"sequence_lengths": 0,
|
|
},
|
|
enable_if=should_torch_compile_mm_encoder,
|
|
is_encoder=True,
|
|
)
|
|
class Qwen3_VisionBlock(nn.Module):
|
|
def __init__(
|
|
self,
|
|
dim: int,
|
|
num_heads: int,
|
|
mlp_hidden_dim: int,
|
|
act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
|
|
norm_layer: Callable[[int], nn.Module] | None = None,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
if norm_layer is None:
|
|
norm_layer = partial(nn.LayerNorm, eps=1e-6)
|
|
self.norm1 = norm_layer(dim)
|
|
self.norm2 = norm_layer(dim)
|
|
self.attn = Qwen2_5_VisionAttention(
|
|
embed_dim=dim,
|
|
num_heads=num_heads,
|
|
projection_size=dim,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.attn",
|
|
)
|
|
self.mlp = Qwen3_VisionMLP(
|
|
dim,
|
|
mlp_hidden_dim,
|
|
act_fn=act_fn,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.mlp",
|
|
)
|
|
|
|
def forward(
|
|
self,
|
|
x: torch.Tensor,
|
|
cu_seqlens: torch.Tensor,
|
|
rotary_pos_emb_cos: torch.Tensor,
|
|
rotary_pos_emb_sin: torch.Tensor,
|
|
max_seqlen: torch.Tensor, # Only used for Flash Attention
|
|
sequence_lengths: torch.Tensor, # Only used for FlashInfer CuDNN backend
|
|
) -> torch.Tensor:
|
|
x = x + self.attn(
|
|
self.norm1(x),
|
|
cu_seqlens=cu_seqlens,
|
|
rotary_pos_emb_cos=rotary_pos_emb_cos,
|
|
rotary_pos_emb_sin=rotary_pos_emb_sin,
|
|
max_seqlen=max_seqlen,
|
|
sequence_lengths=sequence_lengths,
|
|
)
|
|
|
|
x = x + self.mlp(self.norm2(x))
|
|
return x
|
|
|
|
|
|
class Qwen3_VisionPatchMerger(nn.Module):
|
|
def __init__(
|
|
self,
|
|
d_model: int,
|
|
context_dim: int,
|
|
norm_layer: Callable[[int], nn.Module] | None = None,
|
|
spatial_merge_size: int = 2,
|
|
use_postshuffle_norm: bool = False,
|
|
quant_config: QuantizationConfig | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
use_data_parallel = is_vit_use_data_parallel()
|
|
self.hidden_size = context_dim * (spatial_merge_size**2)
|
|
|
|
self.use_postshuffle_norm = use_postshuffle_norm
|
|
if self.use_postshuffle_norm:
|
|
context_dim = self.hidden_size
|
|
|
|
if norm_layer is None:
|
|
norm_layer = partial(nn.LayerNorm, eps=1e-6)
|
|
self.norm = norm_layer(context_dim)
|
|
self.linear_fc1 = ColumnParallelLinear(
|
|
self.hidden_size,
|
|
self.hidden_size,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.linear_fc1",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
self.act_fn = nn.GELU()
|
|
self.linear_fc2 = RowParallelLinear(
|
|
self.hidden_size,
|
|
d_model,
|
|
bias=True,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.linear_fc2",
|
|
disable_tp=use_data_parallel,
|
|
)
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
if self.use_postshuffle_norm:
|
|
x = self.norm(x.view(-1, self.hidden_size))
|
|
else:
|
|
x = self.norm(x).view(-1, self.hidden_size)
|
|
|
|
x_parallel, _ = self.linear_fc1(x)
|
|
x_parallel = self.act_fn(x_parallel)
|
|
out, _ = self.linear_fc2(x_parallel)
|
|
return out
|
|
|
|
|
|
class Qwen3_VisionTransformer(nn.Module):
|
|
hf_to_vllm_mapper = WeightsMapper(
|
|
orig_to_new_stacked={
|
|
"attn.q.": ("attn.qkv.", "q"),
|
|
"attn.k.": ("attn.qkv.", "k"),
|
|
"attn.v.": ("attn.qkv.", "v"),
|
|
}
|
|
)
|
|
|
|
def __init__(
|
|
self,
|
|
vision_config: Qwen3VLVisionConfig,
|
|
norm_eps: float = 1e-6,
|
|
quant_config: QuantizationConfig | None = None,
|
|
input_norm: nn.Module | None = None,
|
|
prefix: str = "",
|
|
) -> None:
|
|
super().__init__()
|
|
self.hidden_size = vision_config.hidden_size
|
|
self.num_heads = vision_config.num_heads
|
|
self.num_position_embeddings = vision_config.num_position_embeddings
|
|
self.patch_size = vision_config.patch_size
|
|
self.spatial_merge_size = vision_config.spatial_merge_size
|
|
self.spatial_merge_unit = self.spatial_merge_size**2
|
|
self.temporal_patch_size = vision_config.temporal_patch_size
|
|
self.deepstack_visual_indexes = (
|
|
vision_config.deepstack_visual_indexes
|
|
if hasattr(vision_config, "deepstack_visual_indexes")
|
|
else []
|
|
)
|
|
self.num_grid_per_side = int(self.num_position_embeddings**0.5)
|
|
self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = (
|
|
LRUCache(capacity=1024)
|
|
)
|
|
|
|
use_data_parallel = is_vit_use_data_parallel()
|
|
self.tp_size = (
|
|
1
|
|
if use_data_parallel
|
|
else parallel_state.get_tensor_model_parallel_world_size()
|
|
)
|
|
|
|
# NOTE: This is used for creating empty tensor for all_gather for
|
|
# DP ViT. Here out_hidden_size is enlarged due to deepstack
|
|
self.out_hidden_size = vision_config.out_hidden_size * (
|
|
1 + len(self.deepstack_visual_indexes)
|
|
)
|
|
|
|
self.patch_embed = Qwen3_VisionPatchEmbed(
|
|
patch_size=self.patch_size,
|
|
temporal_patch_size=self.temporal_patch_size,
|
|
in_channels=vision_config.in_channels,
|
|
hidden_size=self.hidden_size,
|
|
)
|
|
self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()
|
|
|
|
self.pos_embed = nn.Embedding(self.num_position_embeddings, self.hidden_size)
|
|
|
|
norm_layer = partial(nn.LayerNorm, eps=norm_eps)
|
|
head_dim = self.hidden_size // self.num_heads
|
|
|
|
# FP8 attention: Q/K/V become independent contiguous tensors
|
|
# after quantization, so cu_seqlens uses uniform stride (no 3x V).
|
|
self.fp8_padded_hidden_size = get_fp8_padded_hidden_size(
|
|
self.num_heads, head_dim
|
|
)
|
|
|
|
self.rotary_pos_emb = get_rope(
|
|
head_size=head_dim,
|
|
max_position=8192,
|
|
is_neox_style=True,
|
|
rope_parameters={"partial_rotary_factor": 0.5},
|
|
)
|
|
|
|
self.merger = Qwen3_VisionPatchMerger(
|
|
d_model=vision_config.out_hidden_size,
|
|
context_dim=self.hidden_size,
|
|
norm_layer=norm_layer,
|
|
spatial_merge_size=self.spatial_merge_size,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.merger",
|
|
)
|
|
|
|
self.deepstack_merger_list = nn.ModuleList(
|
|
[
|
|
Qwen3_VisionPatchMerger(
|
|
d_model=vision_config.out_hidden_size,
|
|
context_dim=self.hidden_size,
|
|
spatial_merge_size=self.spatial_merge_size,
|
|
use_postshuffle_norm=True,
|
|
norm_layer=norm_layer,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.deepstack_merger_list.{layer_idx}",
|
|
)
|
|
for layer_idx in range(len(self.deepstack_visual_indexes))
|
|
]
|
|
)
|
|
|
|
self.attn_backend = get_vit_attn_backend(
|
|
head_size=head_dim,
|
|
dtype=torch.get_default_dtype(),
|
|
)
|
|
|
|
self.blocks = nn.ModuleList(
|
|
[
|
|
Qwen3_VisionBlock(
|
|
dim=self.hidden_size,
|
|
num_heads=self.num_heads,
|
|
mlp_hidden_dim=vision_config.intermediate_size,
|
|
act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
|
|
norm_layer=norm_layer,
|
|
quant_config=quant_config,
|
|
prefix=f"{prefix}.blocks.{layer_idx}",
|
|
)
|
|
for layer_idx in range(vision_config.depth)
|
|
]
|
|
)
|
|
|
|
# An offloader moves tower parameters to host RAM after this
|
|
# constructor returns, and no weight is left on the accelerator to
|
|
# read the compute device from. Record it here while they are still
|
|
# placed, so `device` keeps working once every weight is streamed.
|
|
first_param = next(self.parameters(), None)
|
|
self._compute_device = (
|
|
torch.device("cpu") if first_param is None else first_param.device
|
|
)
|
|
|
|
@property
|
|
def dtype(self) -> torch.dtype:
|
|
return self.patch_embed.proj.weight.dtype
|
|
|
|
@property
|
|
def device(self) -> torch.device:
|
|
weight_device = self.patch_embed.proj.weight.device
|
|
if weight_device.type != "cpu":
|
|
return weight_device
|
|
return self._compute_device
|
|
|
|
def rot_pos_ids(self, h: int, w: int, spatial_merge_size: int) -> torch.Tensor:
|
|
cache_key = (h, w, spatial_merge_size)
|
|
if cache_key in self._rot_pos_ids_cache:
|
|
return self._rot_pos_ids_cache[cache_key]
|
|
|
|
hpos_ids = np.broadcast_to(np.arange(h).reshape(h, 1), (h, w))
|
|
h_div = h // spatial_merge_size
|
|
w_div = w // spatial_merge_size
|
|
hpos_ids = hpos_ids.reshape(
|
|
h_div,
|
|
spatial_merge_size,
|
|
w_div,
|
|
spatial_merge_size,
|
|
)
|
|
hpos_ids = hpos_ids.transpose(0, 2, 1, 3)
|
|
hpos_ids = hpos_ids.flatten()
|
|
|
|
wpos_ids = np.broadcast_to(np.arange(w).reshape(1, w), (h, w))
|
|
wpos_ids = wpos_ids.reshape(
|
|
h_div,
|
|
spatial_merge_size,
|
|
w_div,
|
|
spatial_merge_size,
|
|
)
|
|
wpos_ids = wpos_ids.transpose(0, 2, 1, 3)
|
|
wpos_ids = wpos_ids.flatten()
|
|
|
|
result = torch.from_numpy(np.stack([hpos_ids, wpos_ids], axis=-1))
|
|
self._rot_pos_ids_cache[cache_key] = result
|
|
return result
|
|
|
|
def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]):
|
|
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
|
|
pos_ids = [
|
|
self.rot_pos_ids(h, w, self.spatial_merge_size)
|
|
if t == 1
|
|
else self.rot_pos_ids(h, w, self.spatial_merge_size).repeat(t, 1)
|
|
for t, h, w in grid_thw
|
|
]
|
|
num_pos = sum(p.shape[0] for p in pos_ids)
|
|
pinned = torch.empty(
|
|
(num_pos, pos_ids[0].shape[1]),
|
|
dtype=pos_ids[0].dtype,
|
|
pin_memory=PIN_MEMORY,
|
|
)
|
|
pos_ids = torch.cat(pos_ids, dim=0, out=pinned).to(
|
|
self.device, non_blocking=True
|
|
)
|
|
|
|
# Use pre-computed cos_sin_cache from RotaryEmbedding
|
|
cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)
|
|
|
|
cos_combined = cos[pos_ids].flatten(1)
|
|
sin_combined = sin[pos_ids].flatten(1)
|
|
|
|
return cos_combined, sin_combined
|
|
|
|
def fast_pos_embed_interpolate(
|
|
self, grid_thw: Sequence[Sequence[int]]
|
|
) -> torch.Tensor:
|
|
interpolate_fn = (
|
|
triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native
|
|
)
|
|
outputs = []
|
|
for t, h, w in grid_thw:
|
|
outputs.append(
|
|
interpolate_fn(
|
|
self.pos_embed_table(),
|
|
t,
|
|
h,
|
|
w,
|
|
self.num_grid_per_side,
|
|
self.spatial_merge_size,
|
|
self.dtype,
|
|
)
|
|
)
|
|
return torch.cat(outputs, dim=0)
|
|
|
|
def pos_embed_table(self) -> torch.Tensor:
|
|
"""The position embedding table on the compute device. The whole tower
|
|
may be streamed from host RAM, but the interpolation kernel cannot read
|
|
host memory.
|
|
"""
|
|
table = self.pos_embed.weight
|
|
if table.device.type == "cpu":
|
|
table = table.to(self.device, non_blocking=True)
|
|
return table
|
|
|
|
def prepare_encoder_metadata(
|
|
self,
|
|
grid_thw_list: Sequence[Sequence[int]],
|
|
*,
|
|
max_batch_size: int | None = None,
|
|
max_frames_per_batch: int | None = None,
|
|
max_seqlen_override: int | None = None,
|
|
device: torch.device | None = None,
|
|
) -> dict[str, torch.Tensor | None]:
|
|
"""Compute encoder metadata from grid_thw_list.
|
|
|
|
Shared by the eager forward path, CUDA graph capture, and
|
|
CUDA graph replay to avoid duplicated implementation.
|
|
|
|
Args:
|
|
grid_thw_list: Grid configurations as list of [t, h, w].
|
|
max_batch_size: If set, pad cu_seqlens to this size
|
|
(needed for CUDA graph capture/replay).
|
|
max_frames_per_batch: If set, overrides max_batch_size for
|
|
cu_seqlens padding. For video inputs each item contributes
|
|
T attention sequences (frames); this sizes the buffer to
|
|
the total frame budget so video replays never overflow.
|
|
max_seqlen_override: If set, use this value for max_seqlen
|
|
instead of computing from cu_seqlens (needed for CUDA
|
|
graph capture to cover worst-case replay scenarios).
|
|
device: Device to place tensors on. Defaults to self.device.
|
|
|
|
"""
|
|
if device is None:
|
|
device = self.device
|
|
|
|
metadata: dict[str, torch.Tensor | None] = {}
|
|
|
|
# Positional embeddings
|
|
metadata["pos_embeds"] = self.fast_pos_embed_interpolate(grid_thw_list)
|
|
rotary_cos, rotary_sin = self.rot_pos_emb(grid_thw_list)
|
|
metadata["rotary_pos_emb_cos"] = rotary_cos
|
|
metadata["rotary_pos_emb_sin"] = rotary_sin
|
|
|
|
# cu_seqlens from grid_thw
|
|
grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
|
|
patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
|
|
cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
|
|
dtype=np.int32
|
|
)
|
|
cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])
|
|
|
|
# Pad cu_seqlens to the required number of sequences.
|
|
# For videos each item contributes T frames = T attention sequences,
|
|
# so the total can exceed max_batch_size. max_frames_per_batch
|
|
# overrides the pad target when set.
|
|
pad_to = (
|
|
max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
|
|
)
|
|
if pad_to is not None:
|
|
num_seqs = len(cu_seqlens) - 1
|
|
if num_seqs < pad_to:
|
|
cu_seqlens = np.concatenate(
|
|
[
|
|
cu_seqlens,
|
|
np.full(
|
|
pad_to - num_seqs,
|
|
cu_seqlens[-1],
|
|
dtype=np.int32,
|
|
),
|
|
]
|
|
)
|
|
|
|
# sequence_lengths (backend-specific)
|
|
metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
|
|
self.attn_backend, cu_seqlens, device
|
|
)
|
|
|
|
# max_seqlen
|
|
if max_seqlen_override is not None:
|
|
max_seqlen_val = max_seqlen_override
|
|
else:
|
|
max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
|
|
self.attn_backend, cu_seqlens
|
|
)
|
|
# Keep max_seqlen on CPU: attention wrappers call .item() on it,
|
|
# and having it on GPU would capture a wasteful D2H copy in CUDA
|
|
# graphs without changing behavior (the scalar is baked at capture).
|
|
metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)
|
|
|
|
# Recompute cu_seqlens (backend-specific transformation)
|
|
metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
|
|
self.attn_backend,
|
|
cu_seqlens,
|
|
self.hidden_size,
|
|
self.tp_size,
|
|
device,
|
|
fp8_padded_hidden_size=self.fp8_padded_hidden_size,
|
|
)
|
|
|
|
return metadata
|
|
|
|
def forward(
|
|
self,
|
|
x: torch.Tensor,
|
|
grid_thw: torch.Tensor | list[list[int]],
|
|
*,
|
|
encoder_metadata: dict[str, torch.Tensor] | None = None,
|
|
) -> torch.Tensor:
|
|
hidden_states = self.input_norm(
|
|
x.to(device=self.device, non_blocking=True), self.dtype
|
|
)
|
|
hidden_states = self.patch_embed(hidden_states)
|
|
|
|
if encoder_metadata is None:
|
|
if isinstance(grid_thw, list):
|
|
grid_thw_list = grid_thw
|
|
else:
|
|
grid_thw_list = grid_thw.tolist()
|
|
encoder_metadata = self.prepare_encoder_metadata(grid_thw_list)
|
|
|
|
pos_embeds = encoder_metadata["pos_embeds"]
|
|
hidden_states = hidden_states + pos_embeds
|
|
hidden_states = hidden_states.unsqueeze(1)
|
|
|
|
deepstack_feature_lists = []
|
|
for layer_num, blk in enumerate(self.blocks):
|
|
hidden_states = blk(
|
|
hidden_states,
|
|
cu_seqlens=encoder_metadata["cu_seqlens"],
|
|
rotary_pos_emb_cos=encoder_metadata["rotary_pos_emb_cos"],
|
|
rotary_pos_emb_sin=encoder_metadata["rotary_pos_emb_sin"],
|
|
max_seqlen=encoder_metadata["max_seqlen"],
|
|
sequence_lengths=encoder_metadata.get("sequence_lengths"),
|
|
)
|
|
if layer_num in self.deepstack_visual_indexes:
|
|
deepstack_merger_idx = self.deepstack_visual_indexes.index(layer_num)
|
|
deepstack_feature = self.deepstack_merger_list[deepstack_merger_idx](
|
|
hidden_states
|
|
)
|
|
deepstack_feature_lists.append(deepstack_feature)
|
|
hidden_states = self.merger(hidden_states)
|
|
hidden_states = torch.cat(
|
|
[hidden_states] + deepstack_feature_lists, dim=1
|
|
) # [seq_len, hidden_size * (1 + depth_of_deepstack)]
|
|
return hidden_states
|
|
|
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
loader = AutoWeightsLoader(self)
|
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
|
|
|
|
|
class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
|
# Timestamps are part of the prompt replacement (each frame group's
|
|
# "<X.X seconds>" text), so they size the placeholder range and must
|
|
# travel with the grid when embeddings are delivered out of band.
|
|
embedding_fields = {
|
|
**Qwen2VLMultiModalDataParser.embedding_fields,
|
|
"video": {
|
|
**Qwen2VLMultiModalDataParser.embedding_fields["video"],
|
|
"timestamps": "metadata",
|
|
},
|
|
}
|
|
|
|
|
|
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
|
def get_hf_config(self):
|
|
return self.ctx.get_hf_config(Qwen3VLConfig)
|
|
|
|
def get_hf_processor(self, **kwargs: object) -> Qwen3VLProcessor:
|
|
return self.ctx.get_hf_processor(
|
|
Qwen3VLProcessor,
|
|
**kwargs,
|
|
)
|
|
|
|
def get_image_processor(self, **kwargs: object) -> Qwen2VLImageProcessor:
|
|
return self.get_hf_processor(**kwargs).image_processor
|
|
|
|
def get_video_processor(self, **kwargs: object) -> Qwen3VLVideoProcessor:
|
|
return self.get_hf_processor(**kwargs).video_processor
|
|
|
|
def get_data_parser(self):
|
|
return Qwen3VLMultiModalDataParser(
|
|
self.get_hf_config().vision_config.spatial_merge_size,
|
|
video_needs_metadata=True,
|
|
expected_hidden_size=self._get_expected_hidden_size(),
|
|
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
|
)
|
|
|
|
def get_supported_mm_processor_kwargs(self) -> dict[str, set[str]]:
|
|
supported = super().get_supported_mm_processor_kwargs()
|
|
|
|
# vLLM has historically accepted `min_pixels` / `max_pixels` for
|
|
# Qwen3-VL video. Include them in the routing schema even though
|
|
# Qwen3VLVideoProcessor only accepts the equivalent `size`; they are
|
|
# folded into `size` before calling HF.
|
|
supported["videos_kwargs"].update({"min_pixels", "max_pixels"})
|
|
return supported
|
|
|
|
def _get_vision_info(
|
|
self,
|
|
*,
|
|
image_width: int,
|
|
image_height: int,
|
|
num_frames: int = 2,
|
|
do_resize: bool = True,
|
|
image_processor: Qwen2VLImageProcessor | Qwen3VLVideoProcessor,
|
|
mm_kwargs: Mapping[str, object],
|
|
modality: Literal["image", "video"] | None = None,
|
|
) -> tuple[ImageSize, int]:
|
|
is_video = isinstance(image_processor, Qwen3VLVideoProcessor)
|
|
|
|
hf_config = self.get_hf_config()
|
|
vision_config = hf_config.vision_config
|
|
patch_size = vision_config.patch_size
|
|
merge_size = vision_config.spatial_merge_size
|
|
temporal_patch_size = vision_config.temporal_patch_size
|
|
|
|
if modality is None:
|
|
modality = "video" if is_video else "image"
|
|
|
|
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs(mm_kwargs)
|
|
scope = "videos_kwargs" if modality == "video" else "images_kwargs"
|
|
scoped_mm_kwargs = merged_mm_kwargs.get(scope, {})
|
|
size = self._get_vision_size(
|
|
scoped_mm_kwargs,
|
|
default_size=image_processor.size,
|
|
)
|
|
|
|
if do_resize:
|
|
if is_video:
|
|
smart_resize = video_smart_resize
|
|
extra_kwargs = {
|
|
"num_frames": num_frames,
|
|
"temporal_factor": temporal_patch_size,
|
|
}
|
|
else:
|
|
smart_resize = image_smart_resize
|
|
extra_kwargs = {}
|
|
|
|
resized_height, resized_width = smart_resize(
|
|
height=image_height,
|
|
width=image_width,
|
|
factor=patch_size * merge_size,
|
|
min_pixels=size["shortest_edge"],
|
|
max_pixels=size["longest_edge"],
|
|
**extra_kwargs,
|
|
)
|
|
preprocessed_size = ImageSize(width=resized_width, height=resized_height)
|
|
else:
|
|
preprocessed_size = ImageSize(width=image_width, height=image_height)
|
|
|
|
padded_num_frames = round_up(num_frames, temporal_patch_size)
|
|
|
|
grid_t = max(padded_num_frames // temporal_patch_size, 1)
|
|
grid_h = preprocessed_size.height // patch_size
|
|
grid_w = preprocessed_size.width // patch_size
|
|
|
|
num_patches = grid_t * grid_h * grid_w
|
|
num_vision_tokens = num_patches // (merge_size**2)
|
|
|
|
return preprocessed_size, num_vision_tokens
|
|
|
|
def _get_max_video_frames(self, max_tokens: int, start_num_frames: int = 2) -> int:
|
|
return super()._get_max_video_frames(
|
|
max_tokens, start_num_frames=start_num_frames
|
|
)
|
|
|
|
def get_num_frames_with_most_features(
|
|
self,
|
|
seq_len: int,
|
|
mm_counts: Mapping[str, int],
|
|
max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES,
|
|
) -> int:
|
|
return super().get_num_frames_with_most_features(
|
|
seq_len, mm_counts, max_frames_per_video=max_frames_per_video
|
|
)
|
|
|
|
def get_max_video_tokens(
|
|
self,
|
|
seq_len: int,
|
|
mm_counts: Mapping[str, int],
|
|
) -> int:
|
|
video_processor = self.get_video_processor()
|
|
|
|
merged_mm_kwargs = self._merge_and_resolve_mm_processor_kwargs({})
|
|
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
|
|
video_size = self._get_vision_size(
|
|
video_mm_kwargs,
|
|
default_size=video_processor.size,
|
|
)
|
|
temporal_patch_size = video_mm_kwargs.get(
|
|
"temporal_patch_size", video_processor.temporal_patch_size
|
|
)
|
|
|
|
# video_max_pixels contains the temporal compression factor,
|
|
# so we divide by 2 to get the maximum number of image pixels.
|
|
video_max_pixels = video_size["longest_edge"]
|
|
target_width, target_height = self.get_image_size_with_most_features(
|
|
max_pixels=video_max_pixels // temporal_patch_size
|
|
)
|
|
num_video_soft_tokens = self.get_num_video_tokens(
|
|
image_width=target_width,
|
|
image_height=target_height,
|
|
num_frames=2,
|
|
image_processor=video_processor,
|
|
mm_kwargs={},
|
|
)
|
|
return num_video_soft_tokens
|
|
|
|
def _calculate_timestamps(
|
|
self, indices: list[int] | torch.Tensor, video_fps: float, merge_size: int
|
|
):
|
|
if not isinstance(indices, list):
|
|
indices = indices.tolist()
|
|
if len(indices) % merge_size != 0:
|
|
# don't update metadata's frames_indices directly
|
|
indices = indices + [indices[-1]] * (merge_size - len(indices) % merge_size)
|
|
timestamps = [idx / video_fps for idx in indices]
|
|
timestamps = [
|
|
(timestamps[i] + timestamps[i + merge_size - 1]) / 2
|
|
for i in range(0, len(timestamps), merge_size)
|
|
]
|
|
return timestamps
|
|
|
|
def _get_video_second_idx(
|
|
self,
|
|
metadata: dict[str, Any],
|
|
do_sample_frames: bool | None = None,
|
|
sampled_fps: float | None = None,
|
|
sampled_num_frames: int | None = None,
|
|
) -> list[int]:
|
|
video_processor = self.get_video_processor()
|
|
temporal_patch_size = video_processor.temporal_patch_size
|
|
indices = metadata["frames_indices"]
|
|
|
|
# metadata["fps"] refers to the true fps of the input video.
|
|
video_fps = metadata["fps"]
|
|
if do_sample_frames is None:
|
|
do_sample_frames = metadata.get("do_sample_frames", False)
|
|
|
|
# If video frames are sampled in HF processor (instead of vLLM
|
|
# video loader), we need to re-calculate the indices from original
|
|
# metadata.
|
|
if do_sample_frames:
|
|
total_num_frames = metadata["total_num_frames"]
|
|
|
|
# When num_frames is explicitly provided, use it directly
|
|
# instead of computing from fps. This mirrors the behavior of
|
|
# HF's Qwen3VLVideoProcessor.sample_frames where num_frames
|
|
# and fps are mutually exclusive.
|
|
if sampled_num_frames is not None:
|
|
num_frames = sampled_num_frames
|
|
else:
|
|
# here video_fps is the fps of the sampled video, and
|
|
# metadata["fps"] refers to the fps of the original video.
|
|
sampled_fps = sampled_fps if sampled_fps else video_processor.fps
|
|
num_frames = int(total_num_frames / metadata["fps"] * sampled_fps)
|
|
|
|
num_frames = min(
|
|
min(
|
|
max(num_frames, video_processor.min_frames),
|
|
video_processor.max_frames,
|
|
),
|
|
total_num_frames,
|
|
)
|
|
indices = (
|
|
np.linspace(0, total_num_frames - 1, num_frames)
|
|
.round()
|
|
.astype(int)
|
|
.tolist()
|
|
)
|
|
timestamps = self._calculate_timestamps(indices, video_fps, temporal_patch_size)
|
|
return timestamps
|
|
|
|
|
|
class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
|
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
|
num_images = mm_counts.get("image", 0)
|
|
num_videos = mm_counts.get("video", 0)
|
|
|
|
image_token = "<|vision_start|><|image_pad|><|vision_end|>"
|
|
video_token = "<|vision_start|><|video_pad|><|vision_end|>"
|
|
|
|
return image_token * num_images + video_token * num_videos
|
|
|
|
def get_dummy_mm_data(
|
|
self,
|
|
seq_len: int,
|
|
mm_counts: Mapping[str, int],
|
|
mm_options: MultiModalDummyOptions,
|
|
) -> MultiModalDataDict:
|
|
video_overrides = mm_options.get("video")
|
|
|
|
target_image_width, target_image_height = (
|
|
self.info.get_image_size_with_most_features()
|
|
)
|
|
|
|
# treat videos as special images
|
|
target_num_frames = 2
|
|
if video_overrides:
|
|
num_frames_override = video_overrides.num_frames
|
|
if num_frames_override:
|
|
if num_frames_override > target_num_frames:
|
|
logger.warning(
|
|
"video.num_frames override (%d) exceeds model's "
|
|
"maximum number of frames (%d), will be ignored",
|
|
num_frames_override,
|
|
target_num_frames,
|
|
)
|
|
if num_frames_override < 2:
|
|
logger.warning(
|
|
"video.num_frames override (%d) cannot be less "
|
|
"than 2, will be ignored",
|
|
num_frames_override,
|
|
)
|
|
target_num_frames = min(target_num_frames, num_frames_override)
|
|
target_num_frames = max(target_num_frames, 2)
|
|
|
|
video_processor = self.info.get_video_processor()
|
|
|
|
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs({})
|
|
video_mm_kwargs = merged_mm_kwargs.get("videos_kwargs", {})
|
|
video_size = self.info._get_vision_size(
|
|
video_mm_kwargs,
|
|
default_size=video_processor.size,
|
|
)
|
|
temporal_patch_size = video_mm_kwargs.get(
|
|
"temporal_patch_size", video_processor.temporal_patch_size
|
|
)
|
|
|
|
# video_max_pixels contains the temporal compression factor,
|
|
# so we divide by 2 to get the maximum number of image pixels.
|
|
video_max_pixels = video_size["longest_edge"]
|
|
|
|
# With the HF processor's per-frame pixel cap enabled
|
|
# (cap_pixels_per_frame, huggingface/transformers#48071), a 2-frame
|
|
# dummy is processed at only 2 * cap pixels, so memory profiling
|
|
# underestimates the largest possible item: a fully sampled video
|
|
# still reaches the whole longest_edge budget. Spread the budget
|
|
# over enough frames that the cap is not binding for the dummy.
|
|
# This takes precedence over a shrinking num_frames override, which
|
|
# would otherwise reintroduce the under-profiling.
|
|
if video_mm_kwargs.get("cap_pixels_per_frame"):
|
|
max_video_tokens = video_mm_kwargs.get(
|
|
"max_video_tokens",
|
|
getattr(video_processor, "max_video_tokens", 768),
|
|
)
|
|
patch_size = video_mm_kwargs.get("patch_size", video_processor.patch_size)
|
|
merge_size = video_mm_kwargs.get("merge_size", video_processor.merge_size)
|
|
per_frame_cap = max_video_tokens * (patch_size * merge_size) ** 2
|
|
target_num_frames = max(
|
|
target_num_frames,
|
|
min(video_processor.max_frames, cdiv(video_max_pixels, per_frame_cap)),
|
|
)
|
|
|
|
target_video_width, target_video_height = (
|
|
self.info.get_image_size_with_most_features(
|
|
max_pixels=video_max_pixels // temporal_patch_size
|
|
)
|
|
)
|
|
target_video_size, _ = self.info._get_vision_info(
|
|
image_width=target_video_width,
|
|
image_height=target_video_height,
|
|
num_frames=target_num_frames,
|
|
image_processor=video_processor,
|
|
mm_kwargs={},
|
|
)
|
|
# NOTE: we need to do this check here since Qwen3-VL resizes video
|
|
# frames depending on how many frames there are.
|
|
target_video_width, target_video_height = (
|
|
target_video_size.width,
|
|
target_video_size.height,
|
|
)
|
|
if video_overrides:
|
|
width_override = video_overrides.width
|
|
if width_override:
|
|
if width_override > target_video_width:
|
|
logger.warning(
|
|
"video.width override (%d) exceeds model's "
|
|
"maximum width (%d), will be ignored",
|
|
width_override,
|
|
target_video_width,
|
|
)
|
|
target_video_width = min(target_video_width, width_override)
|
|
height_override = video_overrides.height
|
|
if height_override:
|
|
if height_override > target_video_height:
|
|
logger.warning(
|
|
"video.height override (%d) exceeds model's "
|
|
"maximum height (%d), will be ignored",
|
|
height_override,
|
|
target_video_height,
|
|
)
|
|
target_video_height = min(target_video_height, height_override)
|
|
|
|
return {
|
|
"image": self._get_dummy_images(
|
|
width=target_image_width,
|
|
height=target_image_height,
|
|
num_images=mm_counts.get("image", 0),
|
|
overrides=mm_options.get("image"),
|
|
),
|
|
"video": self._get_dummy_videos(
|
|
width=target_video_width,
|
|
height=target_video_height,
|
|
num_frames=target_num_frames,
|
|
num_videos=mm_counts.get("video", 0),
|
|
),
|
|
}
|
|
|
|
def _get_dummy_videos(
|
|
self,
|
|
*,
|
|
width: int,
|
|
height: int,
|
|
num_frames: int,
|
|
num_videos: int,
|
|
overrides: VideoDummyOptions | None = None,
|
|
) -> list[VideoItem]:
|
|
videos = super()._get_dummy_videos(
|
|
width=width,
|
|
height=height,
|
|
num_frames=num_frames,
|
|
num_videos=num_videos,
|
|
overrides=overrides,
|
|
)
|
|
videos = [v.copy() for v in videos]
|
|
|
|
video_items: list[VideoItem] = []
|
|
for video in videos:
|
|
video_num_frames = video.shape[0]
|
|
video_metadata = {
|
|
"fps": 2.0,
|
|
"duration": video_num_frames / 2.0,
|
|
"total_num_frames": video_num_frames,
|
|
"frames_indices": list(range(video_num_frames)),
|
|
"video_backend": "opencv",
|
|
"do_sample_frames": False,
|
|
}
|
|
video_items.append((video, video_metadata))
|
|
|
|
return video_items
|
|
|
|
|
|
def _replace_video_token_placeholders(
|
|
prompt_ids: list[int],
|
|
target: list[int],
|
|
replacements: list[list[int]],
|
|
) -> list[int]:
|
|
"""Replace each video placeholder with its expanded sequence.
|
|
|
|
Args:
|
|
prompt_ids: Token IDs of the original (unexpanded) prompt.
|
|
target: Token-ID sequence to search for. New-style
|
|
(transformers>=5.10) processors expand only the bare
|
|
`[video_pad_id]`; older ones expand the full
|
|
`[vision_start_id, video_pad_id, vision_end_id]` triplet.
|
|
replacements: Per-video expanded token sequences, in prompt order.
|
|
|
|
Returns:
|
|
Token IDs with every placeholder replaced.
|
|
|
|
"""
|
|
result: list[int] = []
|
|
repl_idx = 0
|
|
i = 0
|
|
n = len(prompt_ids)
|
|
k = len(target)
|
|
num_repl = len(replacements)
|
|
|
|
while i < n:
|
|
if prompt_ids[i : i + k] == target:
|
|
result.extend(replacements[repl_idx])
|
|
repl_idx += 1
|
|
i += k
|
|
else:
|
|
result.append(prompt_ids[i])
|
|
i += 1
|
|
|
|
assert repl_idx == num_repl, (
|
|
f"Found {repl_idx} video placeholders but expected {num_repl}"
|
|
)
|
|
return result
|
|
|
|
|
|
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
|
|
def _get_per_video_kwargs(
|
|
self, kwargs: Mapping[str, object], num_videos: int | None = None
|
|
) -> dict[str, list[Any]]:
|
|
merged = self.info._merge_and_resolve_mm_processor_kwargs(kwargs)
|
|
per_video_kwargs = {
|
|
key: value
|
|
for key, value in merged.get("videos_kwargs", {}).items()
|
|
if key in ("fps", "num_frames") and isinstance(value, list)
|
|
}
|
|
if num_videos:
|
|
for key, values in per_video_kwargs.items():
|
|
if len(values) < num_videos:
|
|
raise ValueError(
|
|
f"{key} must have at least {num_videos} entries "
|
|
"(one per video); "
|
|
f"got {len(values)}."
|
|
)
|
|
return per_video_kwargs
|
|
|
|
def _get_mm_hashes(self, inputs: ProcessorInputs) -> MultiModalHashes:
|
|
hashes = dict(super()._get_mm_hashes(inputs))
|
|
per_video_kwargs = self._get_per_video_kwargs(
|
|
inputs.hf_processor_mm_kwargs, len(hashes.get("video", []))
|
|
)
|
|
if per_video_kwargs and "video" in hashes:
|
|
algorithm = self.info.ctx.get_mm_config().mm_hasher_algorithm
|
|
hashes["video"] = [
|
|
MultiModalHasher.hash_kwargs(
|
|
algorithm,
|
|
mm_hash=mm_hash,
|
|
**{key: values[i] for key, values in per_video_kwargs.items()},
|
|
)
|
|
for i, mm_hash in enumerate(hashes["video"])
|
|
]
|
|
return hashes
|
|
|
|
def _get_cache_missing_processor_kwargs(
|
|
self,
|
|
inputs: ProcessorInputs,
|
|
mm_is_cached: Mapping[str, list[bool]],
|
|
) -> Mapping[str, object]:
|
|
per_video_kwargs = self._get_per_video_kwargs(
|
|
inputs.hf_processor_mm_kwargs, len(mm_is_cached.get("video", []))
|
|
)
|
|
if not per_video_kwargs or "video" not in mm_is_cached:
|
|
return inputs.hf_processor_mm_kwargs
|
|
kwargs = self.info._merge_and_resolve_mm_processor_kwargs(
|
|
inputs.hf_processor_mm_kwargs
|
|
)
|
|
kwargs["videos_kwargs"].update(
|
|
{
|
|
key: [
|
|
value
|
|
for value, hit in zip(values, mm_is_cached["video"])
|
|
if not hit
|
|
]
|
|
for key, values in per_video_kwargs.items()
|
|
}
|
|
)
|
|
return kwargs
|
|
|
|
@staticmethod
|
|
def _expands_only_video_token(hf_processor: ProcessorMixin) -> bool:
|
|
"""Transformers>=5.10 processors override `replace_video_token`
|
|
to expand only the bare video token, keeping the prompt's outer
|
|
`<|vision_start|>`/`<|vision_end|>` markers."""
|
|
mixin_impl = getattr(ProcessorMixin, "replace_video_token", None)
|
|
proc_impl = getattr(type(hf_processor), "replace_video_token", None)
|
|
return proc_impl is not None and proc_impl is not mixin_impl
|
|
|
|
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
|
return self.dummy_inputs.get_dummy_text(mm_counts)
|
|
|
|
def _apply_hf_processor_main(
|
|
self,
|
|
mm_items: MultiModalDataItems,
|
|
hf_kwargs: Mapping[str, object],
|
|
) -> BatchFeature:
|
|
hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs(
|
|
mm_items, hf_kwargs
|
|
)
|
|
|
|
if not hf_data:
|
|
return self._finalize_hf_mm_data(hf_data, hf_kwargs, passthrough_data)
|
|
|
|
# Merge and resolve the multimodal processor kwargs once so every HF
|
|
# call below starts from the same engine/request state and scoped form.
|
|
merged_mm_kwargs = self.info._merge_and_resolve_mm_processor_kwargs(hf_kwargs)
|
|
|
|
if "images_kwargs" in merged_mm_kwargs:
|
|
image_scoped_kwargs = dict(merged_mm_kwargs["images_kwargs"])
|
|
|
|
# Qwen2VLImageProcessor requires both `size` edges. If a partial size
|
|
# override is passed, fill missing edges from the processor defaults
|
|
# before any HF call.
|
|
if image_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
|
|
default_image_size = self.info.get_image_processor().size
|
|
image_scoped_kwargs["size"] = self.info._get_vision_size(
|
|
image_scoped_kwargs,
|
|
default_size=default_image_size,
|
|
)
|
|
|
|
merged_mm_kwargs["images_kwargs"] = image_scoped_kwargs
|
|
|
|
if "videos_kwargs" in merged_mm_kwargs:
|
|
video_scoped_kwargs = dict(merged_mm_kwargs["videos_kwargs"])
|
|
|
|
# Qwen3VLVideoProcessor requires both `size` edges. If a partial size
|
|
# override is passed, fill missing edges from the processor defaults
|
|
# before any HF call.
|
|
if video_scoped_kwargs.keys() & {"size", "min_pixels", "max_pixels"}:
|
|
default_video_size = self.info.get_video_processor().size
|
|
video_scoped_kwargs["size"] = self.info._get_vision_size(
|
|
video_scoped_kwargs,
|
|
default_size=default_video_size,
|
|
)
|
|
|
|
# Qwen3VLVideoProcessor does not accept vLLM's min/max pixel
|
|
# compatibility kwargs. They have already been folded into `size`,
|
|
# so remove them before any HF call, including calls that contain
|
|
# only text/images.
|
|
video_scoped_kwargs.pop("min_pixels", None)
|
|
video_scoped_kwargs.pop("max_pixels", None)
|
|
|
|
merged_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
|
|
|
|
# Separate video processing from image processing. Because the videos
|
|
# are processed into several image patches
|
|
video_input_ids_lst: list[list[int]] = []
|
|
if videos := hf_data.pop("videos", []):
|
|
assert isinstance(videos, list)
|
|
self._get_per_video_kwargs(hf_kwargs, len(videos))
|
|
video_grid_thw_lst = []
|
|
pixel_values_videos_lst = []
|
|
timestamps_per_video = []
|
|
|
|
hf_config = self.info.get_hf_config()
|
|
tokenizer = self.info.get_tokenizer()
|
|
merge_size = hf_config.vision_config.spatial_merge_size
|
|
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
|
|
vision_start_token_id = hf_config.vision_start_token_id
|
|
vision_end_token_id = hf_config.vision_end_token_id
|
|
video_token_id = hf_config.video_token_id
|
|
|
|
for item_idx, item in enumerate(videos):
|
|
video_array, metadata = item
|
|
|
|
# NOTE: @JJJYmmm new attr metadata.frames_indices indicates
|
|
# the sampled frames indices of pre-sampled videos, which is
|
|
# used to calculate the timestamps. Make sure that
|
|
# do_sample_frames in mm_kwargs is false for presampled videos.
|
|
|
|
# The root copy is shallow, so give each item its own video
|
|
# scope before modifying per-video values below.
|
|
video_mm_kwargs = dict(merged_mm_kwargs)
|
|
video_scoped_kwargs = dict(video_mm_kwargs.get("videos_kwargs", {}))
|
|
video_mm_kwargs["videos_kwargs"] = video_scoped_kwargs
|
|
|
|
sampled_fps = video_scoped_kwargs.get("fps")
|
|
if is_list_of(sampled_fps, float):
|
|
video_scoped_kwargs["fps"] = sampled_fps[item_idx]
|
|
sampled_num_frames = video_scoped_kwargs.get("num_frames")
|
|
if is_list_of(sampled_num_frames, int):
|
|
video_scoped_kwargs["num_frames"] = sampled_num_frames[item_idx]
|
|
if "do_sample_frames" not in video_scoped_kwargs:
|
|
# qwen_vl_utils already has "do_sample_frames" in
|
|
# mm_kwargs, don't overwrite it.
|
|
video_scoped_kwargs["do_sample_frames"] = metadata.get(
|
|
"do_sample_frames", False
|
|
)
|
|
|
|
metadata = VideoMetadata(
|
|
**{k: metadata[k] for k in metadata if k != "do_sample_frames"}
|
|
)
|
|
|
|
# Compute timestamps here where we have access to metadata
|
|
timestamps = self.info._get_video_second_idx(
|
|
metadata=metadata,
|
|
do_sample_frames=video_scoped_kwargs["do_sample_frames"],
|
|
sampled_fps=video_scoped_kwargs.get("fps"),
|
|
sampled_num_frames=video_scoped_kwargs.get("num_frames"),
|
|
)
|
|
timestamps_per_video.append(timestamps)
|
|
|
|
video_mm_data = dict()
|
|
video_mm_data["videos"] = [[video_array]]
|
|
video_mm_data["video_metadata"] = [[metadata]]
|
|
|
|
# When num_frames is specified, explicitly set fps=None
|
|
# to prevent HF's BaseVideoProcessor.preprocess() from
|
|
# filling in the class default (fps=2) via setdefault(),
|
|
# which would conflict with num_frames (mutually exclusive).
|
|
if (
|
|
"num_frames" in video_scoped_kwargs
|
|
and "fps" not in video_scoped_kwargs
|
|
):
|
|
video_scoped_kwargs["fps"] = None
|
|
|
|
video_outputs = self.info.ctx.call_hf_processor(
|
|
self.info.get_hf_processor(**video_mm_kwargs),
|
|
dict(
|
|
text="<|vision_start|><|video_pad|><|vision_end|>",
|
|
**video_mm_data,
|
|
),
|
|
video_mm_kwargs,
|
|
)
|
|
|
|
# Discard HF output input_ids — we use get_video_repl below
|
|
# to generate the correct (EVS-adjusted) token sequence.
|
|
video_outputs.pop("input_ids", None)
|
|
|
|
video_grid_thw = video_outputs["video_grid_thw"]
|
|
num_frames = int(video_grid_thw[0, 0])
|
|
tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // (
|
|
merge_size**2
|
|
)
|
|
|
|
# Apply video pruning (EVS or VidCom2) if enabled.
|
|
if pruning_spec is not None:
|
|
method, prune_q = pruning_spec
|
|
count_fn = (
|
|
vidcom2_compute_retained_tokens_count
|
|
if method == "vidcom2"
|
|
else compute_retained_tokens_count
|
|
)
|
|
num_tokens = count_fn(
|
|
tokens_per_frame=tokens_per_frame_base,
|
|
num_frames=num_frames,
|
|
q=prune_q,
|
|
)
|
|
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
|
|
select_token_id = False
|
|
else:
|
|
tokens_per_frame = [tokens_per_frame_base] * num_frames
|
|
select_token_id = True
|
|
|
|
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
|
|
tokens_per_frame=tokens_per_frame,
|
|
timestamps=timestamps,
|
|
tokenizer=tokenizer,
|
|
vision_start_token_id=vision_start_token_id,
|
|
vision_end_token_id=vision_end_token_id,
|
|
video_token_id=video_token_id,
|
|
select_token_id=select_token_id,
|
|
)
|
|
video_input_ids_lst.append(list(video_repl.full))
|
|
|
|
video_grid_thw_lst.append(video_outputs["video_grid_thw"])
|
|
pixel_values_videos_lst.append(video_outputs["pixel_values_videos"])
|
|
video_outputs = dict(
|
|
pixel_values_videos=torch.cat(pixel_values_videos_lst),
|
|
video_grid_thw=torch.cat(video_grid_thw_lst),
|
|
timestamps=timestamps_per_video,
|
|
)
|
|
else:
|
|
video_outputs = dict()
|
|
|
|
# The root copy is shallow, so copy the video scope before removing
|
|
# per-video kwargs to avoid mutating `merged_mm_kwargs`.
|
|
non_video_mm_kwargs = dict(merged_mm_kwargs)
|
|
if "videos_kwargs" in non_video_mm_kwargs:
|
|
non_video_scoped_kwargs = dict(non_video_mm_kwargs["videos_kwargs"])
|
|
non_video_scoped_kwargs.pop("fps", None)
|
|
non_video_scoped_kwargs.pop("num_frames", None)
|
|
# The context merges model defaults again when calling HF.
|
|
for key in self._get_per_video_kwargs({}):
|
|
non_video_scoped_kwargs[key] = None
|
|
non_video_mm_kwargs["videos_kwargs"] = non_video_scoped_kwargs
|
|
processed_data = self.info.ctx.call_hf_processor(
|
|
self.info.get_hf_processor(**non_video_mm_kwargs),
|
|
hf_data,
|
|
non_video_mm_kwargs,
|
|
)
|
|
|
|
# Replace each placeholder with pre-computed video tokens.
|
|
if video_input_ids_lst:
|
|
hf_config = self.info.get_hf_config()
|
|
if self._expands_only_video_token(self.info.get_hf_processor()):
|
|
video_target = [hf_config.video_token_id]
|
|
else:
|
|
video_target = [
|
|
hf_config.vision_start_token_id,
|
|
hf_config.video_token_id,
|
|
hf_config.vision_end_token_id,
|
|
]
|
|
input_ids = processed_data.pop("input_ids")
|
|
if not isinstance(input_ids, list):
|
|
input_ids = input_ids.tolist()
|
|
(prompt_ids,) = input_ids
|
|
expanded_ids = _replace_video_token_placeholders(
|
|
prompt_ids, video_target, video_input_ids_lst
|
|
)
|
|
processed_data["input_ids"] = [expanded_ids]
|
|
|
|
processed_data.update(video_outputs)
|
|
return self._finalize_hf_mm_data(
|
|
hf_data, hf_kwargs, passthrough_data, processed_data
|
|
)
|
|
|
|
def _get_mm_fields_config(
|
|
self,
|
|
hf_inputs: BatchFeature,
|
|
hf_processor_mm_kwargs: Mapping[str, object],
|
|
) -> Mapping[str, MultiModalFieldConfig]:
|
|
return _create_qwen2vl_field_factory(
|
|
self.info.get_hf_config().vision_config.spatial_merge_size
|
|
)(hf_inputs)
|
|
|
|
def _get_prompt_updates(
|
|
self,
|
|
mm_items: MultiModalDataItems,
|
|
hf_processor_mm_kwargs: Mapping[str, Any],
|
|
out_mm_kwargs: MultiModalKwargsItems,
|
|
) -> Sequence[PromptUpdate]:
|
|
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
|
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
|
tokenizer = self.info.get_tokenizer()
|
|
hf_config = self.info.get_hf_config()
|
|
|
|
video_token_id = hf_config.video_token_id
|
|
vision_start_token_id = hf_config.vision_start_token_id
|
|
vision_end_token_id = hf_config.vision_end_token_id
|
|
|
|
merge_length = image_processor.merge_size**2
|
|
|
|
def get_image_replacement_qwen3vl(item_idx: int):
|
|
out_item = out_mm_kwargs["image"][item_idx]
|
|
grid_thw = out_item["image_grid_thw"].data
|
|
assert isinstance(grid_thw, torch.Tensor)
|
|
|
|
num_tokens = int(grid_thw.prod()) // merge_length
|
|
return [hf_processor.image_token_id] * num_tokens
|
|
|
|
def get_video_replacement_qwen3vl(item_idx: int):
|
|
out_item = out_mm_kwargs["video"][item_idx]
|
|
grid_thw = out_item["video_grid_thw"].data
|
|
assert isinstance(grid_thw, torch.Tensor)
|
|
|
|
sampled_fps = hf_processor_mm_kwargs.get("fps")
|
|
if is_list_of(sampled_fps, float):
|
|
sampled_fps = sampled_fps[item_idx]
|
|
|
|
timestamps = out_item["timestamps"].data
|
|
assert isinstance(timestamps, (list, torch.Tensor))
|
|
assert len(timestamps) == grid_thw[0], (
|
|
f"The timestamps length({len(timestamps)}) should be equal "
|
|
f"video length ({grid_thw[0]})."
|
|
)
|
|
|
|
# Compute tokens per frame, with EVS / VidCom2 support
|
|
num_frames = int(grid_thw[0])
|
|
tokens_per_frame_base = int(grid_thw[1:].prod()) // merge_length
|
|
|
|
pruning_spec = self.info.ctx.get_mm_config().get_video_pruning_spec()
|
|
if pruning_spec is not None:
|
|
method, prune_q = pruning_spec
|
|
count_fn = (
|
|
vidcom2_compute_retained_tokens_count
|
|
if method == "vidcom2"
|
|
else compute_retained_tokens_count
|
|
)
|
|
num_tokens = count_fn(
|
|
tokens_per_frame=tokens_per_frame_base,
|
|
num_frames=num_frames,
|
|
q=prune_q,
|
|
)
|
|
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
|
|
select_token_id = False
|
|
else:
|
|
tokens_per_frame = [tokens_per_frame_base] * num_frames
|
|
select_token_id = True
|
|
|
|
return Qwen3VLMultiModalProcessor.get_video_repl(
|
|
tokens_per_frame=tokens_per_frame,
|
|
timestamps=timestamps,
|
|
tokenizer=tokenizer,
|
|
vision_start_token_id=vision_start_token_id,
|
|
vision_end_token_id=vision_end_token_id,
|
|
video_token_id=video_token_id,
|
|
select_token_id=select_token_id,
|
|
)
|
|
|
|
if self._expands_only_video_token(hf_processor):
|
|
# transformers>=5.10 expands only the bare video_token
|
|
video_target = [video_token_id]
|
|
else:
|
|
# Old-style processors expand the full placeholder
|
|
video_target = [
|
|
vision_start_token_id,
|
|
video_token_id,
|
|
vision_end_token_id,
|
|
]
|
|
|
|
return [
|
|
PromptReplacement(
|
|
modality="image",
|
|
target=[hf_processor.image_token_id],
|
|
replacement=get_image_replacement_qwen3vl,
|
|
),
|
|
PromptReplacement(
|
|
modality="video",
|
|
target=video_target,
|
|
replacement=get_video_replacement_qwen3vl,
|
|
),
|
|
]
|
|
|
|
@staticmethod
|
|
def get_video_repl(
|
|
*,
|
|
tokens_per_frame: list[int],
|
|
timestamps: Sequence[float] | torch.Tensor,
|
|
tokenizer: TokenizerLike,
|
|
vision_start_token_id: int,
|
|
vision_end_token_id: int,
|
|
video_token_id: int,
|
|
select_token_id: bool = False,
|
|
) -> PromptUpdateDetails:
|
|
"""Build prompt replacement for a video in Qwen3VL format.
|
|
|
|
The replacement structure for each frame is:
|
|
timestamp_tokens + vision_start_token + video_tokens + vision_end_token
|
|
|
|
Args:
|
|
tokens_per_frame: Number of video tokens per frame (can vary per frame for
|
|
EVS).
|
|
timestamps: List of timestamps in seconds for each frame
|
|
tokenizer: Tokenizer to encode timestamp strings
|
|
vision_start_token_id: Token ID for vision start marker
|
|
vision_end_token_id: Token ID for vision end marker
|
|
video_token_id: Token ID for video content
|
|
select_token_id: Whether to return token ids instead of text
|
|
|
|
Returns:
|
|
PromptUpdateDetails with full token sequence
|
|
|
|
"""
|
|
assert len(timestamps) == len(tokens_per_frame), (
|
|
"timestamps and tokens_per_frame must have the same length"
|
|
)
|
|
|
|
# Tokenize timestamp strings independently to avoid tokenizer merging
|
|
# tokens across boundaries.
|
|
timestamp_token_ids = [
|
|
tokenizer.encode(f"<{timestamp:.1f} seconds>", add_special_tokens=False)
|
|
for timestamp in timestamps
|
|
]
|
|
|
|
# Build the full token sequence
|
|
all_token_ids = []
|
|
for frame_timestamp_ids, num_tokens in zip(
|
|
timestamp_token_ids, tokens_per_frame
|
|
):
|
|
# Add timestamp tokens
|
|
all_token_ids.extend(frame_timestamp_ids)
|
|
|
|
# Add vision tokens: vision_start + video_tokens + vision_end
|
|
all_token_ids.append(vision_start_token_id)
|
|
all_token_ids.extend([video_token_id] * num_tokens)
|
|
all_token_ids.append(vision_end_token_id)
|
|
|
|
if select_token_id:
|
|
return PromptUpdateDetails.select_token_id(all_token_ids, video_token_id)
|
|
|
|
# NOTE: we use `from_seq` instead of `select_token_id` because we want all
|
|
# tokens in the placeholder to be initially marked as candidates. Then
|
|
# in `get_input_embeddings``, we refine the mask to only replace
|
|
# `video_token_id` / `image_token_id`` positions with video/image embeddings,
|
|
# keeping text embeddings for timestamps and structural tokens.
|
|
return PromptUpdateDetails.from_seq(all_token_ids)
|
|
|
|
|
|
def mrope_positions_factor(vllm_config: VllmConfig) -> int:
|
|
# Video pruning keeps the positions of the tokens it drops. A drafter gets
|
|
# the target's positions, so it follows the target.
|
|
spec = vllm_config.speculative_config
|
|
model_config = (spec and spec.target_model_config) or vllm_config.model_config
|
|
mm_config = model_config.multimodal_config
|
|
pruned = mm_config is not None and mm_config.is_multimodal_pruning_enabled()
|
|
return 4 if pruned else 1
|
|
|
|
|
|
class Qwen3VLDecoderLayer(Qwen3DecoderLayer):
|
|
def __init__(self, **kwargs) -> None:
|
|
factor = mrope_positions_factor(get_current_vllm_config())
|
|
super().__init__(**kwargs, mrope_positions_factor=factor)
|
|
|
|
|
|
@support_torch_compile(
|
|
dynamic_arg_dims={
|
|
"input_ids": 0,
|
|
# positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
|
|
# otherwise (seq_len, ).
|
|
"positions": -1,
|
|
"intermediate_tensors": 0,
|
|
"inputs_embeds": 0,
|
|
# the same shape as input_embeds
|
|
"deepstack_input_embeds": 0,
|
|
}
|
|
)
|
|
class Qwen3LLMModel(Qwen3Model):
|
|
decoder_layer_type = Qwen3VLDecoderLayer
|
|
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor | None,
|
|
positions: torch.Tensor,
|
|
intermediate_tensors: IntermediateTensors | None = None,
|
|
inputs_embeds: torch.Tensor | None = None,
|
|
# args for deepstack
|
|
deepstack_input_embeds: IntermediateTensors | None = None,
|
|
) -> torch.Tensor | IntermediateTensors:
|
|
if get_pp_group().is_first_rank:
|
|
if inputs_embeds is not None:
|
|
hidden_states = inputs_embeds
|
|
else:
|
|
hidden_states = self.embed_input_ids(input_ids)
|
|
residual = None
|
|
else:
|
|
assert intermediate_tensors is not None
|
|
hidden_states = intermediate_tensors["hidden_states"]
|
|
residual = intermediate_tensors["residual"]
|
|
|
|
aux_hidden_states = self._maybe_add_hidden_state([], 0, hidden_states, residual)
|
|
for layer_idx, layer in islice(
|
|
enumerate(self.layers), self.start_layer, self.end_layer
|
|
):
|
|
hidden_states, residual = layer(
|
|
positions,
|
|
hidden_states,
|
|
residual,
|
|
)
|
|
|
|
if deepstack_input_embeds is not None and layer_idx in range(
|
|
0, len(deepstack_input_embeds)
|
|
):
|
|
hidden_states = (
|
|
hidden_states
|
|
+ deepstack_input_embeds[f"deepstack_input_embeds_{layer_idx}"]
|
|
)
|
|
self._maybe_add_hidden_state(
|
|
aux_hidden_states, layer_idx + 1, hidden_states, residual
|
|
)
|
|
|
|
if not get_pp_group().is_last_rank:
|
|
return IntermediateTensors(
|
|
{"hidden_states": hidden_states, "residual": residual}
|
|
)
|
|
hidden_states, _ = self.norm(hidden_states, residual)
|
|
|
|
if len(aux_hidden_states) > 0:
|
|
return hidden_states, aux_hidden_states
|
|
return hidden_states
|
|
|
|
|
|
class Qwen3LLMForCausalLM(Qwen3ForCausalLM):
|
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
|
super(Qwen3ForCausalLM, self).__init__()
|
|
config = vllm_config.model_config.hf_config
|
|
quant_config = vllm_config.quant_config
|
|
|
|
self.config = config
|
|
|
|
self.quant_config = quant_config
|
|
self.model = Qwen3LLMModel(
|
|
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
|
)
|
|
|
|
if get_pp_group().is_last_rank:
|
|
self.lm_head = ParallelLMHead(
|
|
config.vocab_size,
|
|
config.hidden_size,
|
|
quant_config=quant_config,
|
|
prefix="lm_head",
|
|
)
|
|
if config.tie_word_embeddings:
|
|
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
|
else:
|
|
self.lm_head = PPMissingLayer()
|
|
|
|
self.logits_processor = LogitsProcessor(config.vocab_size)
|
|
|
|
self.make_empty_intermediate_tensors = (
|
|
self.model.make_empty_intermediate_tensors
|
|
)
|
|
|
|
|
|
@MULTIMODAL_REGISTRY.register_processor(
|
|
Qwen3VLMultiModalProcessor,
|
|
info=Qwen3VLProcessingInfo,
|
|
dummy_inputs=Qwen3VLDummyInputsBuilder,
|
|
)
|
|
class Qwen3VLForConditionalGeneration(
|
|
nn.Module,
|
|
SupportsMultiModal,
|
|
SupportsEncoderCudaGraph,
|
|
SupportsLoRA,
|
|
SupportsPP,
|
|
SupportsMRoPE,
|
|
SupportsEagle,
|
|
SupportsEagle3,
|
|
SupportsMultiModalPruning,
|
|
):
|
|
packed_modules_mapping: dict[str, list[str]] = {
|
|
"qkv_proj": [
|
|
"q_proj",
|
|
"k_proj",
|
|
"v_proj",
|
|
],
|
|
"gate_up_proj": [
|
|
"gate_proj",
|
|
"up_proj",
|
|
],
|
|
"qkv": ["qkv"], # For vision tower's already-packed QKV
|
|
}
|
|
|
|
supports_encoder_tp_data = True
|
|
supports_tower_connector_lora = True
|
|
supports_mm_device_do_normalize = True
|
|
|
|
supported_video_pruning_methods = ("evs", "vidcom2")
|
|
|
|
# To ensure correct weight loading and mapping.
|
|
hf_to_vllm_mapper = WeightsMapper(
|
|
orig_to_new_prefix={
|
|
"model.visual.": "visual.",
|
|
"lm_head.": "language_model.lm_head.",
|
|
"model.language_model.": "language_model.model.",
|
|
}
|
|
)
|
|
|
|
@classmethod
|
|
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
|
if modality.startswith("image"):
|
|
return "<|vision_start|><|image_pad|><|vision_end|>"
|
|
if modality.startswith("video"):
|
|
return "<|vision_start|><|video_pad|><|vision_end|>"
|
|
|
|
raise ValueError("Only image or video modality is supported")
|
|
|
|
def _init_video_pruning(self, multimodal_config: MultiModalConfig) -> None:
|
|
pruning_spec = multimodal_config.get_video_pruning_spec()
|
|
if pruning_spec is None:
|
|
self.video_pruning_method: VideoPruningMethod | None = None
|
|
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
|
else:
|
|
self.video_pruning_method, self.video_pruning_rate = pruning_spec
|
|
self.is_multimodal_pruning_enabled = (
|
|
multimodal_config.is_multimodal_pruning_enabled()
|
|
)
|
|
|
|
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
|
super().__init__()
|
|
config: Qwen3VLConfig = vllm_config.model_config.hf_config
|
|
quant_config = vllm_config.quant_config
|
|
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
|
|
|
self.config = config
|
|
self.model_config = vllm_config.model_config
|
|
self._tokenizer = cached_tokenizer_from_config(vllm_config.model_config)
|
|
self.multimodal_config = multimodal_config
|
|
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
|
self._init_video_pruning(multimodal_config)
|
|
|
|
with self._mark_tower_model(vllm_config, {"image", "video"}):
|
|
self.visual = Qwen3_VisionTransformer(
|
|
config.vision_config,
|
|
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
|
quant_config=quant_config,
|
|
input_norm=build_mm_input_norm(self.model_config),
|
|
prefix=maybe_prefix(prefix, "visual"),
|
|
)
|
|
|
|
self.use_deepstack = hasattr(
|
|
config.vision_config, "deepstack_visual_indexes"
|
|
) and not isinstance(self.visual, StageMissingLayer)
|
|
self.deepstack_num_level = (
|
|
len(config.vision_config.deepstack_visual_indexes)
|
|
if self.use_deepstack
|
|
else 0
|
|
)
|
|
self.visual_dim = config.vision_config.out_hidden_size
|
|
self.multiscale_dim = self.visual_dim * self.deepstack_num_level
|
|
|
|
if self.use_deepstack:
|
|
self.deepstack_input_embeds = [
|
|
torch.zeros(
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
config.text_config.hidden_size,
|
|
)
|
|
for _ in range(self.deepstack_num_level)
|
|
]
|
|
# Tracks the valid token span currently stored in the buffer.
|
|
# Zero means there is no active deepstack payload to consume.
|
|
self.deepstack_input_embeds_num_tokens = 0
|
|
|
|
with self._mark_language_model(vllm_config):
|
|
self.language_model = Qwen3LLMForCausalLM(
|
|
vllm_config=vllm_config.with_hf_config(
|
|
config.text_config,
|
|
architectures=["Qwen3ForCausalLM"],
|
|
),
|
|
prefix=maybe_prefix(prefix, "language_model"),
|
|
)
|
|
|
|
if not get_pp_group().is_first_rank and hasattr(
|
|
config.vision_config, "deepstack_visual_indexes"
|
|
):
|
|
assert self.language_model.model.start_layer >= len(
|
|
config.vision_config.deepstack_visual_indexes
|
|
), (
|
|
"start_layer should be greater than or equal to "
|
|
"len(deepstack_visual_indexes)"
|
|
)
|
|
|
|
self.make_empty_intermediate_tensors = (
|
|
self.language_model.make_empty_intermediate_tensors
|
|
)
|
|
|
|
def _get_deepstack_input_embeds(
|
|
self,
|
|
num_tokens: int,
|
|
) -> IntermediateTensors | None:
|
|
if not getattr(self, "deepstack_input_embeds", None):
|
|
return None # If vision tower is skipped
|
|
if num_tokens > self.deepstack_input_embeds[0].size(0):
|
|
self._resize_deepstack_input_embeds(num_tokens)
|
|
|
|
# get deepstack_input_embeds from buffer, and clear the buffer
|
|
return IntermediateTensors(
|
|
{
|
|
f"deepstack_input_embeds_{idx}": self.deepstack_input_embeds[idx][
|
|
:num_tokens
|
|
]
|
|
for idx in range(self.deepstack_num_level)
|
|
}
|
|
)
|
|
|
|
def _resize_deepstack_input_embeds(self, num_tokens: int) -> None:
|
|
self.deepstack_input_embeds = [
|
|
torch.zeros(
|
|
num_tokens,
|
|
self.config.text_config.hidden_size,
|
|
device=self.deepstack_input_embeds[0].device,
|
|
dtype=self.deepstack_input_embeds[0].dtype,
|
|
)
|
|
for _ in range(self.deepstack_num_level)
|
|
]
|
|
|
|
def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None:
|
|
if not getattr(self, "deepstack_input_embeds", None):
|
|
return
|
|
|
|
# set deepstack_input_embeds to buffer
|
|
num_tokens = deepstack_input_embeds.size(1)
|
|
if num_tokens > self.deepstack_input_embeds[0].size(0):
|
|
self._resize_deepstack_input_embeds(num_tokens)
|
|
for idx in range(self.deepstack_num_level):
|
|
self.deepstack_input_embeds[idx][:num_tokens].copy_(
|
|
deepstack_input_embeds[idx]
|
|
)
|
|
self.deepstack_input_embeds_num_tokens = num_tokens
|
|
|
|
def _clear_deepstack_input_embeds(self, num_tokens: int) -> None:
|
|
if not getattr(self, "deepstack_input_embeds", None):
|
|
return
|
|
if getattr(self, "deepstack_input_embeds_num_tokens", 0) == 0:
|
|
return
|
|
|
|
# clear deepstack_input_embeds in buffer
|
|
if num_tokens > 0:
|
|
for idx in range(self.deepstack_num_level):
|
|
self.deepstack_input_embeds[idx][:num_tokens].zero_()
|
|
self.deepstack_input_embeds_num_tokens = 0
|
|
|
|
# -- SupportsEncoderCudaGraph protocol methods --
|
|
|
|
def get_encoder_cudagraph_config(self):
|
|
from vllm.v1.worker.encoder_cudagraph_defs import (
|
|
EncoderCudaGraphConfig,
|
|
)
|
|
|
|
# When video pruning is enabled, embed_multimodal post-processes both
|
|
# image and video embeddings (mrope positions are appended for image,
|
|
# prune+append for video). The encoder CUDA graph path bypasses that
|
|
# post-process, producing inconsistent embedding formats vs eager. So
|
|
# disable CUDA graph for all modalities when pruning is on.
|
|
modalities = [] if self.is_multimodal_pruning_enabled else ["image", "video"]
|
|
|
|
# Compute max_frames_per_video for budget sizing.
|
|
max_frames = self.get_max_frames_per_video() if "video" in modalities else 1
|
|
|
|
return EncoderCudaGraphConfig(
|
|
modalities=modalities,
|
|
buffer_keys=[
|
|
"pixel_values",
|
|
"pos_embeds",
|
|
"rotary_pos_emb_cos",
|
|
"rotary_pos_emb_sin",
|
|
"cu_seqlens",
|
|
"max_seqlen",
|
|
"sequence_lengths",
|
|
],
|
|
out_hidden_size=self.visual.out_hidden_size,
|
|
max_frames_per_video=max_frames,
|
|
)
|
|
|
|
def get_input_modality(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
) -> str:
|
|
if "image_grid_thw" in mm_kwargs:
|
|
return "image"
|
|
elif "video_grid_thw" in mm_kwargs:
|
|
return "video"
|
|
raise AssertionError("This line should be unreachable.")
|
|
|
|
def get_max_frames_per_video(self) -> int:
|
|
mm_registry = MULTIMODAL_REGISTRY
|
|
info = mm_registry.get_processing_info(self.model_config)
|
|
assert isinstance(info, Qwen3VLProcessingInfo)
|
|
max_frames_per_video = info.get_num_frames_with_most_features(
|
|
seq_len=self.model_config.max_model_len,
|
|
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
|
|
)
|
|
return max_frames_per_video
|
|
|
|
def get_encoder_cudagraph_budget_range(
|
|
self,
|
|
vllm_config,
|
|
) -> tuple[int, int]:
|
|
# Min: estimated smallest possible encoder input.
|
|
# 224x224 image → 16x16 patches (patch_size=14)
|
|
# spatial_merge_size=2 → 8x8 = 64 tokens
|
|
min_budget = 64
|
|
# Max: capped by max_num_batched_tokens
|
|
# TODO(shen-shanshan): the max_budget auto-infer needs to be optimized later.
|
|
max_budget = min(
|
|
vllm_config.scheduler_config.max_num_batched_tokens,
|
|
self.model_config.max_model_len,
|
|
)
|
|
return (min_budget, max_budget)
|
|
|
|
def _get_pixel_values_by_modality(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
) -> torch.Tensor:
|
|
modality = self.get_input_modality(mm_kwargs)
|
|
if modality == "image":
|
|
return mm_kwargs["pixel_values"]
|
|
elif modality == "video":
|
|
return mm_kwargs["pixel_values_videos"]
|
|
raise AssertionError("This line should be unreachable.")
|
|
|
|
def _get_grid_thw_by_modality(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
) -> list[tuple[int, int, int]]:
|
|
grid_thw_key = f"{self.get_input_modality(mm_kwargs)}_grid_thw"
|
|
grid_thw = mm_kwargs[grid_thw_key]
|
|
if not isinstance(grid_thw, list):
|
|
grid_thw = grid_thw.tolist()
|
|
return grid_thw
|
|
|
|
def get_encoder_cudagraph_item_specs(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
):
|
|
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
|
|
|
|
m = self.visual.spatial_merge_size
|
|
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
|
|
return [
|
|
EncoderItemSpec(
|
|
input_size=t * h * w,
|
|
output_tokens=t * (h // m) * (w // m),
|
|
)
|
|
for t, h, w in grid_thw
|
|
]
|
|
|
|
def select_encoder_cudagraph_items(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
indices: list[int],
|
|
) -> dict[str, Any]:
|
|
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
|
|
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
|
|
|
|
if len(indices) == 0:
|
|
if self.get_input_modality(mm_kwargs) == "image":
|
|
return {
|
|
"pixel_values": pixel_values[:0],
|
|
"image_grid_thw": [],
|
|
}
|
|
else:
|
|
return {
|
|
"pixel_values_videos": pixel_values[:0],
|
|
"video_grid_thw": [],
|
|
}
|
|
|
|
# Compute cumulative patch offsets for slicing pixel_values
|
|
patches_per_item = [t * h * w for t, h, w in grid_thw]
|
|
cum_patches = [0]
|
|
for p in patches_per_item:
|
|
cum_patches.append(cum_patches[-1] + p)
|
|
|
|
selected_pv = torch.cat(
|
|
[pixel_values[cum_patches[i] : cum_patches[i + 1]] for i in indices]
|
|
)
|
|
selected_grid = [grid_thw[i] for i in indices]
|
|
|
|
if self.get_input_modality(mm_kwargs) == "image":
|
|
return {
|
|
"pixel_values": selected_pv,
|
|
"image_grid_thw": selected_grid,
|
|
}
|
|
else:
|
|
return {
|
|
"pixel_values_videos": selected_pv,
|
|
"video_grid_thw": selected_grid,
|
|
}
|
|
|
|
def prepare_encoder_cudagraph_capture_inputs(
|
|
self,
|
|
token_budget: int,
|
|
max_batch_size: int,
|
|
max_frames_per_batch: int,
|
|
device: torch.device,
|
|
dtype: torch.dtype,
|
|
path: str = "default",
|
|
axis_keys: tuple[Hashable, ...] | None = None,
|
|
):
|
|
from vllm.v1.worker.encoder_cudagraph_defs import (
|
|
EncoderCudaGraphCaptureInputs,
|
|
)
|
|
|
|
spatial_merge_size = self.visual.spatial_merge_size
|
|
# Ceil so the buffer fits the worst case of one item using the full
|
|
# budget. Floor under-allocates when budget is not a multiple of
|
|
# max_batch_size.
|
|
per_mm_item_output = (token_budget + max_batch_size - 1) // max_batch_size
|
|
|
|
frames_per_item = max_frames_per_batch // max_batch_size
|
|
if frames_per_item > 1:
|
|
# Build the capture grid using a video-format layout so that
|
|
# cu_seqlens is sized for video replays from the start.
|
|
# cu_seqlens has one entry per attention sequence (one per frame),
|
|
# so using T > 1 per item makes the buffer large enough without
|
|
# relying solely on padding.
|
|
# Ceiling ensures frames_per_item * tokens_per_frame >= per_mm_item_output
|
|
# so the pixel_values buffer covers any valid single-item replay.
|
|
tokens_per_frame = (
|
|
per_mm_item_output + frames_per_item - 1
|
|
) // frames_per_item
|
|
# Video-format grid_config (T=frames_per_item).
|
|
grid_config = [
|
|
[
|
|
frames_per_item,
|
|
spatial_merge_size,
|
|
tokens_per_frame * spatial_merge_size,
|
|
]
|
|
for _ in range(max_batch_size)
|
|
]
|
|
else:
|
|
# Image-format grid_config (T=1).
|
|
grid_config = [
|
|
[1, spatial_merge_size, per_mm_item_output * spatial_merge_size]
|
|
for _ in range(max_batch_size)
|
|
]
|
|
|
|
# Create dummy pixel_values
|
|
patch_embed = self.visual.patch_embed
|
|
in_channels = patch_embed.proj.in_channels
|
|
patch_size = patch_embed.patch_size
|
|
temporal_patch_size = patch_embed.temporal_patch_size
|
|
total_patches = sum(t * h * w for t, h, w in grid_config)
|
|
flattened_patch_size = (
|
|
in_channels * temporal_patch_size * patch_size * patch_size
|
|
)
|
|
dummy_pixel_values = torch.zeros(
|
|
total_patches,
|
|
flattened_patch_size,
|
|
device=device,
|
|
dtype=self.visual.input_norm.input_dtype or dtype,
|
|
)
|
|
|
|
# Override max_seqlen with a safe upper bound for capture.
|
|
# max_seqlen.item() gets baked into the CUDA graph (not replayed),
|
|
# so the capture value must cover any replay scenario.
|
|
# Worst case: 1 item consuming the full budget ->
|
|
# seq_len = token_budget * spatial_merge_size^2.
|
|
metadata = self.visual.prepare_encoder_metadata(
|
|
grid_config,
|
|
max_batch_size=max_batch_size,
|
|
max_frames_per_batch=max_frames_per_batch,
|
|
max_seqlen_override=token_budget * (spatial_merge_size**2),
|
|
device=device,
|
|
)
|
|
|
|
# Just use image-modality dummy input_buffer for capturing, since it's also
|
|
# compatible for video inputs (has the same shape: [num_patches, C*T*P*P]).
|
|
values = metadata | {
|
|
"pixel_values": dummy_pixel_values,
|
|
}
|
|
|
|
return EncoderCudaGraphCaptureInputs(
|
|
values=values,
|
|
)
|
|
|
|
def prepare_encoder_cudagraph_replay_buffers(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
max_batch_size: int,
|
|
max_frames_per_batch: int,
|
|
path: str = "default",
|
|
):
|
|
modality = self.get_input_modality(mm_kwargs)
|
|
grid_thw_list = self._get_grid_thw_by_modality(mm_kwargs)
|
|
|
|
if modality == "image":
|
|
metadata = self.visual.prepare_encoder_metadata(
|
|
grid_thw_list,
|
|
max_batch_size=max_batch_size,
|
|
)
|
|
elif modality == "video":
|
|
metadata = self.visual.prepare_encoder_metadata(
|
|
grid_thw_list,
|
|
max_frames_per_batch=max_frames_per_batch,
|
|
)
|
|
else:
|
|
raise AssertionError("This line should be unreachable.")
|
|
|
|
values = metadata | {
|
|
"pixel_values": self._get_pixel_values_by_modality(mm_kwargs),
|
|
}
|
|
return EncoderCudaGraphReplayBuffers(values=values)
|
|
|
|
def encoder_cudagraph_forward(
|
|
self,
|
|
values: dict[str, torch.Tensor],
|
|
path: str = "default",
|
|
) -> torch.Tensor:
|
|
pixel_values = values.pop("pixel_values")
|
|
metadata = values
|
|
return self.visual(pixel_values, None, encoder_metadata=metadata)
|
|
|
|
def encoder_eager_forward(
|
|
self,
|
|
mm_kwargs: dict[str, Any],
|
|
path: str = "default",
|
|
) -> torch.Tensor:
|
|
pixel_values = self._get_pixel_values_by_modality(mm_kwargs)
|
|
grid_thw = self._get_grid_thw_by_modality(mm_kwargs)
|
|
return self.visual(pixel_values, grid_thw)
|
|
|
|
def _parse_and_validate_image_input(
|
|
self, **kwargs: object
|
|
) -> Qwen2_5_VLImageInputs | None:
|
|
pixel_values = kwargs.pop("pixel_values", None)
|
|
image_embeds = kwargs.pop("image_embeds", None)
|
|
image_grid_thw = kwargs.pop("image_grid_thw", None)
|
|
|
|
if pixel_values is None and image_embeds is None:
|
|
return None
|
|
|
|
if pixel_values is not None:
|
|
return Qwen2_5_VLImagePixelInputs(
|
|
type="pixel_values",
|
|
pixel_values=pixel_values,
|
|
image_grid_thw=image_grid_thw,
|
|
)
|
|
|
|
else:
|
|
return Qwen2_5_VLImageEmbeddingInputs(
|
|
type="image_embeds",
|
|
image_embeds=image_embeds,
|
|
image_grid_thw=image_grid_thw,
|
|
)
|
|
|
|
def _parse_and_validate_video_input(
|
|
self, **kwargs: object
|
|
) -> Qwen2_5_VLVideoInputs | None:
|
|
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
|
video_embeds = kwargs.pop("video_embeds", None)
|
|
video_grid_thw = kwargs.pop("video_grid_thw", None)
|
|
second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
|
|
timestamps = kwargs.pop("timestamps", None)
|
|
|
|
if pixel_values_videos is None and video_embeds is None:
|
|
return None
|
|
|
|
if pixel_values_videos is not None:
|
|
return Qwen2_5_VLVideoPixelInputs(
|
|
type="pixel_values_videos",
|
|
pixel_values_videos=pixel_values_videos,
|
|
video_grid_thw=video_grid_thw,
|
|
second_per_grid_ts=second_per_grid_ts,
|
|
timestamps=timestamps,
|
|
)
|
|
|
|
else:
|
|
return Qwen2_5_VLVideoEmbeddingInputs(
|
|
type="video_embeds",
|
|
video_embeds=video_embeds,
|
|
video_grid_thw=video_grid_thw,
|
|
timestamps=timestamps,
|
|
)
|
|
|
|
def _process_image_input(
|
|
self, image_input: Qwen2_5_VLImageInputs
|
|
) -> tuple[torch.Tensor, ...]:
|
|
grid_thw = image_input["image_grid_thw"]
|
|
assert grid_thw.ndim == 2
|
|
|
|
if image_input["type"] == "image_embeds":
|
|
image_embeds = image_input["image_embeds"].type(self.visual.dtype)
|
|
else:
|
|
pixel_values = image_input["pixel_values"]
|
|
if self.use_data_parallel:
|
|
return run_dp_sharded_mrope_vision_model(
|
|
self.visual, pixel_values, grid_thw.tolist(), rope_type="rope_3d"
|
|
)
|
|
else:
|
|
image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
|
|
|
|
# Split concatenated embeddings for each image item.
|
|
merge_size = self.visual.spatial_merge_size
|
|
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
|
|
return image_embeds.split(sizes)
|
|
|
|
def _process_video_input(
|
|
self, video_input: Qwen2_5_VLVideoInputs
|
|
) -> tuple[torch.Tensor, ...]:
|
|
grid_thw = video_input["video_grid_thw"]
|
|
assert grid_thw.ndim == 2
|
|
|
|
if video_input["type"] == "video_embeds":
|
|
video_embeds = video_input["video_embeds"].type(self.visual.dtype)
|
|
else:
|
|
pixel_values_videos = video_input["pixel_values_videos"]
|
|
if self.use_data_parallel:
|
|
grid_thw_list = grid_thw.tolist()
|
|
return run_dp_sharded_mrope_vision_model(
|
|
self.visual, pixel_values_videos, grid_thw_list, rope_type="rope_3d"
|
|
)
|
|
else:
|
|
video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
|
|
|
|
# Split concatenated embeddings for each video item.
|
|
merge_size = self.visual.spatial_merge_size
|
|
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
|
|
return video_embeds.split(sizes)
|
|
|
|
def _postprocess_image_embeds_evs(
|
|
self,
|
|
image_embeds_split: tuple[torch.Tensor, ...],
|
|
image_input: Qwen2_5_VLImageInputs,
|
|
) -> tuple[torch.Tensor, ...]:
|
|
"""Append mrope positions for each for images.
|
|
This is necessary to recover correct mrope
|
|
positions after video pruning
|
|
|
|
Args:
|
|
image_embeds_split: Tuple of image embeddings for
|
|
each image item.
|
|
image_input: Image input data.
|
|
|
|
Returns:
|
|
Tuple of image embeddings for each image item.
|
|
Resulting embeddings will have extra 5 channels for
|
|
computed mrope positions, consistent with video embeddings.
|
|
|
|
"""
|
|
if self.is_multimodal_pruning_enabled:
|
|
merge_size = self.visual.spatial_merge_size
|
|
grid_thw = image_input["image_grid_thw"]
|
|
grid_thw_list = grid_thw.tolist()
|
|
image_embeds_out = []
|
|
for emb, size in zip(image_embeds_split, grid_thw_list):
|
|
positions = async_tensor_h2d(
|
|
compute_mrope_for_media(size, merge_size), emb.device
|
|
)
|
|
positions = torch.cat(
|
|
[
|
|
positions,
|
|
torch.zeros_like(
|
|
positions[:, 0:1]
|
|
), # Dummy extra fifth channel
|
|
],
|
|
dim=1,
|
|
)
|
|
emb = torch.cat([emb, positions], dim=1)
|
|
image_embeds_out.append(emb)
|
|
image_embeds_split = tuple(image_embeds_out)
|
|
return image_embeds_split
|
|
|
|
def _postprocess_video_embeds_evs(
|
|
self,
|
|
video_embeds_split: tuple[torch.Tensor, ...],
|
|
video_input: Qwen2_5_VLVideoInputs,
|
|
) -> tuple[torch.Tensor, ...]:
|
|
"""Prunes video embeddings via Efficient Video Sampling (EVS)
|
|
and then appends mrope positions for each retained embeddings
|
|
|
|
Args:
|
|
video_embeds_split: Tuple of video embeddings for each video item.
|
|
video_input: Video input data.
|
|
|
|
Returns:
|
|
Tuple of video embeddings for each video item.
|
|
Resulting embeddings will have extra 5 channels for computed mrope
|
|
positions, and whether the index corresponds to a video embedding.
|
|
|
|
"""
|
|
grid_thw = video_input["video_grid_thw"]
|
|
assert grid_thw.ndim == 2
|
|
grid_thw_list = grid_thw.tolist()
|
|
merge_size = self.visual.spatial_merge_size
|
|
|
|
# Apply EVS to each video.
|
|
video_embeds_out = []
|
|
assert video_input.timestamps is not None
|
|
for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)):
|
|
# Compute positions.
|
|
timestamps = video_input.timestamps[video_idx]
|
|
num_frames = len(timestamps)
|
|
|
|
t, h, w = size
|
|
if self.is_multimodal_pruning_enabled:
|
|
assert self.video_pruning_rate is not None
|
|
# Compute the retention mask for each video (EVS or VidCom2).
|
|
if self.video_pruning_method == "vidcom2":
|
|
mask_fn = vidcom2_compute_retention_mask
|
|
else:
|
|
mask_fn = compute_retention_mask
|
|
retention_mask = mask_fn(
|
|
emb,
|
|
size,
|
|
spatial_merge_size=self.visual.spatial_merge_size,
|
|
q=self.video_pruning_rate,
|
|
)
|
|
|
|
with gpu_sync_allowed():
|
|
# Apply retention mask.
|
|
emb = emb[retention_mask]
|
|
|
|
# Calculate the actual number of retained tokens per frame.
|
|
num_frames, rows, cols = t, h // merge_size, w // merge_size
|
|
retention_mask_thw = retention_mask.reshape(num_frames, rows, cols)
|
|
num_tokens_per_frame = (
|
|
retention_mask_thw.sum(dim=(1, 2)).long().tolist()
|
|
)
|
|
else:
|
|
feature_size = emb.shape[0] // num_frames
|
|
num_tokens_per_frame = [feature_size] * num_frames
|
|
retention_mask = None
|
|
|
|
emb = self._create_final_video_embeddings(
|
|
video_embeddings=emb,
|
|
num_tokens_per_frame=num_tokens_per_frame,
|
|
timestamps=timestamps,
|
|
video_grid_thw=size,
|
|
retention_mask=retention_mask,
|
|
)
|
|
|
|
video_embeds_out.append(emb)
|
|
|
|
return tuple(video_embeds_out)
|
|
|
|
def _create_final_video_embeddings(
|
|
self,
|
|
video_embeddings: torch.Tensor,
|
|
num_tokens_per_frame: list[int],
|
|
timestamps: list[float],
|
|
video_grid_thw: list[int],
|
|
retention_mask: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
"""Create final embeddings that combine video embeddings with
|
|
text embeddings of indicator tokens.
|
|
|
|
These final embeddings contain:
|
|
- Actual video embeddings in positions corresponding to video content
|
|
- Text embeddings for indicator tokens (<img>, </img>, and
|
|
frame separation text) in their respective positions
|
|
|
|
These embeddings will replace the placeholder embeddings to create
|
|
input_embeds for the LLM.
|
|
"""
|
|
device = video_embeddings.device
|
|
|
|
# Generate video replacement token IDs using get_video_repl
|
|
# This tokenizes each frame separator independently, then uses pre-tokenized
|
|
# special tokens to ensure consistent tokenization regardless of
|
|
# num_tokens_per_frame values.
|
|
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
|
|
tokens_per_frame=num_tokens_per_frame,
|
|
tokenizer=self._tokenizer,
|
|
timestamps=timestamps,
|
|
vision_start_token_id=self.config.vision_start_token_id,
|
|
vision_end_token_id=self.config.vision_end_token_id,
|
|
video_token_id=self.config.video_token_id,
|
|
select_token_id=self.is_multimodal_pruning_enabled,
|
|
)
|
|
|
|
repl_token_ids = torch.tensor(video_repl.full, device=device)
|
|
embed_token_id = _cached_tensor(self.config.video_token_id, device=device)
|
|
is_video_embed = torch.isin(repl_token_ids, embed_token_id)
|
|
|
|
# Get text embeddings for indicator tokens (has only `visual_dim``).
|
|
text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids)
|
|
|
|
if self.use_deepstack:
|
|
(
|
|
deepstack_input_embeds,
|
|
multimodal_embeddings,
|
|
) = self._compute_deepstack_embeds(
|
|
inputs_embeds=text_embeddings,
|
|
multimodal_embeddings=[video_embeddings],
|
|
is_multimodal=is_video_embed,
|
|
)
|
|
else:
|
|
deepstack_input_embeds = None
|
|
multimodal_embeddings = [video_embeddings]
|
|
|
|
merged_embeddings = _merge_multimodal_embeddings(
|
|
inputs_embeds=text_embeddings,
|
|
multimodal_embeddings=multimodal_embeddings,
|
|
is_multimodal=is_video_embed,
|
|
)
|
|
|
|
to_concat = [merged_embeddings]
|
|
if deepstack_input_embeds is not None:
|
|
to_concat.append(
|
|
deepstack_input_embeds.permute(1, 0, 2).reshape(
|
|
deepstack_input_embeds.shape[1], -1
|
|
)
|
|
)
|
|
|
|
expanded_positions = None
|
|
if self.is_multimodal_pruning_enabled:
|
|
is_vision_start = repl_token_ids.eq(self.config.vision_start_token_id)
|
|
expanded_positions = self._get_expanded_positions(
|
|
device=merged_embeddings.device,
|
|
seq_len=merged_embeddings.shape[0],
|
|
video_grid_thw=video_grid_thw,
|
|
num_tokens_per_frame=num_tokens_per_frame,
|
|
timestamps=timestamps,
|
|
is_video_embed=is_video_embed,
|
|
is_vision_start=is_vision_start,
|
|
retention_mask=retention_mask,
|
|
)
|
|
to_concat.append(expanded_positions)
|
|
|
|
final_video_embeddings = torch.cat(to_concat, dim=-1)
|
|
|
|
return final_video_embeddings
|
|
|
|
def _get_expanded_positions(
|
|
self,
|
|
device,
|
|
seq_len,
|
|
video_grid_thw,
|
|
num_tokens_per_frame,
|
|
timestamps,
|
|
is_video_embed,
|
|
is_vision_start,
|
|
retention_mask,
|
|
):
|
|
embed_token_id = _cached_tensor(self.config.video_token_id, device=device)
|
|
|
|
# Expand positions to match the full sequence length
|
|
# (includes both video tokens and indicator tokens)
|
|
# Shape: [full_length, 5] where positions are filled for video tokens
|
|
# and zeros for indicator tokens.
|
|
# Channel 3 flags VISION_START tokens so that
|
|
# recompute_mrope_positions can reliably count timestamp tokens
|
|
# (even when early frames have all video tokens pruned).
|
|
# Channel 4 flags video-embedding tokens.
|
|
expanded_positions = torch.zeros(
|
|
seq_len,
|
|
5, # [t_index, h_index, w_index, is_vision_start, is_video]
|
|
device=device,
|
|
dtype=torch.long,
|
|
)
|
|
_, h, w = video_grid_thw
|
|
merge_size = self.visual.spatial_merge_size
|
|
num_frames = len(num_tokens_per_frame)
|
|
unpruned_token_ids = Qwen3VLMultiModalProcessor.get_video_repl(
|
|
tokens_per_frame=[(h // merge_size) * (w // merge_size)] * num_frames,
|
|
tokenizer=self._tokenizer,
|
|
timestamps=timestamps,
|
|
vision_start_token_id=self.config.vision_start_token_id,
|
|
vision_end_token_id=self.config.vision_end_token_id,
|
|
video_token_id=self.config.video_token_id,
|
|
).full
|
|
unpruned_token_ids_tensor = torch.tensor(unpruned_token_ids, device=device)
|
|
mm_feature = MultiModalFeatureSpec(
|
|
data=MultiModalKwargsItem(
|
|
{
|
|
"video_grid_thw": MultiModalFieldElem(
|
|
data=torch.tensor(video_grid_thw),
|
|
field=MultiModalFieldConfig.batched("video").field,
|
|
),
|
|
}
|
|
),
|
|
modality="video",
|
|
identifier="DUMMY",
|
|
mm_position=PlaceholderRange(offset=0, length=len(unpruned_token_ids)),
|
|
)
|
|
original_mrope_cpu = self.get_mrope_input_positions(
|
|
input_tokens=unpruned_token_ids,
|
|
mm_features=[mm_feature],
|
|
)[0]
|
|
original_mrope = async_tensor_h2d(original_mrope_cpu, device).permute(1, 0)
|
|
full_is_video_embed = unpruned_token_ids_tensor == embed_token_id
|
|
|
|
with gpu_sync_allowed():
|
|
expanded_positions[is_video_embed, :3] = original_mrope[
|
|
full_is_video_embed
|
|
][retention_mask]
|
|
expanded_positions[~is_video_embed, :3] = original_mrope[
|
|
~full_is_video_embed
|
|
]
|
|
expanded_positions[..., 3] = is_vision_start
|
|
expanded_positions[..., 4] = is_video_embed
|
|
|
|
return expanded_positions
|
|
|
|
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
|
mm_input_by_modality: dict[
|
|
str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
|
|
] = {}
|
|
for input_key in kwargs:
|
|
if (
|
|
input_key in ("pixel_values", "image_embeds")
|
|
and "image" not in mm_input_by_modality
|
|
):
|
|
mm_input_by_modality["image"] = self._parse_and_validate_image_input(
|
|
**kwargs
|
|
)
|
|
if (
|
|
input_key in ("pixel_values_videos", "video_embeds")
|
|
and "video" not in mm_input_by_modality
|
|
):
|
|
mm_input_by_modality["video"] = self._parse_and_validate_video_input(
|
|
**kwargs
|
|
)
|
|
return mm_input_by_modality
|
|
|
|
@staticmethod
|
|
def _iter_mm_grid_hw(
|
|
input_tokens: list[int],
|
|
mm_features: list[MultiModalFeatureSpec],
|
|
video_token_id: int,
|
|
vision_start_token_id: int,
|
|
vision_end_token_id: int,
|
|
spatial_merge_size: int,
|
|
) -> Iterator[tuple[int, int, int, int]]:
|
|
"""Iterate over multimodal features and yield position info.
|
|
|
|
Args:
|
|
input_tokens: List of token IDs in the input sequence.
|
|
mm_features: List of multimodal feature specifications containing
|
|
image/video data and position information.
|
|
video_token_id: Token ID used for video tokens.
|
|
vision_start_token_id: Token ID marking the start of a vision sequence.
|
|
vision_end_token_id: Token ID marking the end of a vision sequence.
|
|
spatial_merge_size: Size of the spatial merge operation used to
|
|
compute logical grid dimensions from the original feature grid.
|
|
|
|
Yields:
|
|
offset: Position of the first video/image token in the sequence.
|
|
llm_grid_h: Logical grid height (may not match actual token count with EVS).
|
|
llm_grid_w: Logical grid width (may not match actual token count with EVS).
|
|
actual_num_tokens: Actual number of video/image tokens in the placeholder.
|
|
|
|
"""
|
|
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
|
offset = mm_feature.mm_position.offset
|
|
data = mm_feature.data
|
|
assert data is not None
|
|
if mm_feature.modality == "image":
|
|
grid = data["image_grid_thw"].data
|
|
assert isinstance(grid, torch.Tensor)
|
|
t, h, w = grid.tolist()
|
|
assert t == 1, f"Image must have 1 frame, got {t}"
|
|
llm_grid_h = h // spatial_merge_size
|
|
llm_grid_w = w // spatial_merge_size
|
|
yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w
|
|
elif mm_feature.modality == "video":
|
|
grid = data["video_grid_thw"].data
|
|
assert isinstance(grid, torch.Tensor)
|
|
t, h, w = grid.tolist()
|
|
llm_grid_h = h // spatial_merge_size
|
|
llm_grid_w = w // spatial_merge_size
|
|
|
|
for _ in range(t):
|
|
# When EVS is enabled, some frames may have 0 video tokens in the
|
|
# placeholder. We use `vision_start_token_id` to locate each frame
|
|
# since it is always present for every frame.
|
|
# We then look for the first `video_token_id` after
|
|
# `vision_start_token_id` and before `vision_end_token_id`.
|
|
offset = input_tokens.index(vision_start_token_id, offset)
|
|
vision_end_offset = input_tokens.index(vision_end_token_id, offset)
|
|
|
|
try:
|
|
actual_num_tokens = 0
|
|
video_offset = input_tokens.index(
|
|
video_token_id, offset, vision_end_offset
|
|
)
|
|
# NOTE: looking at the
|
|
# `Qwen3VLMultiModalProcessor.get_video_repl` code, we can
|
|
# see that we can use the below formula to get the token
|
|
# count, since everything in between `video_offset` and
|
|
# `vision_end_offset` is populated as `video_token_id`.
|
|
# This saves us from manually counting the number tokens
|
|
# that match `video_token_id` in between.
|
|
actual_num_tokens += vision_end_offset - video_offset
|
|
except ValueError:
|
|
# No `video_token_id` in this frame (EVS with 0 tokens for
|
|
# this frame) -> use `offset + 1`` to move past
|
|
# `vision_start_token_id`.
|
|
video_offset = offset + 1
|
|
|
|
yield video_offset, llm_grid_h, llm_grid_w, actual_num_tokens
|
|
# Move offset past this frame for next iteration.
|
|
offset = vision_end_offset + 1
|
|
else:
|
|
raise ValueError(f"Unsupported modality: {mm_feature.modality}")
|
|
|
|
def get_mrope_input_positions(
|
|
self,
|
|
input_tokens: list[int],
|
|
mm_features: list[MultiModalFeatureSpec],
|
|
) -> tuple[torch.Tensor, int]:
|
|
return self._get_mrope_input_positions(
|
|
input_tokens=input_tokens,
|
|
mm_features=mm_features,
|
|
config=self.config,
|
|
)
|
|
|
|
@staticmethod
|
|
def _get_mrope_input_positions(
|
|
input_tokens: list[int],
|
|
mm_features: list[MultiModalFeatureSpec],
|
|
config: Qwen3VLConfig,
|
|
):
|
|
llm_pos_ids_list: list[np.ndarray] = []
|
|
st = 0
|
|
for (
|
|
offset,
|
|
llm_grid_h,
|
|
llm_grid_w,
|
|
actual_num_tokens,
|
|
) in Qwen3VLForConditionalGeneration._iter_mm_grid_hw(
|
|
input_tokens,
|
|
mm_features,
|
|
video_token_id=config.video_token_id,
|
|
vision_start_token_id=config.vision_start_token_id,
|
|
vision_end_token_id=config.vision_end_token_id,
|
|
spatial_merge_size=config.vision_config.spatial_merge_size,
|
|
):
|
|
# Skip frames with 0 tokens (EVS placeholder with tokens lumped elsewhere)
|
|
if actual_num_tokens == 0:
|
|
continue
|
|
|
|
text_len = offset - st
|
|
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
|
|
llm_pos_ids_list.append(
|
|
np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
|
|
)
|
|
|
|
# Check if this is a "lumped placeholder" (all tokens from multiple frames
|
|
# assigned to the 0-th frame - see
|
|
# `Qwen3VLMultiModalProcessor.get_video_repl`.
|
|
expected_tokens_per_frame = llm_grid_h * llm_grid_w
|
|
if actual_num_tokens > expected_tokens_per_frame:
|
|
# Lumped placeholder: create grid positions for all "logical" frames
|
|
# represented.
|
|
num_logical_frames = actual_num_tokens // expected_tokens_per_frame
|
|
remainder = actual_num_tokens % expected_tokens_per_frame
|
|
|
|
# Create positions for complete frames.
|
|
for _ in range(num_logical_frames):
|
|
grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(
|
|
3, -1
|
|
)
|
|
llm_pos_ids_list.append(grid_indices + text_len + st_idx)
|
|
st_idx = llm_pos_ids_list[-1].max() + 1
|
|
text_len = 0 # No text between frames within the lump
|
|
|
|
# Handle remainder tokens if any (partial frame).
|
|
# NOTE: this should never be the case. Should we have an assert?
|
|
if remainder > 0:
|
|
# Create a partial grid - take first 'remainder' positions
|
|
full_grid = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)
|
|
grid_indices = full_grid[:, :remainder]
|
|
llm_pos_ids_list.append(grid_indices + text_len + st_idx)
|
|
else:
|
|
# Normal case: frame has exactly the expected tokens (after actual EVS
|
|
# pruning).
|
|
grid_indices = np.indices((1, llm_grid_h, llm_grid_w)).reshape(3, -1)
|
|
llm_pos_ids_list.append(grid_indices + text_len + st_idx)
|
|
|
|
st = offset + actual_num_tokens
|
|
|
|
if st < len(input_tokens):
|
|
st_idx = llm_pos_ids_list[-1].max() + 1 if len(llm_pos_ids_list) > 0 else 0
|
|
text_len = len(input_tokens) - st
|
|
llm_pos_ids_list.append(
|
|
np.broadcast_to(np.arange(text_len), (3, text_len)) + st_idx
|
|
)
|
|
|
|
llm_positions = np.concatenate(llm_pos_ids_list, axis=1).reshape(3, -1)
|
|
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
|
|
return torch.from_numpy(llm_positions), mrope_position_delta
|
|
|
|
def recompute_mrope_positions(
|
|
self,
|
|
input_ids: list[int] | torch.Tensor,
|
|
multimodal_embeddings: Sequence[torch.Tensor],
|
|
mrope_positions: torch.LongTensor,
|
|
num_computed_tokens: int,
|
|
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
|
|
"""Update part of input mrope positions (starting with
|
|
num_computed_tokens index). Original mrope_positions are computed
|
|
for unpruned sequence and becomes incorrect once pruning occurs,
|
|
so once we prune media tokens we should reflect this in the
|
|
mrope_positions before we feed it to LLM.
|
|
|
|
Args:
|
|
input_ids: (N,) All input tokens of the prompt containing
|
|
entire sequence.
|
|
multimodal_embeddings: Sequence of multimodal embeddings that
|
|
fits into the prefill chunk that is being processed.
|
|
mrope_positions: Existing mrope positions (3, N) for entire
|
|
sequence
|
|
num_computed_tokens: A number of computed tokens so far.
|
|
|
|
Returns:
|
|
Tuple of (multimodal_embeddings, mrope_positions,
|
|
mrope_position_delta).
|
|
|
|
"""
|
|
return self._recompute_mrope_positions(
|
|
input_ids=input_ids,
|
|
multimodal_embeddings=multimodal_embeddings,
|
|
mrope_positions=mrope_positions,
|
|
num_computed_tokens=num_computed_tokens,
|
|
image_token_id=self.config.image_token_id,
|
|
video_token_id=self.config.video_token_id,
|
|
vision_start_token_id=self.config.vision_start_token_id,
|
|
)
|
|
|
|
@staticmethod
|
|
def _recompute_mrope_positions(
|
|
input_ids: list[int] | torch.Tensor,
|
|
multimodal_embeddings: Sequence[torch.Tensor],
|
|
mrope_positions: torch.LongTensor,
|
|
num_computed_tokens: int,
|
|
vision_start_token_id: int,
|
|
image_token_id: int,
|
|
video_token_id: int,
|
|
) -> tuple[Sequence[torch.Tensor], torch.Tensor, int]:
|
|
# Device
|
|
device = (
|
|
multimodal_embeddings[0].device
|
|
if len(multimodal_embeddings)
|
|
else mrope_positions.device
|
|
)
|
|
|
|
# Tensors. input_ids may already be a (device-side) tensor.
|
|
if isinstance(input_ids, torch.Tensor):
|
|
assert input_ids.device == device
|
|
input_ids_t = input_ids.to(dtype=torch.long)
|
|
else:
|
|
input_ids_t = async_tensor_h2d(input_ids, device=device, dtype=torch.long)
|
|
|
|
mm_embeddings_out = []
|
|
mm_embeddings_pos = []
|
|
# Strip position information from embeddings (last 5 channels)
|
|
# For Qwen3 VL, handle potentially empty frames (from unpacking)
|
|
for mm in multimodal_embeddings:
|
|
if mm.shape[0] > 0: # Only process non-empty frames
|
|
mm_embeddings_out.append(mm[:, :-5])
|
|
mm_embeddings_pos.append(mm[:, -5:].permute(1, 0).long())
|
|
else:
|
|
# Empty frame - keep as is
|
|
mm_embeddings_out.append(mm)
|
|
# Create empty position tensor with correct shape
|
|
mm_embeddings_pos.append(
|
|
torch.empty(5, 0, device=device, dtype=torch.long)
|
|
)
|
|
|
|
with gpu_sync_allowed():
|
|
positions, mrope_positions_delta = recompute_mrope_positions(
|
|
input_ids_t,
|
|
mm_embeddings_pos,
|
|
mrope_positions,
|
|
num_computed_tokens,
|
|
vision_start_token_id,
|
|
image_token_id,
|
|
video_token_id,
|
|
)
|
|
|
|
return mm_embeddings_out, positions, mrope_positions_delta
|
|
|
|
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings | None:
|
|
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
|
|
if not mm_input_by_modality:
|
|
return None
|
|
|
|
# The result multimodal_embeddings is tuple of tensors, with each
|
|
# tensor corresponding to a multimodal data item (image or video).
|
|
multimodal_embeddings: list[torch.Tensor] = []
|
|
|
|
# NOTE: It is important to iterate over the keys in this dictionary
|
|
# to preserve the order of the modalities.
|
|
for modality in mm_input_by_modality:
|
|
multimodal_input = mm_input_by_modality[modality]
|
|
if modality == "image":
|
|
image_embeddings = self._process_image_input(multimodal_input)
|
|
image_embeddings = self._postprocess_image_embeds_evs(
|
|
image_embeddings, multimodal_input
|
|
)
|
|
multimodal_embeddings.extend(image_embeddings)
|
|
if modality == "video":
|
|
video_embeddings = self._process_video_input(multimodal_input)
|
|
if self.is_multimodal_pruning_enabled:
|
|
video_embeddings = self._postprocess_video_embeds_evs(
|
|
video_embeddings, multimodal_input
|
|
)
|
|
multimodal_embeddings.extend(video_embeddings)
|
|
|
|
embeddings_tuple = tuple(multimodal_embeddings)
|
|
return embeddings_tuple
|
|
|
|
def _compute_deepstack_embeds(
|
|
self,
|
|
inputs_embeds: torch.Tensor,
|
|
multimodal_embeddings: MultiModalEmbeddings,
|
|
is_multimodal: torch.Tensor,
|
|
) -> tuple[torch.Tensor, MultiModalEmbeddings]:
|
|
visual_lens = [len(x) for x in multimodal_embeddings]
|
|
multimodal_embeddings_cat = torch.cat(multimodal_embeddings, dim=0)
|
|
|
|
(
|
|
multimodal_embeddings_main,
|
|
multimodal_embeddings_multiscale,
|
|
) = torch.split(
|
|
multimodal_embeddings_cat,
|
|
[self.visual_dim, self.multiscale_dim],
|
|
dim=-1,
|
|
)
|
|
|
|
multimodal_embeddings = torch.split(
|
|
multimodal_embeddings_main, visual_lens, dim=0
|
|
)
|
|
multimodal_embeddings_multiscale = torch.split(
|
|
multimodal_embeddings_multiscale, visual_lens, dim=0
|
|
)
|
|
|
|
deepstack_input_embeds = inputs_embeds.new_zeros(
|
|
inputs_embeds.size(0), self.deepstack_num_level * inputs_embeds.size(1)
|
|
)
|
|
|
|
deepstack_input_embeds = _merge_multimodal_embeddings(
|
|
inputs_embeds=deepstack_input_embeds,
|
|
multimodal_embeddings=multimodal_embeddings_multiscale,
|
|
is_multimodal=is_multimodal,
|
|
)
|
|
deepstack_input_embeds = deepstack_input_embeds.view(
|
|
inputs_embeds.shape[0], self.deepstack_num_level, self.visual_dim
|
|
)
|
|
deepstack_input_embeds = deepstack_input_embeds.permute(1, 0, 2)
|
|
|
|
return deepstack_input_embeds, multimodal_embeddings
|
|
|
|
def embed_input_ids(
|
|
self,
|
|
input_ids: torch.Tensor,
|
|
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
|
*,
|
|
is_multimodal: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
inputs_embeds = self._embed_text_input_ids(
|
|
input_ids,
|
|
self.language_model.embed_input_ids,
|
|
is_multimodal=is_multimodal,
|
|
)
|
|
|
|
if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
|
|
return inputs_embeds
|
|
|
|
is_multimodal = _require_is_multimodal(is_multimodal)
|
|
|
|
if self.use_deepstack:
|
|
(
|
|
deepstack_input_embeds,
|
|
multimodal_embeddings,
|
|
) = self._compute_deepstack_embeds(
|
|
inputs_embeds=inputs_embeds,
|
|
multimodal_embeddings=multimodal_embeddings,
|
|
is_multimodal=is_multimodal,
|
|
)
|
|
else:
|
|
deepstack_input_embeds = None
|
|
|
|
inputs_embeds = _merge_multimodal_embeddings(
|
|
inputs_embeds=inputs_embeds,
|
|
multimodal_embeddings=multimodal_embeddings,
|
|
is_multimodal=is_multimodal,
|
|
)
|
|
|
|
if deepstack_input_embeds is not None:
|
|
self._set_deepstack_input_embeds(deepstack_input_embeds)
|
|
|
|
return inputs_embeds
|
|
|
|
def forward(
|
|
self,
|
|
input_ids: torch.Tensor | None,
|
|
positions: torch.Tensor,
|
|
intermediate_tensors: IntermediateTensors | None = None,
|
|
inputs_embeds: torch.Tensor | None = None,
|
|
**kwargs: object,
|
|
) -> torch.Tensor | IntermediateTensors:
|
|
"""Run forward pass for Qwen3VL.
|
|
|
|
Args:
|
|
input_ids: Flattened (concatenated) input_ids corresponding to a
|
|
batch.
|
|
positions: Flattened (concatenated) position ids corresponding to a
|
|
batch.
|
|
**NOTE**: If mrope is enabled (default setting for Qwen3VL
|
|
opensource models), the shape will be `(3, seq_len)`,
|
|
otherwise it will be `(seq_len,).
|
|
intermediate_tensors: Intermediate tensors from previous pipeline
|
|
stages.
|
|
inputs_embeds: Pre-computed input embeddings.
|
|
**kwargs: Additional keyword arguments including:
|
|
- pixel_values: Pixel values to be fed to a model.
|
|
`None` if no images are passed.
|
|
- image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in
|
|
LLM. `None` if no images are passed.
|
|
- pixel_values_videos: Pixel values of videos to be fed to a
|
|
model. `None` if no videos are passed.
|
|
- video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in
|
|
LLM. `None` if no videos are passed.
|
|
|
|
"""
|
|
if intermediate_tensors is not None:
|
|
inputs_embeds = None
|
|
|
|
if inputs_embeds is not None and get_pp_group().is_first_rank:
|
|
deepstack_input_embeds = self._get_deepstack_input_embeds(
|
|
inputs_embeds.size(0)
|
|
)
|
|
else:
|
|
deepstack_input_embeds = None
|
|
|
|
hidden_states = self.language_model.model(
|
|
input_ids=input_ids,
|
|
positions=positions,
|
|
intermediate_tensors=intermediate_tensors,
|
|
inputs_embeds=inputs_embeds,
|
|
# args for deepstack
|
|
deepstack_input_embeds=deepstack_input_embeds,
|
|
)
|
|
|
|
if inputs_embeds is not None and get_pp_group().is_first_rank:
|
|
self._clear_deepstack_input_embeds(inputs_embeds.size(0))
|
|
|
|
return hidden_states
|
|
|
|
def compute_logits(
|
|
self,
|
|
hidden_states: torch.Tensor,
|
|
) -> torch.Tensor | None:
|
|
return self.language_model.compute_logits(hidden_states)
|
|
|
|
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
|
loader = AutoWeightsLoader(self)
|
|
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
|
|
|
def get_mm_mapping(self) -> MultiModelKeys:
|
|
"""Get the module prefix in multimodal models."""
|
|
return MultiModelKeys.from_string_field(
|
|
language_model="language_model",
|
|
connector=["visual.merger", "visual.deepstack_merger_list"],
|
|
tower_model="visual.",
|
|
)
|
|
|
|
def get_mm_lora_token_counts(
|
|
self,
|
|
*,
|
|
modality: str,
|
|
mm_kwargs: MultiModalKwargsItem | None,
|
|
num_mm_embeds: int,
|
|
) -> tuple[int, int | None]:
|
|
del modality, mm_kwargs
|
|
hf_config = self.config
|
|
vision_config = hf_config.vision_config
|
|
merge_size = vision_config.spatial_merge_size
|
|
return num_mm_embeds * merge_size**2, num_mm_embeds
|
|
|
|
|
|
@lru_cache
|
|
def _cached_tensor(x, device) -> torch.Tensor:
|
|
return torch.tensor(x, device=device)
|