[Bugfix][Qwen2.5-VL] Honor video fps for temporal M-RoPE (#47736)

Signed-off-by: Ting Sun <suntcrick@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ting SUN
2026-09-22 05:56:12 +00:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5abf85d27a
commit a33b3bac5d
7 changed files with 117 additions and 8 deletions
+67 -1
View File
@@ -41,12 +41,14 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
Qwen2_5_VLConfig,
Qwen2_5_VLVisionConfig,
)
from transformers.video_utils import VideoMetadata
from vllm.compilation.decorators import (
should_torch_compile_mm_encoder,
support_torch_compile,
)
from vllm.config import VllmConfig
from vllm.config.multimodal import VideoDummyOptions
from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils
from vllm.logger import init_logger
@@ -72,9 +74,11 @@ from vllm.multimodal.inputs import (
MultiModalFieldConfig,
MultiModalKwargsItem,
MultiModalKwargsItems,
VideoItem,
)
from vllm.multimodal.parse import MultiModalDataItems
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
from vllm.multimodal.processing.processor import HFMultiModalInputs
from vllm.multimodal.video_prune.evs import (
compute_mrope_for_media,
compute_retained_tokens_count,
@@ -102,8 +106,9 @@ from .interfaces import (
SupportsPP,
SupportsQuant,
)
from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
from .qwen2_vl import (
Qwen2VLDummyInputsBuilder,
Qwen2VLMultiModalDataParser,
Qwen2VLMultiModalProcessor,
Qwen2VLProcessingInfo,
)
@@ -1137,6 +1142,40 @@ class Qwen2_5_VisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
def _get_dummy_videos(
self,
*,
width: int,
height: int,
num_frames: int,
num_videos: int,
overrides: VideoDummyOptions | None = None,
) -> list[VideoItem]:
videos = super()._get_dummy_videos(
width=width,
height=height,
num_frames=num_frames,
num_videos=num_videos,
overrides=overrides,
)
videos = [v.copy() for v in videos]
video_items = []
for video in videos:
video_num_frames = video.shape[0]
video_metadata = {
"fps": 2.0,
"duration": video_num_frames / 2.0,
"total_num_frames": video_num_frames,
"frames_indices": list(range(video_num_frames)),
"video_backend": "opencv",
"do_sample_frames": False,
}
video_items.append((video, video_metadata))
return video_items
class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(Qwen2_5_VLConfig)
@@ -1148,6 +1187,14 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
**kwargs,
)
def get_data_parser(self):
return Qwen2VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
)
class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
def _get_mm_fields_config(
@@ -1160,6 +1207,25 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
)
def _get_hf_mm_inputs(
self,
mm_items: MultiModalDataItems,
hf_kwargs: Mapping[str, object],
) -> HFMultiModalInputs:
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
hf_data = hf_inputs.hf_data
if videos := hf_data.get("videos"):
# HF expects video metadata as a separate argument.
hf_data["videos"] = [video for video, _ in videos]
hf_data["video_metadata"] = [
VideoMetadata(
**{k: metadata[k] for k in metadata if k != "do_sample_frames"}
)
for _, metadata in videos
]
return hf_inputs
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,