[Bugfix][Qwen2.5-VL] Honor video fps for temporal M-RoPE (#47736)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5abf85d27a
commit
a33b3bac5d
@@ -41,12 +41,14 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
|
||||
Qwen2_5_VLConfig,
|
||||
Qwen2_5_VLVisionConfig,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.compilation.decorators import (
|
||||
should_torch_compile_mm_encoder,
|
||||
support_torch_compile,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import VideoDummyOptions
|
||||
from vllm.distributed import parallel_state
|
||||
from vllm.distributed import utils as dist_utils
|
||||
from vllm.logger import init_logger
|
||||
@@ -72,9 +74,11 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalFieldConfig,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalKwargsItems,
|
||||
VideoItem,
|
||||
)
|
||||
from vllm.multimodal.parse import MultiModalDataItems
|
||||
from vllm.multimodal.processing import PromptReplacement, PromptUpdate
|
||||
from vllm.multimodal.processing.processor import HFMultiModalInputs
|
||||
from vllm.multimodal.video_prune.evs import (
|
||||
compute_mrope_for_media,
|
||||
compute_retained_tokens_count,
|
||||
@@ -102,8 +106,9 @@ from .interfaces import (
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
|
||||
from .qwen2_vl import (
|
||||
Qwen2VLDummyInputsBuilder,
|
||||
Qwen2VLMultiModalDataParser,
|
||||
Qwen2VLMultiModalProcessor,
|
||||
Qwen2VLProcessingInfo,
|
||||
)
|
||||
@@ -1137,6 +1142,40 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
|
||||
class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
|
||||
def _get_dummy_videos(
|
||||
self,
|
||||
*,
|
||||
width: int,
|
||||
height: int,
|
||||
num_frames: int,
|
||||
num_videos: int,
|
||||
overrides: VideoDummyOptions | None = None,
|
||||
) -> list[VideoItem]:
|
||||
videos = super()._get_dummy_videos(
|
||||
width=width,
|
||||
height=height,
|
||||
num_frames=num_frames,
|
||||
num_videos=num_videos,
|
||||
overrides=overrides,
|
||||
)
|
||||
videos = [v.copy() for v in videos]
|
||||
|
||||
video_items = []
|
||||
for video in videos:
|
||||
video_num_frames = video.shape[0]
|
||||
video_metadata = {
|
||||
"fps": 2.0,
|
||||
"duration": video_num_frames / 2.0,
|
||||
"total_num_frames": video_num_frames,
|
||||
"frames_indices": list(range(video_num_frames)),
|
||||
"video_backend": "opencv",
|
||||
"do_sample_frames": False,
|
||||
}
|
||||
video_items.append((video, video_metadata))
|
||||
return video_items
|
||||
|
||||
|
||||
class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
def get_hf_config(self):
|
||||
return self.ctx.get_hf_config(Qwen2_5_VLConfig)
|
||||
@@ -1148,6 +1187,14 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_data_parser(self):
|
||||
return Qwen2VLMultiModalDataParser(
|
||||
self.get_hf_config().vision_config.spatial_merge_size,
|
||||
video_needs_metadata=True,
|
||||
expected_hidden_size=self._get_expected_hidden_size(),
|
||||
allow_missing_mm_embeddings=self.allow_missing_mm_embeddings,
|
||||
)
|
||||
|
||||
|
||||
class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
def _get_mm_fields_config(
|
||||
@@ -1160,6 +1207,25 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
second_per_grid_ts=MultiModalFieldConfig.batched("video", keep_on_cpu=True),
|
||||
)
|
||||
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> HFMultiModalInputs:
|
||||
hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs)
|
||||
hf_data = hf_inputs.hf_data
|
||||
|
||||
if videos := hf_data.get("videos"):
|
||||
# HF expects video metadata as a separate argument.
|
||||
hf_data["videos"] = [video for video, _ in videos]
|
||||
hf_data["video_metadata"] = [
|
||||
VideoMetadata(
|
||||
**{k: metadata[k] for k in metadata if k != "do_sample_frames"}
|
||||
)
|
||||
for _, metadata in videos
|
||||
]
|
||||
return hf_inputs
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
|
||||
Reference in New Issue
Block a user