From d272418f459a82e1012b60116ac00659a7017cde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=E4=B8=B6?= <30801931+Sirius29@users.noreply.github.com> Date: Sat, 20 Jun 2026 22:09:18 +0800 Subject: [PATCH] [Perf] Optimize Qwen3-VL multi-video prompt processing (#46026) Signed-off-by: Sirius29 <422058530@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../multimodal/processing/test_qwen3_vl.py | 46 ++++++++ vllm/model_executor/models/qwen3_vl.py | 104 +++++++++++++----- 2 files changed, 124 insertions(+), 26 deletions(-) diff --git a/tests/models/multimodal/processing/test_qwen3_vl.py b/tests/models/multimodal/processing/test_qwen3_vl.py index d69c31b582..9155fde503 100644 --- a/tests/models/multimodal/processing/test_qwen3_vl.py +++ b/tests/models/multimodal/processing/test_qwen3_vl.py @@ -92,3 +92,49 @@ def test_processor_num_frames_timestamp( assert len(video_phs) == 1, ( f"Expected exactly 1 video placeholder, got {len(video_phs)}" ) + + +@pytest.mark.parametrize("model_id", [MODEL_ID]) +@pytest.mark.parametrize("num_videos", [2, 4]) +def test_processor_multi_video( + model_id: str, + num_videos: int, +) -> None: + """Verify that multi-video processing produces correct placeholders. + + This exercises the token-level replacement path in + ``_call_hf_processor`` which avoids the quadratic text-level + prompt expansion. + """ + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"image": 0, "video": num_videos}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + + prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos + mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos} + + processed = processor( + prompt, + mm_items=processor.info.parse_mm_data(mm_data), + hf_processor_mm_kwargs={"num_frames": 8}, + ) + + token_ids = processed["prompt_token_ids"] + assert len(token_ids) > 0 + + video_phs = processed["mm_placeholders"].get("video", []) + assert len(video_phs) == num_videos, ( + f"Expected {num_videos} video placeholders, got {len(video_phs)}" + ) + + # All placeholders should have the same length (same video params) + # and must not overlap. + lengths = {ph.length for ph in video_phs} + assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}" + for i in range(1, len(video_phs)): + prev_end = video_phs[i - 1].offset + video_phs[i - 1].length + assert video_phs[i].offset >= prev_end, ( + f"Placeholder {i} overlaps with placeholder {i - 1}" + ) diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 1423770be0..3183a23ffd 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1202,6 +1202,49 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): return video_items +def _replace_video_token_placeholders( + prompt_ids: list[int], + target: list[int], + replacements: list[list[int]], +) -> list[int]: + """Replace each 3-token video placeholder with its expanded sequence. + + Args: + prompt_ids: Token IDs of the original (unexpanded) prompt. + target: 3-element list ``[vision_start_id, video_pad_id, + vision_end_id]`` to search for. + replacements: Per-video expanded token sequences, in prompt order. + + Returns: + Token IDs with every placeholder triplet replaced. + """ + result: list[int] = [] + repl_idx = 0 + i = 0 + n = len(prompt_ids) + t0, t1, t2 = target + num_repl = len(replacements) + + while i < n: + if ( + i + 2 < n + and prompt_ids[i] == t0 + and prompt_ids[i + 1] == t1 + and prompt_ids[i + 2] == t2 + ): + result.extend(replacements[repl_idx]) + repl_idx += 1 + i += 3 + else: + result.append(prompt_ids[i]) + i += 1 + + assert repl_idx == num_repl, ( + f"Found {repl_idx} video placeholders but expected {num_repl}" + ) + return result + + class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]): def _call_hf_processor( self, @@ -1211,15 +1254,23 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) tok_kwargs: Mapping[str, object], ) -> BatchFeature: mm_data = dict(mm_data) - processor = self.info.get_hf_processor(**mm_kwargs) # Separate video processing from image processing. Because the videos # are processed into several image patches + video_input_ids_lst: list[list[int]] = [] if videos := mm_data.pop("videos", []): video_grid_thw_lst = [] pixel_values_videos_lst = [] timestamps_per_video = [] + hf_config = self.info.get_hf_config() + tokenizer = self.info.get_tokenizer() + merge_size = hf_config.vision_config.spatial_merge_size + video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate + vision_start_token_id = hf_config.vision_start_token_id + vision_end_token_id = hf_config.vision_end_token_id + video_token_id = hf_config.video_token_id + for item in videos: video_array, metadata = item @@ -1269,55 +1320,38 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) tok_kwargs=tok_kwargs, ) - merge_size = processor.video_processor.merge_size - # Get video grid info for EVS calculation. + # Discard HF output input_ids — we use get_video_repl below + # to generate the correct (EVS-adjusted) token sequence. + video_outputs.pop("input_ids", None) + video_grid_thw = video_outputs["video_grid_thw"] num_frames = int(video_grid_thw[0, 0]) tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // ( merge_size**2 ) - # Apply EVS if enabled. - video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate if video_pruning_rate is not None and video_pruning_rate > 0.0: num_tokens = compute_retained_tokens_count( tokens_per_frame=tokens_per_frame_base, num_frames=num_frames, q=video_pruning_rate, ) - # Here we just need placeholders that won't actually be replaced - - # we just need to make sure the total number of tokens is correct - # assign all tokens to the first frame. tokens_per_frame = [num_tokens] + [0] * (num_frames - 1) select_token_id = False else: tokens_per_frame = [tokens_per_frame_base] * num_frames select_token_id = True - # Generate the video replacement with EVS-adjusted token counts - tokenizer = self.info.get_tokenizer() - hf_config = self.info.get_hf_config() video_repl = Qwen3VLMultiModalProcessor.get_video_repl( tokens_per_frame=tokens_per_frame, timestamps=timestamps, tokenizer=tokenizer, - vision_start_token_id=hf_config.vision_start_token_id, - vision_end_token_id=hf_config.vision_end_token_id, - video_token_id=hf_config.video_token_id, + vision_start_token_id=vision_start_token_id, + vision_end_token_id=vision_end_token_id, + video_token_id=video_token_id, select_token_id=select_token_id, ) - - # Convert token IDs to text for the HF processor flow - video_placeholder = tokenizer.decode( - video_repl.full, skip_special_tokens=False - ) - input_ids = video_outputs.pop("input_ids") - video_placeholder = processor.tokenizer.batch_decode(input_ids)[0] - prompt = prompt.replace( - "<|vision_start|><|video_pad|><|vision_end|>", - video_placeholder, - 1, - ) + video_input_ids_lst.append(list(video_repl.full)) video_grid_thw_lst.append(video_outputs["video_grid_thw"]) pixel_values_videos_lst.append(video_outputs["pixel_values_videos"]) @@ -1335,6 +1369,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) mm_kwargs=mm_kwargs, tok_kwargs=tok_kwargs, ) + + # Replace each placeholder triplet with pre-computed video tokens. + if video_input_ids_lst: + hf_config = self.info.get_hf_config() + video_target = [ + hf_config.vision_start_token_id, + hf_config.video_token_id, + hf_config.vision_end_token_id, + ] + input_ids = processed_outputs.pop("input_ids") + if not isinstance(input_ids, list): + input_ids = input_ids.tolist() + (prompt_ids,) = input_ids + expanded_ids = _replace_video_token_placeholders( + prompt_ids, video_target, video_input_ids_lst + ) + processed_outputs["input_ids"] = [expanded_ids] + combined_outputs = dict( processed_outputs, **video_outputs,