[Perf] Optimize Qwen3-VL multi-video prompt processing (#46026)

Signed-off-by: Sirius29 <422058530@qq.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
L丶
2026-06-20 07:09:18 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7ff7f5c8eb
commit d272418f45
2 changed files with 124 additions and 26 deletions
@@ -92,3 +92,49 @@ def test_processor_num_frames_timestamp(
assert len(video_phs) == 1, (
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
)
@pytest.mark.parametrize("model_id", [MODEL_ID])
@pytest.mark.parametrize("num_videos", [2, 4])
def test_processor_multi_video(
model_id: str,
num_videos: int,
) -> None:
"""Verify that multi-video processing produces correct placeholders.
This exercises the token-level replacement path in
``_call_hf_processor`` which avoids the quadratic text-level
prompt expansion.
"""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": num_videos},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos
mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={"num_frames": 8},
)
token_ids = processed["prompt_token_ids"]
assert len(token_ids) > 0
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == num_videos, (
f"Expected {num_videos} video placeholders, got {len(video_phs)}"
)
# All placeholders should have the same length (same video params)
# and must not overlap.
lengths = {ph.length for ph in video_phs}
assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}"
for i in range(1, len(video_phs)):
prev_end = video_phs[i - 1].offset + video_phs[i - 1].length
assert video_phs[i].offset >= prev_end, (
f"Placeholder {i} overlaps with placeholder {i - 1}"
)
+78 -26
View File
@@ -1202,6 +1202,49 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
return video_items
def _replace_video_token_placeholders(
prompt_ids: list[int],
target: list[int],
replacements: list[list[int]],
) -> list[int]:
"""Replace each 3-token video placeholder with its expanded sequence.
Args:
prompt_ids: Token IDs of the original (unexpanded) prompt.
target: 3-element list ``[vision_start_id, video_pad_id,
vision_end_id]`` to search for.
replacements: Per-video expanded token sequences, in prompt order.
Returns:
Token IDs with every placeholder triplet replaced.
"""
result: list[int] = []
repl_idx = 0
i = 0
n = len(prompt_ids)
t0, t1, t2 = target
num_repl = len(replacements)
while i < n:
if (
i + 2 < n
and prompt_ids[i] == t0
and prompt_ids[i + 1] == t1
and prompt_ids[i + 2] == t2
):
result.extend(replacements[repl_idx])
repl_idx += 1
i += 3
else:
result.append(prompt_ids[i])
i += 1
assert repl_idx == num_repl, (
f"Found {repl_idx} video placeholders but expected {num_repl}"
)
return result
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
def _call_hf_processor(
self,
@@ -1211,15 +1254,23 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
tok_kwargs: Mapping[str, object],
) -> BatchFeature:
mm_data = dict(mm_data)
processor = self.info.get_hf_processor(**mm_kwargs)
# Separate video processing from image processing. Because the videos
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
if videos := mm_data.pop("videos", []):
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
hf_config = self.info.get_hf_config()
tokenizer = self.info.get_tokenizer()
merge_size = hf_config.vision_config.spatial_merge_size
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
vision_start_token_id = hf_config.vision_start_token_id
vision_end_token_id = hf_config.vision_end_token_id
video_token_id = hf_config.video_token_id
for item in videos:
video_array, metadata = item
@@ -1269,55 +1320,38 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
tok_kwargs=tok_kwargs,
)
merge_size = processor.video_processor.merge_size
# Get video grid info for EVS calculation.
# Discard HF output input_ids — we use get_video_repl below
# to generate the correct (EVS-adjusted) token sequence.
video_outputs.pop("input_ids", None)
video_grid_thw = video_outputs["video_grid_thw"]
num_frames = int(video_grid_thw[0, 0])
tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // (
merge_size**2
)
# Apply EVS if enabled.
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
if video_pruning_rate is not None and video_pruning_rate > 0.0:
num_tokens = compute_retained_tokens_count(
tokens_per_frame=tokens_per_frame_base,
num_frames=num_frames,
q=video_pruning_rate,
)
# Here we just need placeholders that won't actually be replaced -
# we just need to make sure the total number of tokens is correct
# assign all tokens to the first frame.
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
select_token_id = False
else:
tokens_per_frame = [tokens_per_frame_base] * num_frames
select_token_id = True
# Generate the video replacement with EVS-adjusted token counts
tokenizer = self.info.get_tokenizer()
hf_config = self.info.get_hf_config()
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
tokens_per_frame=tokens_per_frame,
timestamps=timestamps,
tokenizer=tokenizer,
vision_start_token_id=hf_config.vision_start_token_id,
vision_end_token_id=hf_config.vision_end_token_id,
video_token_id=hf_config.video_token_id,
vision_start_token_id=vision_start_token_id,
vision_end_token_id=vision_end_token_id,
video_token_id=video_token_id,
select_token_id=select_token_id,
)
# Convert token IDs to text for the HF processor flow
video_placeholder = tokenizer.decode(
video_repl.full, skip_special_tokens=False
)
input_ids = video_outputs.pop("input_ids")
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
prompt = prompt.replace(
"<|vision_start|><|video_pad|><|vision_end|>",
video_placeholder,
1,
)
video_input_ids_lst.append(list(video_repl.full))
video_grid_thw_lst.append(video_outputs["video_grid_thw"])
pixel_values_videos_lst.append(video_outputs["pixel_values_videos"])
@@ -1335,6 +1369,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
mm_kwargs=mm_kwargs,
tok_kwargs=tok_kwargs,
)
# Replace each placeholder triplet with pre-computed video tokens.
if video_input_ids_lst:
hf_config = self.info.get_hf_config()
video_target = [
hf_config.vision_start_token_id,
hf_config.video_token_id,
hf_config.vision_end_token_id,
]
input_ids = processed_outputs.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
expanded_ids = _replace_video_token_placeholders(
prompt_ids, video_target, video_input_ids_lst
)
processed_outputs["input_ids"] = [expanded_ids]
combined_outputs = dict(
processed_outputs,
**video_outputs,