[Perf] Optimize Qwen3-VL multi-video prompt processing (#46026)
Signed-off-by: Sirius29 <422058530@qq.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7ff7f5c8eb
commit
d272418f45
@@ -92,3 +92,49 @@ def test_processor_num_frames_timestamp(
|
||||
assert len(video_phs) == 1, (
|
||||
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", [MODEL_ID])
|
||||
@pytest.mark.parametrize("num_videos", [2, 4])
|
||||
def test_processor_multi_video(
|
||||
model_id: str,
|
||||
num_videos: int,
|
||||
) -> None:
|
||||
"""Verify that multi-video processing produces correct placeholders.
|
||||
|
||||
This exercises the token-level replacement path in
|
||||
``_call_hf_processor`` which avoids the quadratic text-level
|
||||
prompt expansion.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"image": 0, "video": num_videos},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>" * num_videos
|
||||
mm_data = {"video": [_build_video_mm_data(num_frames=8)["video"][0]] * num_videos}
|
||||
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(mm_data),
|
||||
hf_processor_mm_kwargs={"num_frames": 8},
|
||||
)
|
||||
|
||||
token_ids = processed["prompt_token_ids"]
|
||||
assert len(token_ids) > 0
|
||||
|
||||
video_phs = processed["mm_placeholders"].get("video", [])
|
||||
assert len(video_phs) == num_videos, (
|
||||
f"Expected {num_videos} video placeholders, got {len(video_phs)}"
|
||||
)
|
||||
|
||||
# All placeholders should have the same length (same video params)
|
||||
# and must not overlap.
|
||||
lengths = {ph.length for ph in video_phs}
|
||||
assert len(lengths) == 1, f"Placeholder lengths differ: {lengths}"
|
||||
for i in range(1, len(video_phs)):
|
||||
prev_end = video_phs[i - 1].offset + video_phs[i - 1].length
|
||||
assert video_phs[i].offset >= prev_end, (
|
||||
f"Placeholder {i} overlaps with placeholder {i - 1}"
|
||||
)
|
||||
|
||||
@@ -1202,6 +1202,49 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
return video_items
|
||||
|
||||
|
||||
def _replace_video_token_placeholders(
|
||||
prompt_ids: list[int],
|
||||
target: list[int],
|
||||
replacements: list[list[int]],
|
||||
) -> list[int]:
|
||||
"""Replace each 3-token video placeholder with its expanded sequence.
|
||||
|
||||
Args:
|
||||
prompt_ids: Token IDs of the original (unexpanded) prompt.
|
||||
target: 3-element list ``[vision_start_id, video_pad_id,
|
||||
vision_end_id]`` to search for.
|
||||
replacements: Per-video expanded token sequences, in prompt order.
|
||||
|
||||
Returns:
|
||||
Token IDs with every placeholder triplet replaced.
|
||||
"""
|
||||
result: list[int] = []
|
||||
repl_idx = 0
|
||||
i = 0
|
||||
n = len(prompt_ids)
|
||||
t0, t1, t2 = target
|
||||
num_repl = len(replacements)
|
||||
|
||||
while i < n:
|
||||
if (
|
||||
i + 2 < n
|
||||
and prompt_ids[i] == t0
|
||||
and prompt_ids[i + 1] == t1
|
||||
and prompt_ids[i + 2] == t2
|
||||
):
|
||||
result.extend(replacements[repl_idx])
|
||||
repl_idx += 1
|
||||
i += 3
|
||||
else:
|
||||
result.append(prompt_ids[i])
|
||||
i += 1
|
||||
|
||||
assert repl_idx == num_repl, (
|
||||
f"Found {repl_idx} video placeholders but expected {num_repl}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]):
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
@@ -1211,15 +1254,23 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
mm_data = dict(mm_data)
|
||||
processor = self.info.get_hf_processor(**mm_kwargs)
|
||||
|
||||
# Separate video processing from image processing. Because the videos
|
||||
# are processed into several image patches
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
if videos := mm_data.pop("videos", []):
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
timestamps_per_video = []
|
||||
|
||||
hf_config = self.info.get_hf_config()
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
merge_size = hf_config.vision_config.spatial_merge_size
|
||||
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
|
||||
vision_start_token_id = hf_config.vision_start_token_id
|
||||
vision_end_token_id = hf_config.vision_end_token_id
|
||||
video_token_id = hf_config.video_token_id
|
||||
|
||||
for item in videos:
|
||||
video_array, metadata = item
|
||||
|
||||
@@ -1269,55 +1320,38 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
merge_size = processor.video_processor.merge_size
|
||||
# Get video grid info for EVS calculation.
|
||||
# Discard HF output input_ids — we use get_video_repl below
|
||||
# to generate the correct (EVS-adjusted) token sequence.
|
||||
video_outputs.pop("input_ids", None)
|
||||
|
||||
video_grid_thw = video_outputs["video_grid_thw"]
|
||||
num_frames = int(video_grid_thw[0, 0])
|
||||
tokens_per_frame_base = int(video_grid_thw[0, 1:].prod()) // (
|
||||
merge_size**2
|
||||
)
|
||||
|
||||
# Apply EVS if enabled.
|
||||
video_pruning_rate = self.info.ctx.get_mm_config().video_pruning_rate
|
||||
if video_pruning_rate is not None and video_pruning_rate > 0.0:
|
||||
num_tokens = compute_retained_tokens_count(
|
||||
tokens_per_frame=tokens_per_frame_base,
|
||||
num_frames=num_frames,
|
||||
q=video_pruning_rate,
|
||||
)
|
||||
# Here we just need placeholders that won't actually be replaced -
|
||||
# we just need to make sure the total number of tokens is correct
|
||||
# assign all tokens to the first frame.
|
||||
tokens_per_frame = [num_tokens] + [0] * (num_frames - 1)
|
||||
select_token_id = False
|
||||
else:
|
||||
tokens_per_frame = [tokens_per_frame_base] * num_frames
|
||||
select_token_id = True
|
||||
|
||||
# Generate the video replacement with EVS-adjusted token counts
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
hf_config = self.info.get_hf_config()
|
||||
video_repl = Qwen3VLMultiModalProcessor.get_video_repl(
|
||||
tokens_per_frame=tokens_per_frame,
|
||||
timestamps=timestamps,
|
||||
tokenizer=tokenizer,
|
||||
vision_start_token_id=hf_config.vision_start_token_id,
|
||||
vision_end_token_id=hf_config.vision_end_token_id,
|
||||
video_token_id=hf_config.video_token_id,
|
||||
vision_start_token_id=vision_start_token_id,
|
||||
vision_end_token_id=vision_end_token_id,
|
||||
video_token_id=video_token_id,
|
||||
select_token_id=select_token_id,
|
||||
)
|
||||
|
||||
# Convert token IDs to text for the HF processor flow
|
||||
video_placeholder = tokenizer.decode(
|
||||
video_repl.full, skip_special_tokens=False
|
||||
)
|
||||
input_ids = video_outputs.pop("input_ids")
|
||||
video_placeholder = processor.tokenizer.batch_decode(input_ids)[0]
|
||||
prompt = prompt.replace(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>",
|
||||
video_placeholder,
|
||||
1,
|
||||
)
|
||||
video_input_ids_lst.append(list(video_repl.full))
|
||||
|
||||
video_grid_thw_lst.append(video_outputs["video_grid_thw"])
|
||||
pixel_values_videos_lst.append(video_outputs["pixel_values_videos"])
|
||||
@@ -1335,6 +1369,24 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
mm_kwargs=mm_kwargs,
|
||||
tok_kwargs=tok_kwargs,
|
||||
)
|
||||
|
||||
# Replace each placeholder triplet with pre-computed video tokens.
|
||||
if video_input_ids_lst:
|
||||
hf_config = self.info.get_hf_config()
|
||||
video_target = [
|
||||
hf_config.vision_start_token_id,
|
||||
hf_config.video_token_id,
|
||||
hf_config.vision_end_token_id,
|
||||
]
|
||||
input_ids = processed_outputs.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
(prompt_ids,) = input_ids
|
||||
expanded_ids = _replace_video_token_placeholders(
|
||||
prompt_ids, video_target, video_input_ids_lst
|
||||
)
|
||||
processed_outputs["input_ids"] = [expanded_ids]
|
||||
|
||||
combined_outputs = dict(
|
||||
processed_outputs,
|
||||
**video_outputs,
|
||||
|
||||
Reference in New Issue
Block a user