[MM] Simplify _apply_hf_processor_main (#53275)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-08-22 01:49:37 -07:00
committed by GitHub
parent 7ca49fbe4b
commit 0b19ebcacd
90 changed files with 1848 additions and 1706 deletions
@@ -136,7 +136,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
# no audio track" case. (The audio limit above is 1, not 0, so that
# `mm_counts["audio"]` is still declared for this request; a 0 limit
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
# trips an unrelated assertion in `_apply_hf_processor_mm_only`.)
# trips an unrelated assertion in `_apply_hf_processor_main`.)
mm_data = {"video": [video]}
with pytest.raises(ValueError, match="doesn't have audio track"):
@@ -105,9 +105,9 @@ def test_audio_chunk_counting(mock_ctx):
audio_2 = np.zeros(75 * sr)
mm_data = {"audio": [audio_1, audio_2]}
prompt = "<|user|>Listen.<|end|>"
processed = processor._call_hf_processor(prompt, mm_data, {})
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
processed = processor._apply_hf_processor_main(mm_items, {})
chunk_counts = processed["chunk_counts"]
@@ -385,17 +385,19 @@ def test_processing_correctness(
num_batches: int,
simplify_rate: float,
):
if model_id == "google/gemma-3n-E2B-it":
pytest.skip("Fix later")
if model_id == "OpenGVLab/InternVL2-2B":
pytest.skip("Fix later")
if model_id == "openvla/openvla-7b":
pytest.skip(
"OpenVLA uses a custom vLLM processor because its HF remote "
"processor is incompatible with current Transformers."
)
if model_id == "jinaai/jina-reranker-m0":
pytest.skip("Fix later")
pytest.skip(
"JinaVL reverses the multi-modal item order inside "
"_apply_hf_processor_main to match its document-before-query "
"score template, which breaks the positional correspondence "
"between multi-modal kwargs and hashes assumed by the "
"processing cache. Needs a fix that keeps both consistent."
)
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
pytest.skip(
"Voxtral Realtime doesn't make use of any place-holder "
@@ -403,22 +405,12 @@ def test_processing_correctness(
"correctness test as is. Let's revisit adapting this "
"test once more realtime models exist."
)
if model_id == "CohereLabs/cohere-transcribe-03-2026":
pytest.skip("Fix later")
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
pytest.skip(
"MOSS-Audio uses a custom processor that dynamically expands "
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
# TODO: Remove when transformers 5.15.0 is released, which contains
# https://github.com/huggingface/transformers/pull/47483.
if model_id == "microsoft/VibeVoice-ASR-HF":
pytest.skip(
"VibeVoice ASR requires audio as a positional argument and hence "
"cannot pass the processing correctness test as is. Its generation "
"is covered by test_transformers_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
@@ -202,7 +202,7 @@ def test_validate_relative_bare_path_blocked():
# ---------------------------------------------------------------------------
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
#
# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
# (``video_needs_metadata=True``). The helper materialises PIL frames and
@@ -103,7 +103,7 @@ def test_processor_multi_video(
"""Verify that multi-video processing produces correct placeholders.
This exercises the token-level replacement path in
``_call_hf_processor`` which avoids the quadratic text-level
``_apply_hf_processor_main`` which avoids the quadratic text-level
prompt expansion.
"""
ctx = build_model_context(
@@ -153,7 +153,7 @@ def test_processor_multi_video_list_kwargs(
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
crash.
Before the fix, ``_call_hf_processor`` copied the whole kwargs to every
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
video without slicing, so ``_get_video_second_idx`` received the list
where a scalar was expected and raised ``TypeError``.
"""