[MM] Simplify _apply_hf_processor_main (#53275)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -136,7 +136,7 @@ def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
|
||||
# no audio track" case. (The audio limit above is 1, not 0, so that
|
||||
# `mm_counts["audio"]` is still declared for this request; a 0 limit
|
||||
# would make vLLM omit the "audio" key from `mm_counts` entirely, which
|
||||
# trips an unrelated assertion in `_apply_hf_processor_mm_only`.)
|
||||
# trips an unrelated assertion in `_apply_hf_processor_main`.)
|
||||
mm_data = {"video": [video]}
|
||||
|
||||
with pytest.raises(ValueError, match="doesn't have audio track"):
|
||||
|
||||
@@ -105,9 +105,9 @@ def test_audio_chunk_counting(mock_ctx):
|
||||
audio_2 = np.zeros(75 * sr)
|
||||
|
||||
mm_data = {"audio": [audio_1, audio_2]}
|
||||
prompt = "<|user|>Listen.<|end|>"
|
||||
|
||||
processed = processor._call_hf_processor(prompt, mm_data, {})
|
||||
mm_items = processor.info.parse_mm_data(mm_data, validate=False)
|
||||
processed = processor._apply_hf_processor_main(mm_items, {})
|
||||
|
||||
chunk_counts = processed["chunk_counts"]
|
||||
|
||||
|
||||
@@ -385,17 +385,19 @@ def test_processing_correctness(
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
if model_id == "google/gemma-3n-E2B-it":
|
||||
pytest.skip("Fix later")
|
||||
if model_id == "OpenGVLab/InternVL2-2B":
|
||||
pytest.skip("Fix later")
|
||||
if model_id == "openvla/openvla-7b":
|
||||
pytest.skip(
|
||||
"OpenVLA uses a custom vLLM processor because its HF remote "
|
||||
"processor is incompatible with current Transformers."
|
||||
)
|
||||
if model_id == "jinaai/jina-reranker-m0":
|
||||
pytest.skip("Fix later")
|
||||
pytest.skip(
|
||||
"JinaVL reverses the multi-modal item order inside "
|
||||
"_apply_hf_processor_main to match its document-before-query "
|
||||
"score template, which breaks the positional correspondence "
|
||||
"between multi-modal kwargs and hashes assumed by the "
|
||||
"processing cache. Needs a fix that keeps both consistent."
|
||||
)
|
||||
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
||||
pytest.skip(
|
||||
"Voxtral Realtime doesn't make use of any place-holder "
|
||||
@@ -403,22 +405,12 @@ def test_processing_correctness(
|
||||
"correctness test as is. Let's revisit adapting this "
|
||||
"test once more realtime models exist."
|
||||
)
|
||||
if model_id == "CohereLabs/cohere-transcribe-03-2026":
|
||||
pytest.skip("Fix later")
|
||||
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
|
||||
pytest.skip(
|
||||
"MOSS-Audio uses a custom processor that dynamically expands "
|
||||
"audio placeholders from processed audio lengths. Its vLLM "
|
||||
"processor paths are covered by test_moss_audio.py."
|
||||
)
|
||||
# TODO: Remove when transformers 5.15.0 is released, which contains
|
||||
# https://github.com/huggingface/transformers/pull/47483.
|
||||
if model_id == "microsoft/VibeVoice-ASR-HF":
|
||||
pytest.skip(
|
||||
"VibeVoice ASR requires audio as a positional argument and hence "
|
||||
"cannot pass the processing correctness test as is. Its generation "
|
||||
"is covered by test_transformers_audio.py."
|
||||
)
|
||||
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
|
||||
pytest.skip(
|
||||
"LLaVA-OneVision-2 video processing routes frames through custom "
|
||||
|
||||
@@ -202,7 +202,7 @@ def test_validate_relative_bare_path_blocked():
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frame backend marker -> PIL + timestamps (_frame_video_to_pil_and_timestamps)
|
||||
#
|
||||
# Non-codec videos reach _call_hf_processor as a ``(frames_ndarray, metadata)``
|
||||
# Non-codec videos reach _apply_hf_processor_main as a ``(frames_ndarray, metadata)``
|
||||
# tuple -- produced by the registered ``LlavaOnevision2VideoBackend`` for real
|
||||
# ``video_url`` inputs, or by the dummy-inputs builder during profiling
|
||||
# (``video_needs_metadata=True``). The helper materialises PIL frames and
|
||||
|
||||
@@ -103,7 +103,7 @@ def test_processor_multi_video(
|
||||
"""Verify that multi-video processing produces correct placeholders.
|
||||
|
||||
This exercises the token-level replacement path in
|
||||
``_call_hf_processor`` which avoids the quadratic text-level
|
||||
``_apply_hf_processor_main`` which avoids the quadratic text-level
|
||||
prompt expansion.
|
||||
"""
|
||||
ctx = build_model_context(
|
||||
@@ -153,7 +153,7 @@ def test_processor_multi_video_list_kwargs(
|
||||
``mm_processor_kwargs`` (one ``fps``/``num_frames`` per video) must not
|
||||
crash.
|
||||
|
||||
Before the fix, ``_call_hf_processor`` copied the whole kwargs to every
|
||||
Before the fix, ``_apply_hf_processor_main`` copied the whole kwargs to every
|
||||
video without slicing, so ``_get_video_second_idx`` received the list
|
||||
where a scalar was expected and raised ``TypeError``.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user