[Multimodal] Add Qwen2-VL/Qwen2.5-VL processor-mapped video loader (#45555)
Signed-off-by: Ranran <hzz5361@psu.edu> Signed-off-by: Ranran Haoran Zhang <ranzhang@redhat.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Isotr0py
parent
c441ad1c07
commit
635c38338a
@@ -25,7 +25,7 @@ def server():
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--max-model-len",
|
||||
"5000",
|
||||
"16384",
|
||||
"--enforce-eager",
|
||||
"--limit-mm-per-prompt",
|
||||
json.dumps({"video": MAXIMUM_VIDEOS}),
|
||||
@@ -143,4 +143,4 @@ def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str):
|
||||
assert output.model == model_name
|
||||
assert len(output.data) == 1
|
||||
assert len(output.data[0].probs) == 2
|
||||
assert output.usage.prompt_tokens == 4807
|
||||
assert output.usage.prompt_tokens == 8993
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.multimodal.video import (
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
Qwen2VLVideoBackend,
|
||||
Qwen3VLVideoBackend,
|
||||
VideoLoader,
|
||||
VideoSourceMetadata,
|
||||
@@ -70,11 +71,12 @@ def test_video_loader_type_doesnt_exist():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_repo, expected_loader_cls",
|
||||
"model_repo, expected_loader_cls, hf_sample_kwargs",
|
||||
[
|
||||
pytest.param(
|
||||
"allenai/Molmo2-4B",
|
||||
Molmo2VideoBackend,
|
||||
None,
|
||||
marks=pytest.mark.skip(
|
||||
reason="Video processor not aligned, investigate later.",
|
||||
),
|
||||
@@ -83,23 +85,44 @@ def test_video_loader_type_doesnt_exist():
|
||||
pytest.param(
|
||||
"zai-org/GLM-4.1V-9B-Thinking",
|
||||
DynamicVideoBackend,
|
||||
None,
|
||||
id="glm4v",
|
||||
),
|
||||
pytest.param(
|
||||
"zai-org/GLM-4.6V-Flash",
|
||||
GLM46VVideoBackend,
|
||||
None,
|
||||
id="glm46v",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
Qwen3VLVideoBackend,
|
||||
None,
|
||||
id="qwen3vl",
|
||||
),
|
||||
# Qwen2-VL/Qwen2.5-VL ship no ``video_processor_type`` in their
|
||||
# preprocessor config, so resolution relies on the model_type ->
|
||||
# video processor fallback in get_video_processor_cls_name_from_config.
|
||||
# They also ship no default fps/num_frames, so the HF sampler needs an
|
||||
# explicit target rate; pass fps=2 to match the loader default.
|
||||
pytest.param(
|
||||
"Qwen/Qwen2-VL-7B-Instruct",
|
||||
Qwen2VLVideoBackend,
|
||||
{"fps": 2},
|
||||
id="qwen2vl",
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-VL-7B-Instruct",
|
||||
Qwen2VLVideoBackend,
|
||||
{"fps": 2},
|
||||
id="qwen2_5_vl",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_video_processor_from_model_repo(
|
||||
model_repo: str,
|
||||
expected_loader_cls: type,
|
||||
hf_sample_kwargs: dict[str, int | float] | None,
|
||||
):
|
||||
"""Test that a model repo resolves to the correct video loader backend.
|
||||
|
||||
@@ -143,7 +166,7 @@ def test_video_processor_from_model_repo(
|
||||
fps=vllm_meta["fps"],
|
||||
duration=vllm_meta["duration"],
|
||||
)
|
||||
hf_indices = processor.sample_frames(hf_metadata)
|
||||
hf_indices = processor.sample_frames(hf_metadata, **(hf_sample_kwargs or {}))
|
||||
vllm_indices = np.array(vllm_meta["frames_indices"])
|
||||
np.testing.assert_array_equal(
|
||||
hf_indices,
|
||||
|
||||
Reference in New Issue
Block a user