Merge branch 'main' into patches
This commit is contained in:
@@ -69,13 +69,8 @@ AUDIO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
|
||||
def test_transformers_audio_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
monkeypatch,
|
||||
model_id: str,
|
||||
):
|
||||
@pytest.fixture(autouse=True)
|
||||
def use_spawn_for_audio_models(monkeypatch):
|
||||
"""Single-process workaround for V1 fork safety deadlock issue
|
||||
(vllm-project/vllm/issues/17676). Running multiple audio models together
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
@@ -87,6 +82,13 @@ def test_transformers_audio_generation(
|
||||
disable_envs_cache()
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(AUDIO_MODEL_SETTINGS))
|
||||
def test_transformers_audio_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model_id: str,
|
||||
):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
audio_lora_path = settings.get("audio_lora_path")
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.envs import disable_envs_cache
|
||||
|
||||
from ....conftest import HfRunner, VllmRunner
|
||||
from ...utils import check_logprobs_close
|
||||
from .vlm_utils import model_utils
|
||||
|
||||
VIDEO_ASSET = VideoAsset("baby_reading", num_frames=8)
|
||||
|
||||
VIDEO_MODEL_SETTINGS: dict[str, dict[str, Any]] = {
|
||||
"Qwen/Qwen3-VL-2B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf": {
|
||||
"prompt": (
|
||||
"<|im_start|>user <video>\nDescribe this video.<|im_end|>"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"CohereLabs/North-Micro-Vision-Instruct": {
|
||||
"prompt": (
|
||||
"<BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>"
|
||||
"<|VISION_START|><|VIDEO_PAD|><|VISION_END|>Describe this video."
|
||||
"<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
|
||||
),
|
||||
},
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def use_spawn_for_video_models(monkeypatch):
|
||||
"""Single-process workaround for V1 fork safety deadlock issue
|
||||
(vllm-project/vllm/issues/17676). Running multiple video models together
|
||||
under pytest can cause (possibly flaky) hangs, so they are grouped under
|
||||
the same config. Using VLLM_WORKER_MULTIPROC_METHOD=spawn avoids the
|
||||
deadlock and allows worker processes to terminate cleanly, and release
|
||||
GPU memory between test runs until the issue is fixed."""
|
||||
# TODO: Remove monkeypatch once
|
||||
# https://github.com/vllm-project/vllm/issues/17676 is fixed.
|
||||
disable_envs_cache()
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_transformers_video_generation(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
model_id: str,
|
||||
):
|
||||
prompt = VIDEO_MODEL_SETTINGS[model_id]["prompt"]
|
||||
video = (VIDEO_ASSET.np_ndarrays, VIDEO_ASSET.metadata)
|
||||
|
||||
with vllm_runner(
|
||||
model_id,
|
||||
model_impl="transformers",
|
||||
dtype="bfloat16",
|
||||
max_model_len=8192,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
[prompt], 128, num_logprobs=10, videos=[video]
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model_id, dtype="bfloat16", auto_cls=AutoModelForImageTextToText
|
||||
) as hf_model:
|
||||
hf_model = model_utils.qwen3_vl_patch_hf_runner(hf_model)
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
[prompt], 128, num_logprobs=10, videos=[video]
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import numpy as np
|
||||
import pytest
|
||||
from transformers import Qwen3VLVideoProcessor
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
|
||||
VIDEO_MODEL_SETTINGS = {
|
||||
"Qwen/Qwen3-VL-2B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf": {
|
||||
"prompt": (
|
||||
"<|im_start|>user <video>\nDescribe this video.<|im_end|>"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
"CohereLabs/North-Micro-Vision-Instruct": {
|
||||
"prompt": (
|
||||
"<BOS_TOKEN><|START_OF_TURN_TOKEN|><|USER_TOKEN|>"
|
||||
"<|VISION_START|><|VIDEO_PAD|><|VISION_END|>Describe this video."
|
||||
"<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
|
||||
),
|
||||
},
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct": {
|
||||
"prompt": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _video(
|
||||
num_frames: int = 8,
|
||||
seed: int = 0,
|
||||
fps: float = 1.0,
|
||||
do_sample_frames: bool = False,
|
||||
) -> tuple[np.ndarray, dict]:
|
||||
rng = np.random.default_rng(seed)
|
||||
frames = rng.integers(256, size=(num_frames, 64, 64, 3), dtype=np.uint8)
|
||||
metadata = {
|
||||
"fps": fps,
|
||||
"total_num_frames": num_frames,
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"do_sample_frames": do_sample_frames,
|
||||
}
|
||||
return frames, metadata
|
||||
|
||||
|
||||
def _hf_video_kwargs(*videos):
|
||||
return {
|
||||
"videos": [frames for frames, _ in videos],
|
||||
"video_metadata": [
|
||||
{k: v for k, v in metadata.items() if k != "do_sample_frames"}
|
||||
for _, metadata in videos
|
||||
],
|
||||
"do_sample_frames": videos[0][1]["do_sample_frames"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_video_multimodal_processor(model_id):
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
|
||||
video = _video()
|
||||
frames, _ = video
|
||||
result = mm_processor(
|
||||
prompt=VIDEO_MODEL_SETTINGS[model_id]["prompt"],
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": video}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
hf_inputs = hf_processor(
|
||||
text=VIDEO_MODEL_SETTINGS[model_id]["prompt"],
|
||||
**_hf_video_kwargs(video),
|
||||
return_mm_token_type_ids=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist()
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["video"]
|
||||
(num_video_tokens,) = hf_processor._get_num_multimodal_tokens(
|
||||
video_sizes=[frames.shape[:3]]
|
||||
)["num_video_tokens"]
|
||||
assert placeholder.get_num_embeds() == num_video_tokens
|
||||
assert placeholder.get_num_embeds() == int(
|
||||
(hf_inputs["mm_token_type_ids"] == 2).sum()
|
||||
)
|
||||
|
||||
(item,) = result["mm_kwargs"]["video"]
|
||||
rows = item["pixel_values_videos"].data.shape[0]
|
||||
if "video_grid_thw" in item:
|
||||
assert rows == int(item["num_video_patches"].data)
|
||||
else:
|
||||
assert rows == len(frames)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_video_multiple_inputs(model_id):
|
||||
"""Multiple videos per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modeling backend."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video"},
|
||||
{"type": "video"},
|
||||
{"type": "text", "text": "Compare these videos."},
|
||||
],
|
||||
}
|
||||
]
|
||||
prompt = hf_processor.apply_chat_template(
|
||||
messages, add_generation_prompt=True, tokenize=False
|
||||
)
|
||||
videos = [_video(8, seed=0), _video(8, seed=1)]
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": videos}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["video"]) == 2
|
||||
assert len(result["mm_kwargs"]["video"]) == 2
|
||||
hf_inputs = hf_processor(
|
||||
text=prompt, **_hf_video_kwargs(*videos), return_tensors="pt"
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist()
|
||||
|
||||
|
||||
def test_video_fields_not_claimed_by_image():
|
||||
"""A prompt holding an image and a video keeps the fields of each on its own
|
||||
modality, so the image branch does not claim the video processor's outputs."""
|
||||
model_id = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt=(
|
||||
"<|vision_start|><|image_pad|><|vision_end|> and "
|
||||
"<|vision_start|><|video_pad|><|vision_end|> What do these show?"
|
||||
),
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": _video()[0][0], "video": _video()}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
(image_item,) = result["mm_kwargs"]["image"]
|
||||
(video_item,) = result["mm_kwargs"]["video"]
|
||||
assert {"pixel_values", "image_grid_thw"} <= set(image_item.keys())
|
||||
assert {"pixel_values_videos", "video_grid_thw"} <= set(video_item.keys())
|
||||
assert len(result["mm_placeholders"]["image"]) == 1
|
||||
assert len(result["mm_placeholders"]["video"]) == 1
|
||||
|
||||
|
||||
def test_repeated_video_hits_the_processor_cache():
|
||||
"""Check that mm caching is actually working."""
|
||||
model_config = ModelConfig(
|
||||
model="Qwen/Qwen3-VL-2B-Instruct", model_impl="transformers"
|
||||
)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
video = _video()
|
||||
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=VIDEO_MODEL_SETTINGS["Qwen/Qwen3-VL-2B-Instruct"]["prompt"],
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": video}),
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
first, second = process(), process()
|
||||
|
||||
assert cache.make_stats().hits > 0
|
||||
assert first["prompt_token_ids"] == second["prompt_token_ids"]
|
||||
assert first["mm_hashes"] == second["mm_hashes"]
|
||||
|
||||
|
||||
def test_video_unsupported_when_processor_cannot_count_tokens(monkeypatch):
|
||||
"""A processor that cannot count video tokens is served as image-only."""
|
||||
monkeypatch.delattr(Qwen3VLVideoProcessor, "get_num_of_video_patches")
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="Qwen/Qwen3-VL-2B-Instruct", model_impl="transformers")
|
||||
)
|
||||
|
||||
assert "video" not in mm_processor.info.get_supported_mm_limits()
|
||||
|
||||
|
||||
def test_merged_token_fields_split_per_video():
|
||||
"""VideoLLaMA3's compression mask has one row per merged token, not per patch."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="lkhl/VideoLLaMA3-2B-Image-HF", model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user\n<|video_pad|>\n<|video_pad|>\nDescribe.<|im_end|>\n",
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": [_video(8), _video(4)]}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
items = result["mm_kwargs"]["video"]
|
||||
assert [len(item["video_compression_mask"].data) for item in items] == [128, 64]
|
||||
|
||||
|
||||
def test_video_sampled_by_processor():
|
||||
"""Frames left for the processor to sample yield the tokens HF does."""
|
||||
model_id = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
prompt = VIDEO_MODEL_SETTINGS[model_id]["prompt"]
|
||||
video = _video(64, fps=30.0, do_sample_frames=True)
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"video": video}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
hf_inputs = hf_processor(
|
||||
text=prompt,
|
||||
**_hf_video_kwargs(video),
|
||||
return_mm_token_type_ids=True,
|
||||
return_tensors="pt",
|
||||
)
|
||||
|
||||
assert result["prompt_token_ids"] == hf_inputs["input_ids"][0].tolist()
|
||||
(placeholder,) = result["mm_placeholders"]["video"]
|
||||
assert placeholder.get_num_embeds() == int(
|
||||
(hf_inputs["mm_token_type_ids"] == 2).sum()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", list(VIDEO_MODEL_SETTINGS))
|
||||
def test_dummy_video_has_the_most_tokens(model_id):
|
||||
"""Profiling must build a dummy video with exactly the most tokens a video
|
||||
can produce, or memory is under- or over-reserved."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
info = mm_processor.info
|
||||
seq_len = info.ctx.model_config.max_model_len
|
||||
|
||||
result = mm_processor.get_dummy_mm_inputs({"video": 1})
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["video"]
|
||||
max_tokens = info.get_max_video_tokens(seq_len, {"video": 1})
|
||||
assert placeholder.get_num_embeds() == max_tokens
|
||||
# Shrinking the dummy frames must never cost tokens
|
||||
num_frames = info.get_num_frames_with_most_features(seq_len, {"video": 1})
|
||||
(huge_frame_tokens,) = info.get_hf_processor()._get_num_multimodal_tokens(
|
||||
video_sizes=[[num_frames, 10_000, 10_000]]
|
||||
)["num_video_tokens"]
|
||||
assert max_tokens >= huge_frame_tokens
|
||||
@@ -74,13 +74,14 @@ def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
|
||||
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
|
||||
slots[-1] = -1
|
||||
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
|
||||
qkv_width = 2 * (MAIN_HQ + MAIN_HK) * MAIN_D
|
||||
return dict(
|
||||
main_qkv=torch.randn(
|
||||
num_tokens,
|
||||
2 * (MAIN_HQ + MAIN_HK) * MAIN_D,
|
||||
qkv_width + (HQ + 1) * D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
),
|
||||
)[:, :qkv_width],
|
||||
main_q_norm_weight=norm_weights[0],
|
||||
main_k_norm_weight=norm_weights[1],
|
||||
main_eps=EPS,
|
||||
@@ -316,9 +317,12 @@ def test_qsa_fused_prepare_matches_unfused(
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
# TP=4 QKVG width makes the indexer slice row-strided, as in the merged
|
||||
# projection. The exact width is unimportant; the larger stride is tested.
|
||||
main_qkvg_width = 3584
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
num_tokens, main_qkvg_width + (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)[:, main_qkvg_width:]
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
|
||||
@@ -587,12 +587,14 @@ class MarkingStub(SupportsMultiModal, nn.Module):
|
||||
_pre_trained_model_classes = Base._pre_trained_model_classes
|
||||
|
||||
|
||||
def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
|
||||
def build_marked_model(
|
||||
image_limit: int, video_limit: int = 0, skip_tokenizer_init: bool = False
|
||||
):
|
||||
"""Build the HF model inside the marking context and return it."""
|
||||
model_config = ModelConfig(
|
||||
model=MULTIMODAL_MODEL,
|
||||
model_impl="transformers",
|
||||
limit_mm_per_prompt={"image": image_limit},
|
||||
limit_mm_per_prompt={"image": image_limit, "video": video_limit},
|
||||
)
|
||||
# Set after construction: building the config itself needs the tokenizer
|
||||
model_config.skip_tokenizer_init = skip_tokenizer_init
|
||||
@@ -606,10 +608,15 @@ def build_marked_model(image_limit: int, skip_tokenizer_init: bool = False):
|
||||
return stub.model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("image_limit", "skipped"), [(0, True), (4, False)])
|
||||
def test_tower_weights_skipped_when_modality_disabled(image_limit, skipped):
|
||||
"""`--limit-mm-per-prompt image=0` should drop the vision tower's weights."""
|
||||
vision_tower = build_marked_model(image_limit).vision_tower
|
||||
@pytest.mark.parametrize(
|
||||
("image_limit", "video_limit", "skipped"),
|
||||
[(0, 0, True), (4, 0, False), (0, 4, False)],
|
||||
)
|
||||
def test_tower_weights_skipped_when_modality_disabled(
|
||||
image_limit, video_limit, skipped
|
||||
):
|
||||
"""Only `--limit-mm-per-prompt image=0,video=0` drops the tower they share."""
|
||||
vision_tower = build_marked_model(image_limit, video_limit).vision_tower
|
||||
assert isinstance(vision_tower, StageMissingLayer) is skipped
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user