[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,558 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Set as AbstractSet
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.models.utils import build_model_context
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ....multimodal.utils import random_audio, random_image, random_video
|
||||
from ...registry import (
|
||||
_MULTIMODAL_EXAMPLE_MODELS,
|
||||
_TRANSFORMERS_BACKEND_MODELS,
|
||||
HF_EXAMPLE_MODELS,
|
||||
)
|
||||
|
||||
|
||||
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
"""Add metadata to video mm_data."""
|
||||
|
||||
def create_metadata(frames: np.ndarray):
|
||||
num_frames = len(frames)
|
||||
return {
|
||||
"total_num_frames": num_frames,
|
||||
"fps": 2.0,
|
||||
"duration": num_frames / 2.0,
|
||||
"video_backend": "opencv",
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"do_sample_frames": True,
|
||||
}
|
||||
|
||||
# Ensure video metadata is included
|
||||
if "video" in mm_data:
|
||||
video = mm_data["video"]
|
||||
if isinstance(video, list):
|
||||
# multiple videos
|
||||
mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
|
||||
else:
|
||||
# single video
|
||||
mm_data["video"] = (video, create_metadata(video))
|
||||
return mm_data
|
||||
|
||||
|
||||
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
"""Patch the multimodal data for GLM-ASR model.
|
||||
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
|
||||
"""
|
||||
if "audio" in mm_data:
|
||||
audio = mm_data["audio"]
|
||||
if isinstance(audio, list) and len(audio) > 1:
|
||||
# Limit to single audio to match text requirement
|
||||
mm_data["audio"] = [audio[0]]
|
||||
return mm_data
|
||||
|
||||
|
||||
_IGNORE_MM_KEYS = {
|
||||
# Dithering causes minor divergence
|
||||
"cohere_asr": {"input_features"},
|
||||
# In Ultravox, the audio_features can be different depending on padding
|
||||
# The slight difference should not be a problem though, since
|
||||
# attention_mask lets us ignore the difference.
|
||||
"ultravox": {"audio_features"},
|
||||
}
|
||||
|
||||
MM_DATA_PATCHES = {
|
||||
"glmasr": glmasr_patch_mm_data,
|
||||
}
|
||||
|
||||
_XPU_EXCLUDED_MODEL_IDS = {
|
||||
"baidu/Unlimited-OCR",
|
||||
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
|
||||
"moonshotai/Kimi-K3",
|
||||
"Qwen/Qwen2.5-Omni-7B-AWQ",
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
}
|
||||
|
||||
_CPU_EXCLUDED_MODEL_IDS = {
|
||||
# DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now.
|
||||
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
|
||||
}
|
||||
|
||||
|
||||
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
|
||||
for model_arch in model_arch_list:
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
|
||||
yield model_info.default
|
||||
|
||||
for extra_type, extra_model_id in model_info.extras.items():
|
||||
if "fp" in extra_type:
|
||||
continue # Redundant to test quantized models
|
||||
|
||||
yield extra_model_id
|
||||
|
||||
|
||||
def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
|
||||
model_ids = list(_iter_model_ids_to_test(model_arch_list))
|
||||
|
||||
if current_platform.is_xpu():
|
||||
for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS:
|
||||
while excluded_model_id in model_ids:
|
||||
model_ids.remove(excluded_model_id)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS:
|
||||
while excluded_model_id in model_ids:
|
||||
model_ids.remove(excluded_model_id)
|
||||
|
||||
return model_ids
|
||||
|
||||
|
||||
def get_model_ids_to_test():
|
||||
transformers_arch_ids = {
|
||||
model_id
|
||||
for info in _TRANSFORMERS_BACKEND_MODELS.values()
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
}
|
||||
vllm_only_archs = {
|
||||
arch
|
||||
for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
|
||||
if not any(
|
||||
model_id in transformers_arch_ids
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
)
|
||||
}
|
||||
|
||||
return _get_model_ids_to_test(vllm_only_archs)
|
||||
|
||||
|
||||
def get_transformers_backend_model_ids_to_test():
|
||||
return sorted(
|
||||
{
|
||||
model_id
|
||||
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
|
||||
if "MultiModal" in arch
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_token_prompt(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> list[int]:
|
||||
tokenizer: TokenizerLike = processor.info.get_tokenizer()
|
||||
model_config = processor.info.ctx.model_config
|
||||
|
||||
if processor.info.data_parser.video_needs_metadata:
|
||||
mm_data = add_video_metadata(mm_data)
|
||||
|
||||
model_type = model_config.hf_config.model_type
|
||||
if model_type in MM_DATA_PATCHES:
|
||||
mm_data = MM_DATA_PATCHES[model_type](mm_data)
|
||||
|
||||
parsed_data = processor.info.parse_mm_data(mm_data)
|
||||
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
|
||||
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
inputs = processor.get_dummy_inputs(
|
||||
model_config.max_model_len,
|
||||
mm_counts,
|
||||
mm_options=MultiModalDummyOptions(),
|
||||
# Assume all Mistral models define this extra argument
|
||||
mm_data=mm_data, # type: ignore[call-arg]
|
||||
)
|
||||
else:
|
||||
inputs = processor.get_dummy_inputs(
|
||||
model_config.max_model_len,
|
||||
mm_counts,
|
||||
mm_options=MultiModalDummyOptions(),
|
||||
)
|
||||
|
||||
if not isinstance(inputs.prompt, list):
|
||||
raise TypeError(type(inputs.prompt))
|
||||
|
||||
return inputs.prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,durations",
|
||||
[
|
||||
("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]),
|
||||
("nvidia/audio-flamingo-3-hf", [31.0]),
|
||||
("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]),
|
||||
("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]),
|
||||
("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]),
|
||||
],
|
||||
)
|
||||
def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
|
||||
"""EC metadata preserves the raw-audio prompt without running the HF processor."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)})
|
||||
mm_config = ctx.model_config.multimodal_config
|
||||
mm_config.enable_mm_embeds = True
|
||||
mm_config.allow_missing_mm_embeddings = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
sample_rate = processor.info.get_feature_extractor().sampling_rate
|
||||
audios = [
|
||||
np.zeros(int(duration * sample_rate), dtype=np.float32)
|
||||
for duration in durations
|
||||
]
|
||||
prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)})
|
||||
raw = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
features = [
|
||||
MultiModalFeatureSpec(
|
||||
data=item,
|
||||
modality="audio",
|
||||
identifier=str(index),
|
||||
mm_position=position,
|
||||
)
|
||||
for index, (item, position) in enumerate(
|
||||
zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"])
|
||||
)
|
||||
]
|
||||
published = collect_ec_item_metadata(
|
||||
features, PlaceholderMetadataResolver(ctx.model_config)
|
||||
)
|
||||
metadata = json.loads(
|
||||
json.dumps([entry["metadata"] for entry in published.values()])
|
||||
)
|
||||
assert len(metadata) == len(audios)
|
||||
assert all(metadata)
|
||||
audio_data = _get_embeds_data("audio", metadata, processor)
|
||||
|
||||
def unexpected_preprocessing(*args, **kwargs):
|
||||
pytest.fail("The EC consumer must not preprocess audio")
|
||||
|
||||
monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing)
|
||||
remote = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio_data}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert remote["prompt_token_ids"] == raw["prompt_token_ids"]
|
||||
assert remote["mm_placeholders"] == raw["mm_placeholders"]
|
||||
# Omni also consumes these lengths when constructing M-RoPE positions.
|
||||
if "audio_feature_lengths" in audio_data:
|
||||
assert torch.equal(
|
||||
remote["mm_kwargs"].get_data()["audio_feature_lengths"],
|
||||
raw["mm_kwargs"].get_data()["audio_feature_lengths"],
|
||||
)
|
||||
|
||||
mm_config.allow_missing_mm_embeddings = False
|
||||
ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
with pytest.raises(ValueError, match="should contain the fields"):
|
||||
ordinary_processor.info.parse_mm_data({"audio": audio_data})
|
||||
|
||||
|
||||
def random_vision_chunk(
|
||||
rng: np.random.RandomState,
|
||||
min_wh: int,
|
||||
max_wh: int,
|
||||
min_frames: int,
|
||||
max_frames: int,
|
||||
) -> dict:
|
||||
num_frames = rng.randint(min_frames, max_frames + 1)
|
||||
if num_frames == 1:
|
||||
# Single image chunk
|
||||
wh = rng.randint(min_wh, max_wh + 1)
|
||||
image = random_image(rng, wh, wh + 1)
|
||||
return {"type": "image", "image": image}
|
||||
frames = []
|
||||
for _ in range(num_frames):
|
||||
wh = rng.randint(min_wh, max_wh + 1)
|
||||
frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
|
||||
frames.append(frame)
|
||||
video_array = np.stack(frames, axis=0)
|
||||
return {"type": "video_chunk", "video_chunk": video_array}
|
||||
|
||||
|
||||
def _test_processing_correctness(
|
||||
model_id_or_arch: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
model_impl: str = "auto",
|
||||
):
|
||||
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
|
||||
# Use model architecture to get the default model id
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
|
||||
model_id = model_info.default
|
||||
else:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
|
||||
model_id = model_id_or_arch
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=model_info.dtype,
|
||||
model_impl=model_impl,
|
||||
)
|
||||
# Ensure that the cache can fit all of the data
|
||||
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 2048
|
||||
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
factories = model_cls._processor_factory
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
|
||||
processing_info = factories.info(ctx)
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
# Keep integer limits for local data generation
|
||||
limit_mm_per_prompt_ints = {
|
||||
modality: 3 if limit is None else limit
|
||||
for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
|
||||
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
|
||||
if modality == "video":
|
||||
return VideoDummyOptions(count=count)
|
||||
if modality == "image":
|
||||
return ImageDummyOptions(count=count)
|
||||
if modality == "audio":
|
||||
return AudioDummyOptions(count=count)
|
||||
return BaseDummyOptions(count=count)
|
||||
|
||||
# Assign normalized DummyOptions to the model config
|
||||
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
|
||||
{
|
||||
modality: _to_dummy_options(modality, count)
|
||||
for modality, count in limit_mm_per_prompt_ints.items()
|
||||
}
|
||||
)
|
||||
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
# GLM-ASR requires a minimum audio length of 70ms
|
||||
min_audio_len = 512 if model_config.hf_config.model_type != "glmasr" else 1120
|
||||
input_to_hit = {
|
||||
"image": Image.new("RGB", size=(128, 128)),
|
||||
"video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
|
||||
"audio": (np.zeros((min_audio_len,)), 16000),
|
||||
"vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
|
||||
}
|
||||
input_factory = {
|
||||
"image": partial(random_image, rng, min_wh=128, max_wh=256),
|
||||
"video": partial(
|
||||
random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
|
||||
),
|
||||
"audio": partial(
|
||||
random_audio,
|
||||
rng,
|
||||
min_len=min_audio_len,
|
||||
max_len=min_audio_len + 512,
|
||||
sr=16000,
|
||||
),
|
||||
"vision_chunk": partial(
|
||||
random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
|
||||
),
|
||||
}
|
||||
|
||||
for batch_idx in range(num_batches):
|
||||
mm_data = {
|
||||
k: [
|
||||
(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
|
||||
for _ in range(rng.randint(limit + 1))
|
||||
]
|
||||
for k, limit in limit_mm_per_prompt_ints.items()
|
||||
}
|
||||
|
||||
# Drop unnecessary keys and test single -> multi conversion
|
||||
if rng.rand() < simplify_rate:
|
||||
for k in list(mm_data.keys()):
|
||||
if not mm_data[k]:
|
||||
del mm_data[k]
|
||||
elif len(mm_data[k]) == 1:
|
||||
mm_data[k] = mm_data[k][0]
|
||||
|
||||
_test_processing_correctness_one(
|
||||
model_config,
|
||||
mm_data,
|
||||
processor,
|
||||
batch_idx,
|
||||
hit_rate,
|
||||
num_batches,
|
||||
simplify_rate,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
|
||||
def _test_processing_correctness_one(
|
||||
model_config: ModelConfig,
|
||||
mm_data: MultiModalDataDict,
|
||||
processor: BaseMultiModalProcessor,
|
||||
batch_idx: int,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
cache: MultiModalProcessorOnlyCache,
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
token_prompt = get_token_prompt(processor, mm_data)
|
||||
mm_items = processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
baseline_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
cached_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_tokenized_result,
|
||||
cached_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_processing_correctness(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
if model_id == "openvla/openvla-7b":
|
||||
pytest.skip(
|
||||
"OpenVLA uses a custom vLLM processor because its HF remote "
|
||||
"processor is incompatible with current Transformers."
|
||||
)
|
||||
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
||||
pytest.skip(
|
||||
"Voxtral Realtime doesn't make use of any place-holder "
|
||||
"tokens and hence cannot pass the processing "
|
||||
"correctness test as is. Let's revisit adapting this "
|
||||
"test once more realtime models exist."
|
||||
)
|
||||
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
|
||||
pytest.skip(
|
||||
"MOSS-Audio uses a custom processor that dynamically expands "
|
||||
"audio placeholders from processed audio lengths. Its vLLM "
|
||||
"processor paths are covered by test_moss_audio.py."
|
||||
)
|
||||
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
|
||||
pytest.skip(
|
||||
"LLaVA-OneVision-2 video processing routes frames through custom "
|
||||
"video backends (qwen_vl_utils / codec) that require real encoded "
|
||||
"video bytes and metadata. The synthetic numpy-array videos used by "
|
||||
"this test yield empty video features, so the generic correctness "
|
||||
"check cannot exercise the video path. Image processing is covered "
|
||||
"by registration/inference tests."
|
||||
)
|
||||
|
||||
_test_processing_correctness(
|
||||
model_id,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_processing_correctness_transformers(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
_test_processing_correctness(
|
||||
model_id,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
|
||||
def _assert_inputs_equal(
|
||||
a: MultiModalInput,
|
||||
b: MultiModalInput,
|
||||
*,
|
||||
ignore_mm_keys: set[str] | None = None,
|
||||
msg: str = "",
|
||||
):
|
||||
if ignore_mm_keys is None:
|
||||
ignore_mm_keys = set()
|
||||
|
||||
ignore_prompt_keys = ("prompt", "mm_kwargs")
|
||||
a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}
|
||||
b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}
|
||||
|
||||
assert a_rest == b_rest, msg
|
||||
|
||||
a_data = a["mm_kwargs"].get_data()
|
||||
b_data = b["mm_kwargs"].get_data()
|
||||
|
||||
for key in ignore_mm_keys:
|
||||
a_data.pop(key, None)
|
||||
b_data.pop(key, None)
|
||||
|
||||
assert batched_tensors_equal(a_data, b_data), msg
|
||||
@@ -1,241 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Video placeholder accounting for GLM-5.3-Flash.
|
||||
|
||||
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
|
||||
placeholders per timestamp, so the timestamps returned by
|
||||
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
|
||||
while ``video_grid_thw`` decides how many rows the vision tower produces. If
|
||||
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
|
||||
inside a worker and takes the engine down with it.
|
||||
|
||||
The checks below are arithmetic: no weights, no GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.models.glm5_next.video_processing_glm5_next import (
|
||||
Glm5NextVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
|
||||
def _pixel_path_grid(
|
||||
video_processor: Glm5NextVideoProcessor,
|
||||
num_frames: int,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
|
||||
resized_height, resized_width = smart_resize(
|
||||
num_frames=num_frames,
|
||||
height=height,
|
||||
width=width,
|
||||
temporal_factor=video_processor.temporal_patch_size,
|
||||
factor=video_processor.patch_size
|
||||
* video_processor.merge_size
|
||||
* video_processor.patch_expand_factor,
|
||||
min_pixels=video_processor.min_image_tokens,
|
||||
max_pixels=video_processor.max_image_tokens,
|
||||
)
|
||||
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
|
||||
return (
|
||||
padded_frames // video_processor.temporal_patch_size,
|
||||
resized_height // video_processor.patch_size,
|
||||
resized_width // video_processor.patch_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
|
||||
[
|
||||
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
|
||||
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
|
||||
# 1080p, 20 s: same factor of 3 at a full-size canvas.
|
||||
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
|
||||
# 1080p, 60 s: the one duration window where the two samplers agree
|
||||
# anyway -- a regression guard, the count must not move.
|
||||
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
|
||||
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
|
||||
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
|
||||
],
|
||||
)
|
||||
def test_video_placeholders_match_encoder_rows(
|
||||
processor,
|
||||
total_num_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
height: int,
|
||||
width: int,
|
||||
expected_grid: tuple[int, int, int],
|
||||
):
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
frame_indices = video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
|
||||
)
|
||||
grid_t, grid_h, grid_w = _pixel_path_grid(
|
||||
video_processor, len(frame_indices), height, width
|
||||
)
|
||||
assert (grid_t, grid_h, grid_w) == expected_grid
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": total_num_frames,
|
||||
"fps": fps,
|
||||
"duration": duration,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
total_num_frames,
|
||||
)
|
||||
|
||||
merge_length = video_processor.merge_size**2
|
||||
tokens_per_frame = grid_h * grid_w // merge_length
|
||||
encoder_rows = grid_t * grid_h * grid_w // merge_length
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
assert len(timestamps) * tokens_per_frame == encoder_rows
|
||||
assert timestamps == sorted(timestamps)
|
||||
assert timestamps[0] == 0
|
||||
assert timestamps[-1] <= duration
|
||||
|
||||
|
||||
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
|
||||
"""The loader may pre-sample and hand the frames over as they are."""
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
num_frames = 32
|
||||
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 256,
|
||||
"fps": 8.0,
|
||||
"duration": 32.0,
|
||||
"do_sample_frames": False,
|
||||
"frames_indices": list(range(0, 256, 256 // num_frames)),
|
||||
},
|
||||
num_frames,
|
||||
)
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
|
||||
|
||||
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
|
||||
"""A clip the sampler cannot pick a single frame from is a bad request."""
|
||||
with pytest.raises(ValueError, match="selected no frames"):
|
||||
processor.info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 2,
|
||||
"fps": 8.0,
|
||||
"duration": 0.25,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
)
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (310, 470), color=(17, 89, 231)),
|
||||
Image.new("RGB", (480, 320), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def _image_info(**kwargs):
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
**kwargs,
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
).info
|
||||
|
||||
|
||||
def test_image_encoder_cache_covers_full_token_budget():
|
||||
"""The most-features probe must reach the processor's token ceiling.
|
||||
|
||||
The inherited square probe refits to 89x89 = 7921 tokens under the
|
||||
max_image_tokens=8000 budget, so the encoder cache came up short and
|
||||
ordinary non-square images in 7922-8000 tokens were refused with
|
||||
HTTP 400 (#59539).
|
||||
"""
|
||||
info = _image_info()
|
||||
assert info.get_max_image_tokens() == 8000
|
||||
|
||||
# The shapes from the issue, with the token counts the processor
|
||||
# actually produces; every one must fit the cache.
|
||||
for width, height, expected_tokens in [
|
||||
(4032, 3024, 7931), # phone photo, 4:3
|
||||
(3840, 2160, 7973), # 4K frame, 16:9
|
||||
(3508, 2480, 7950), # A4 at 300 dpi
|
||||
(2600, 2400, 7998), # 13:12
|
||||
(3000, 3000, 7921), # square worst case before the fix
|
||||
]:
|
||||
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
|
||||
assert num_tokens == expected_tokens
|
||||
assert num_tokens <= info.get_max_image_tokens()
|
||||
|
||||
# The profiling dummy covers the real worst case, not the square one.
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert size.width * size.height == 2240 * 2800
|
||||
|
||||
|
||||
def test_image_encoder_cache_follows_max_pixels_override():
|
||||
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
|
||||
assert info.get_max_image_tokens() == 100
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert (
|
||||
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
|
||||
== 100
|
||||
)
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoImageProcessor
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_MODEL_ID = "moonshotai/Kimi-K3"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_processors():
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
k25_fused = processor.info.image_processor
|
||||
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
|
||||
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
|
||||
return k25_fused, k3_hf
|
||||
|
||||
|
||||
def _noise_image(width: int, height: int) -> Image.Image:
|
||||
rng = np.random.default_rng(0)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("width", "height"),
|
||||
[(310, 470), (7300, 30), (13, 9)],
|
||||
)
|
||||
def test_fused_image_processor_matches_hf(image_processors, width: int, height: int):
|
||||
fused, hf = image_processors
|
||||
image = _noise_image(width, height)
|
||||
expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt")
|
||||
actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt")
|
||||
|
||||
torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"])
|
||||
torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"])
|
||||
assert fused.media_tokens_calculator(
|
||||
{"type": "image", "image": image}
|
||||
) == hf.media_tokens_calculator({"type": "image", "image": image})
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2})
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (310, 470), color=(17, 89, 231)),
|
||||
Image.new("RGB", (480, 320), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = processor.info.get_hf_config().image_placeholder * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Nemotron-Parse's device-side normalisation path."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_MODEL_ID = "nvidia/NVIDIA-Nemotron-Parse-2.0"
|
||||
|
||||
|
||||
def _noise_image(width: int, height: int) -> Image.Image:
|
||||
rng = np.random.default_rng(0)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(("width", "height"), [(1240, 1754), (310, 470)])
|
||||
def test_mm_device_do_normalize(width: int, height: int):
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
images = [_noise_image(width, height)]
|
||||
prompt = [0]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
|
||||
"pixel_values"
|
||||
]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
|
||||
"pixel_values"
|
||||
]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
assert raw_values.shape == normalized_values.shape
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [False, True])
|
||||
@pytest.mark.parametrize("layout", ["ndarray_hwc", "tensor_hwc", "tensor_chw"])
|
||||
def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = mm_device_do_normalize
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = _noise_image(310, 470)
|
||||
array = np.asarray(image)
|
||||
item = {
|
||||
"ndarray_hwc": array,
|
||||
"tensor_hwc": torch.from_numpy(array),
|
||||
"tensor_chw": torch.from_numpy(array).permute(2, 0, 1),
|
||||
}[layout]
|
||||
|
||||
def pixel_values(img: object) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [img]})
|
||||
return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_explicit_kwargs_without_device_normalize():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_kwargs={"do_rescale": False, "do_normalize": False},
|
||||
)
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
|
||||
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
|
||||
# The remote processor ignores these kwargs and still normalises on the host.
|
||||
assert values.is_floating_point()
|
||||
@@ -1,272 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import tempfile
|
||||
from collections.abc import Iterable
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import supports_multimodal
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.multimodal.utils import group_and_batch_mm_kwargs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
from ....utils import create_new_process_for_each_test
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import dummy_hf_overrides
|
||||
from .test_common import get_model_ids_to_test, get_token_prompt
|
||||
|
||||
ImageInput = list[Image.Image]
|
||||
VideoInput: TypeAlias = (
|
||||
list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]]
|
||||
)
|
||||
AudioInput = list[tuple[np.ndarray, int]]
|
||||
|
||||
|
||||
def _resize_data(
|
||||
_data: Image.Image | np.ndarray, size_factor: float
|
||||
) -> Image.Image | np.ndarray:
|
||||
assert size_factor <= 1, "Size factor must be less than 1"
|
||||
# Image input
|
||||
if isinstance(_data, Image.Image):
|
||||
W, H = _data.width, _data.height
|
||||
W, H = map(lambda x: int(x * size_factor), (W, H))
|
||||
return _data.resize((W, H))
|
||||
# Video input with PIL Images
|
||||
elif is_list_of(_data, Image.Image):
|
||||
W, H = next(iter(_data)).width, next(iter(_data)).height
|
||||
T = len(_data)
|
||||
T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H))
|
||||
return [d.resize((W, H)) for d in _data[:T]]
|
||||
# Video input with numpy arrays
|
||||
elif isinstance(_data, np.ndarray) and _data.ndim >= 4:
|
||||
T, H, W, C = _data.shape[-4:]
|
||||
T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W))
|
||||
return _data[..., :T, :H, :W, :C]
|
||||
# Audio input
|
||||
elif isinstance(_data, np.ndarray) and _data.ndim == 1:
|
||||
return _data[: int(len(_data) * size_factor)]
|
||||
raise AssertionError("This line should be unreachable.")
|
||||
|
||||
|
||||
def resize_mm_data(
|
||||
data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...]
|
||||
) -> ImageInput | VideoInput | AudioInput:
|
||||
size_factors = size_factors[: len(data)]
|
||||
if is_list_of(data, (Image.Image, np.ndarray, list)):
|
||||
return [_resize_data(d, s) for d, s in zip(data, size_factors)]
|
||||
elif is_list_of(data, tuple):
|
||||
return [_resize_data(d, s) for (d, _), s in zip(data, size_factors)]
|
||||
raise ValueError("Unsupported multimodal data type.")
|
||||
|
||||
|
||||
def create_batched_mm_kwargs(
|
||||
model_config: ModelConfig,
|
||||
processor: BaseMultiModalProcessor,
|
||||
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
|
||||
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
|
||||
processing_info = processor.info
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
mm_counts = {
|
||||
modality: 3 if limit is None else limit
|
||||
for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
processor_inputs = processor.get_dummy_inputs(
|
||||
seq_len=model_config.max_model_len,
|
||||
mm_counts=mm_counts,
|
||||
mm_options=MultiModalDummyOptions(),
|
||||
)
|
||||
mm_items = processor_inputs.mm_data_items
|
||||
resized_mm_data = {
|
||||
modality: resize_mm_data(items.data, size_factors)
|
||||
for modality, items in mm_items.items()
|
||||
}
|
||||
|
||||
# video metadata will be added back to the resized video data here.
|
||||
token_prompt = get_token_prompt(processor, resized_mm_data)
|
||||
|
||||
mm_kwargs = processor(
|
||||
prompt=token_prompt,
|
||||
mm_items=processor.info.parse_mm_data(resized_mm_data),
|
||||
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
|
||||
)["mm_kwargs"].require_data()
|
||||
|
||||
return group_and_batch_mm_kwargs(
|
||||
[
|
||||
(modality, item)
|
||||
for modality in supported_mm_limits
|
||||
for item in mm_kwargs[modality]
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# TODO(Isotr0py): Don't initialize model during test
|
||||
@contextmanager
|
||||
def initialize_dummy_model(
|
||||
model_cls: type[nn.Module],
|
||||
model_config: ModelConfig,
|
||||
):
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
current_device = torch.get_default_device()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config, cache_config=CacheConfig(block_size=16)
|
||||
)
|
||||
with set_current_vllm_config(vllm_config=vllm_config):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=0,
|
||||
backend="nccl",
|
||||
)
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
|
||||
with set_default_torch_dtype(model_config.dtype):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
model = model_cls(vllm_config=vllm_config)
|
||||
torch.set_default_device(current_device)
|
||||
yield model
|
||||
|
||||
del model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
|
||||
def test_model_tensor_schema(model_id: str):
|
||||
if model_id == "moonshotai/Kimi-K2.5":
|
||||
# FIXME(Isotr0py): Fix Kimi-K2.5's offline inference about vision chunks.
|
||||
pytest.skip(
|
||||
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
|
||||
)
|
||||
|
||||
if (
|
||||
model_id == "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
|
||||
and not current_platform.is_cuda_alike()
|
||||
):
|
||||
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
|
||||
|
||||
if model_id == "zai-org/GLM-5.3-Flash" and (current_platform.is_xpu()):
|
||||
pytest.skip("GLM-5.3-Flash is not supported on XPU")
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_arch = next(
|
||||
arch for arch, info in HF_EXAMPLE_MODELS.hf_models.items() if info == model_info
|
||||
)
|
||||
|
||||
hf_overrides_fn = partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch=model_arch,
|
||||
exist_overrides=model_info.hf_overrides,
|
||||
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
|
||||
)
|
||||
|
||||
# ROCm: Detect if model uses AWQ quantization and set appropriate dtype
|
||||
if "awq" in model_id.lower() and current_platform.is_rocm():
|
||||
dtype = "float16"
|
||||
else:
|
||||
dtype = model_info.dtype
|
||||
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
hf_overrides=hf_overrides_fn,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
assert supports_multimodal(model_cls)
|
||||
|
||||
factories = model_cls._processor_factory
|
||||
|
||||
# Capture helpers return capture-buffer containers (e.g.
|
||||
# EncoderCudaGraphCaptureInputs), not TensorSchema mm inputs.
|
||||
capture_helpers = {"prepare_encoder_cudagraph_capture_inputs"}
|
||||
|
||||
inputs_parse_methods = []
|
||||
for attr_name in dir(model_cls):
|
||||
if attr_name in capture_helpers:
|
||||
continue
|
||||
attr = getattr(model_cls, attr_name)
|
||||
if hasattr(attr, "__annotations__"):
|
||||
return_type = attr.__annotations__.get("return", None)
|
||||
if return_type is not None and "Input" in str(return_type):
|
||||
inputs_parse_methods.append(attr_name)
|
||||
|
||||
if not any(inputs_parse_methods):
|
||||
pytest.skip(f"{model_arch} does not support tensor schema validation.")
|
||||
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
processing_info = factories.info(ctx)
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
limit_mm_per_prompt = {
|
||||
modality: 3 if limit is None else limit
|
||||
for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
|
||||
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
|
||||
if modality == "video":
|
||||
return VideoDummyOptions(count=count)
|
||||
if modality == "image":
|
||||
return ImageDummyOptions(count=count)
|
||||
if modality == "audio":
|
||||
return AudioDummyOptions(count=count)
|
||||
return BaseDummyOptions(count=count)
|
||||
|
||||
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
|
||||
{
|
||||
modality: _to_dummy_options(modality, count)
|
||||
for modality, count in limit_mm_per_prompt.items()
|
||||
}
|
||||
)
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
with initialize_dummy_model(model_cls, model_config) as model:
|
||||
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
|
||||
for method_name in inputs_parse_methods:
|
||||
print(
|
||||
f"Testing `{method_name}` with modality={modality} "
|
||||
f"and mm_kwargs{list(mm_kwargs.keys())}"
|
||||
)
|
||||
getattr(model, method_name)(modality=modality, **mm_kwargs)
|
||||
Reference in New Issue
Block a user