[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,558 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Set as AbstractSet
from functools import partial
import numpy as np
import pytest
import torch
from PIL import Image
from tests.models.utils import build_model_context
from vllm.config import ModelConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.platforms import current_platform
from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.utils.mistral import is_mistral_tokenizer
from ....multimodal.utils import random_audio, random_image, random_video
from ...registry import (
_MULTIMODAL_EXAMPLE_MODELS,
_TRANSFORMERS_BACKEND_MODELS,
HF_EXAMPLE_MODELS,
)
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""Add metadata to video mm_data."""
def create_metadata(frames: np.ndarray):
num_frames = len(frames)
return {
"total_num_frames": num_frames,
"fps": 2.0,
"duration": num_frames / 2.0,
"video_backend": "opencv",
"frames_indices": list(range(num_frames)),
"do_sample_frames": True,
}
# Ensure video metadata is included
if "video" in mm_data:
video = mm_data["video"]
if isinstance(video, list):
# multiple videos
mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
else:
# single video
mm_data["video"] = (video, create_metadata(video))
return mm_data
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""Patch the multimodal data for GLM-ASR model.
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
"""
if "audio" in mm_data:
audio = mm_data["audio"]
if isinstance(audio, list) and len(audio) > 1:
# Limit to single audio to match text requirement
mm_data["audio"] = [audio[0]]
return mm_data
_IGNORE_MM_KEYS = {
# Dithering causes minor divergence
"cohere_asr": {"input_features"},
# In Ultravox, the audio_features can be different depending on padding
# The slight difference should not be a problem though, since
# attention_mask lets us ignore the difference.
"ultravox": {"audio_features"},
}
MM_DATA_PATCHES = {
"glmasr": glmasr_patch_mm_data,
}
_XPU_EXCLUDED_MODEL_IDS = {
"baidu/Unlimited-OCR",
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
"moonshotai/Kimi-K3",
"Qwen/Qwen2.5-Omni-7B-AWQ",
"thinkingmachines/Inkling-NVFP4",
}
_CPU_EXCLUDED_MODEL_IDS = {
# DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now.
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
}
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
for model_arch in model_arch_list:
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
yield model_info.default
for extra_type, extra_model_id in model_info.extras.items():
if "fp" in extra_type:
continue # Redundant to test quantized models
yield extra_model_id
def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
model_ids = list(_iter_model_ids_to_test(model_arch_list))
if current_platform.is_xpu():
for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS:
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
if current_platform.is_cpu():
for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS:
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
return model_ids
def get_model_ids_to_test():
transformers_arch_ids = {
model_id
for info in _TRANSFORMERS_BACKEND_MODELS.values()
for model_id in (info.default, *info.extras.values())
}
vllm_only_archs = {
arch
for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
if not any(
model_id in transformers_arch_ids
for model_id in (info.default, *info.extras.values())
)
}
return _get_model_ids_to_test(vllm_only_archs)
def get_transformers_backend_model_ids_to_test():
return sorted(
{
model_id
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
if "MultiModal" in arch
for model_id in (info.default, *info.extras.values())
}
)
def get_token_prompt(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
) -> list[int]:
tokenizer: TokenizerLike = processor.info.get_tokenizer()
model_config = processor.info.ctx.model_config
if processor.info.data_parser.video_needs_metadata:
mm_data = add_video_metadata(mm_data)
model_type = model_config.hf_config.model_type
if model_type in MM_DATA_PATCHES:
mm_data = MM_DATA_PATCHES[model_type](mm_data)
parsed_data = processor.info.parse_mm_data(mm_data)
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
if is_mistral_tokenizer(tokenizer):
inputs = processor.get_dummy_inputs(
model_config.max_model_len,
mm_counts,
mm_options=MultiModalDummyOptions(),
# Assume all Mistral models define this extra argument
mm_data=mm_data, # type: ignore[call-arg]
)
else:
inputs = processor.get_dummy_inputs(
model_config.max_model_len,
mm_counts,
mm_options=MultiModalDummyOptions(),
)
if not isinstance(inputs.prompt, list):
raise TypeError(type(inputs.prompt))
return inputs.prompt
@pytest.mark.parametrize(
"model_id,durations",
[
("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]),
("nvidia/audio-flamingo-3-hf", [31.0]),
("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]),
("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]),
("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]),
],
)
def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
"""EC metadata preserves the raw-audio prompt without running the HF processor."""
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)})
mm_config = ctx.model_config.multimodal_config
mm_config.enable_mm_embeds = True
mm_config.allow_missing_mm_embeddings = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
sample_rate = processor.info.get_feature_extractor().sampling_rate
audios = [
np.zeros(int(duration * sample_rate), dtype=np.float32)
for duration in durations
]
prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)})
raw = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
features = [
MultiModalFeatureSpec(
data=item,
modality="audio",
identifier=str(index),
mm_position=position,
)
for index, (item, position) in enumerate(
zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"])
)
]
published = collect_ec_item_metadata(
features, PlaceholderMetadataResolver(ctx.model_config)
)
metadata = json.loads(
json.dumps([entry["metadata"] for entry in published.values()])
)
assert len(metadata) == len(audios)
assert all(metadata)
audio_data = _get_embeds_data("audio", metadata, processor)
def unexpected_preprocessing(*args, **kwargs):
pytest.fail("The EC consumer must not preprocess audio")
monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing)
remote = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audio_data}),
hf_processor_mm_kwargs={},
)
assert remote["prompt_token_ids"] == raw["prompt_token_ids"]
assert remote["mm_placeholders"] == raw["mm_placeholders"]
# Omni also consumes these lengths when constructing M-RoPE positions.
if "audio_feature_lengths" in audio_data:
assert torch.equal(
remote["mm_kwargs"].get_data()["audio_feature_lengths"],
raw["mm_kwargs"].get_data()["audio_feature_lengths"],
)
mm_config.allow_missing_mm_embeddings = False
ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
with pytest.raises(ValueError, match="should contain the fields"):
ordinary_processor.info.parse_mm_data({"audio": audio_data})
def random_vision_chunk(
rng: np.random.RandomState,
min_wh: int,
max_wh: int,
min_frames: int,
max_frames: int,
) -> dict:
num_frames = rng.randint(min_frames, max_frames + 1)
if num_frames == 1:
# Single image chunk
wh = rng.randint(min_wh, max_wh + 1)
image = random_image(rng, wh, wh + 1)
return {"type": "image", "image": image}
frames = []
for _ in range(num_frames):
wh = rng.randint(min_wh, max_wh + 1)
frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
frames.append(frame)
video_array = np.stack(frames, axis=0)
return {"type": "video_chunk", "video_chunk": video_array}
def _test_processing_correctness(
model_id_or_arch: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
model_impl: str = "auto",
):
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
# Use model architecture to get the default model id
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
model_id = model_info.default
else:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
model_id = model_id_or_arch
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_config = ModelConfig(
model_id,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
hf_overrides=model_info.hf_overrides,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=model_info.dtype,
model_impl=model_impl,
)
# Ensure that the cache can fit all of the data
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
model_config.multimodal_config.mm_processor_cache_gb = 2048
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
cache = MultiModalProcessorOnlyCache(model_config)
processing_info = factories.info(ctx)
supported_mm_limits = processing_info.get_supported_mm_limits()
# Keep integer limits for local data generation
limit_mm_per_prompt_ints = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
if modality == "video":
return VideoDummyOptions(count=count)
if modality == "image":
return ImageDummyOptions(count=count)
if modality == "audio":
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
# Assign normalized DummyOptions to the model config
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
{
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt_ints.items()
}
)
processor = factories.build_processor(ctx)
rng = np.random.RandomState(0)
# GLM-ASR requires a minimum audio length of 70ms
min_audio_len = 512 if model_config.hf_config.model_type != "glmasr" else 1120
input_to_hit = {
"image": Image.new("RGB", size=(128, 128)),
"video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
"audio": (np.zeros((min_audio_len,)), 16000),
"vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
}
input_factory = {
"image": partial(random_image, rng, min_wh=128, max_wh=256),
"video": partial(
random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
),
"audio": partial(
random_audio,
rng,
min_len=min_audio_len,
max_len=min_audio_len + 512,
sr=16000,
),
"vision_chunk": partial(
random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
),
}
for batch_idx in range(num_batches):
mm_data = {
k: [
(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
for _ in range(rng.randint(limit + 1))
]
for k, limit in limit_mm_per_prompt_ints.items()
}
# Drop unnecessary keys and test single -> multi conversion
if rng.rand() < simplify_rate:
for k in list(mm_data.keys()):
if not mm_data[k]:
del mm_data[k]
elif len(mm_data[k]) == 1:
mm_data[k] = mm_data[k][0]
_test_processing_correctness_one(
model_config,
mm_data,
processor,
batch_idx,
hit_rate,
num_batches,
simplify_rate,
cache=cache,
)
def _test_processing_correctness_one(
model_config: ModelConfig,
mm_data: MultiModalDataDict,
processor: BaseMultiModalProcessor,
batch_idx: int,
hit_rate: float,
num_batches: int,
simplify_rate: float,
cache: MultiModalProcessorOnlyCache,
):
model_type = model_config.hf_config.model_type
token_prompt = get_token_prompt(processor, mm_data)
mm_items = processor.info.parse_mm_data(mm_data)
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
baseline_tokenized_result = processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_tokenized_result = processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
cache=cache,
)
_assert_inputs_equal(
baseline_tokenized_result,
cached_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{token_prompt=}, {mm_data=})"
),
)
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
if model_id == "openvla/openvla-7b":
pytest.skip(
"OpenVLA uses a custom vLLM processor because its HF remote "
"processor is incompatible with current Transformers."
)
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
pytest.skip(
"Voxtral Realtime doesn't make use of any place-holder "
"tokens and hence cannot pass the processing "
"correctness test as is. Let's revisit adapting this "
"test once more realtime models exist."
)
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
pytest.skip(
"MOSS-Audio uses a custom processor that dynamically expands "
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
"video backends (qwen_vl_utils / codec) that require real encoded "
"video bytes and metadata. The synthetic numpy-array videos used by "
"this test yield empty video features, so the generic correctness "
"check cannot exercise the video path. Image processing is covered "
"by registration/inference tests."
)
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
)
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness_transformers(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
model_impl="transformers",
)
def _assert_inputs_equal(
a: MultiModalInput,
b: MultiModalInput,
*,
ignore_mm_keys: set[str] | None = None,
msg: str = "",
):
if ignore_mm_keys is None:
ignore_mm_keys = set()
ignore_prompt_keys = ("prompt", "mm_kwargs")
a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}
b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}
assert a_rest == b_rest, msg
a_data = a["mm_kwargs"].get_data()
b_data = b["mm_kwargs"].get_data()
for key in ignore_mm_keys:
a_data.pop(key, None)
b_data.pop(key, None)
assert batched_tensors_equal(a_data, b_data), msg
@@ -1,241 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Video placeholder accounting for GLM-5.3-Flash.
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
placeholders per timestamp, so the timestamps returned by
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
while ``video_grid_thw`` decides how many rows the vision tower produces. If
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
inside a worker and takes the engine down with it.
The checks below are arithmetic: no weights, no GPU.
"""
import pytest
import torch
from PIL import Image
from transformers.models.glm5_next.video_processing_glm5_next import (
Glm5NextVideoProcessor,
smart_resize,
)
from transformers.video_utils import VideoMetadata
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ...utils import build_model_context
@pytest.fixture(scope="module")
def processor():
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"video": 1},
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
def _pixel_path_grid(
video_processor: Glm5NextVideoProcessor,
num_frames: int,
height: int,
width: int,
) -> tuple[int, int, int]:
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
resized_height, resized_width = smart_resize(
num_frames=num_frames,
height=height,
width=width,
temporal_factor=video_processor.temporal_patch_size,
factor=video_processor.patch_size
* video_processor.merge_size
* video_processor.patch_expand_factor,
min_pixels=video_processor.min_image_tokens,
max_pixels=video_processor.max_image_tokens,
)
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
return (
padded_frames // video_processor.temporal_patch_size,
resized_height // video_processor.patch_size,
resized_width // video_processor.patch_size,
)
@pytest.mark.parametrize(
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
[
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
# 1080p, 20 s: same factor of 3 at a full-size canvas.
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
# 1080p, 60 s: the one duration window where the two samplers agree
# anyway -- a regression guard, the count must not move.
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
],
)
def test_video_placeholders_match_encoder_rows(
processor,
total_num_frames: int,
fps: float,
duration: float,
height: int,
width: int,
expected_grid: tuple[int, int, int],
):
info = processor.info
video_processor = info.get_video_processor()
frame_indices = video_processor.sample_frames(
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
)
grid_t, grid_h, grid_w = _pixel_path_grid(
video_processor, len(frame_indices), height, width
)
assert (grid_t, grid_h, grid_w) == expected_grid
timestamps = info._get_video_second_idx_glm46v(
{
"total_num_frames": total_num_frames,
"fps": fps,
"duration": duration,
"do_sample_frames": True,
},
total_num_frames,
)
merge_length = video_processor.merge_size**2
tokens_per_frame = grid_h * grid_w // merge_length
encoder_rows = grid_t * grid_h * grid_w // merge_length
assert len(timestamps) == grid_t
assert len(timestamps) * tokens_per_frame == encoder_rows
assert timestamps == sorted(timestamps)
assert timestamps[0] == 0
assert timestamps[-1] <= duration
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
"""The loader may pre-sample and hand the frames over as they are."""
info = processor.info
video_processor = info.get_video_processor()
num_frames = 32
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
timestamps = info._get_video_second_idx_glm46v(
{
"total_num_frames": 256,
"fps": 8.0,
"duration": 32.0,
"do_sample_frames": False,
"frames_indices": list(range(0, 256, 256 // num_frames)),
},
num_frames,
)
assert len(timestamps) == grid_t
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
"""A clip the sampler cannot pick a single frame from is a bad request."""
with pytest.raises(ValueError, match="selected no frames"):
processor.info._get_video_second_idx_glm46v(
{
"total_num_frames": 2,
"fps": 8.0,
"duration": 0.25,
"do_sample_frames": True,
},
2,
)
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 2},
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)
def _image_info(**kwargs):
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 1},
**kwargs,
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
).info
def test_image_encoder_cache_covers_full_token_budget():
"""The most-features probe must reach the processor's token ceiling.
The inherited square probe refits to 89x89 = 7921 tokens under the
max_image_tokens=8000 budget, so the encoder cache came up short and
ordinary non-square images in 7922-8000 tokens were refused with
HTTP 400 (#59539).
"""
info = _image_info()
assert info.get_max_image_tokens() == 8000
# The shapes from the issue, with the token counts the processor
# actually produces; every one must fit the cache.
for width, height, expected_tokens in [
(4032, 3024, 7931), # phone photo, 4:3
(3840, 2160, 7973), # 4K frame, 16:9
(3508, 2480, 7950), # A4 at 300 dpi
(2600, 2400, 7998), # 13:12
(3000, 3000, 7921), # square worst case before the fix
]:
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
assert num_tokens == expected_tokens
assert num_tokens <= info.get_max_image_tokens()
# The profiling dummy covers the real worst case, not the square one.
size = info.get_image_size_with_most_features()
assert size.width * size.height == 2240 * 2800
def test_image_encoder_cache_follows_max_pixels_override():
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
assert info.get_max_image_tokens() == 100
size = info.get_image_size_with_most_features()
assert (
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
== 100
)
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from PIL import Image
from transformers import AutoImageProcessor
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
KimiK25FusedVisionProcessor,
)
from ...utils import build_model_context
_MODEL_ID = "moonshotai/Kimi-K3"
@pytest.fixture(scope="module")
def image_processors():
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
k25_fused = processor.info.image_processor
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
return k25_fused, k3_hf
def _noise_image(width: int, height: int) -> Image.Image:
rng = np.random.default_rng(0)
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
@pytest.mark.parametrize(
("width", "height"),
[(310, 470), (7300, 30), (13, 9)],
)
def test_fused_image_processor_matches_hf(image_processors, width: int, height: int):
fused, hf = image_processors
image = _noise_image(width, height)
expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt")
actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt")
torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"])
torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"])
assert fused.media_tokens_calculator(
{"type": "image", "image": image}
) == hf.media_tokens_calculator({"type": "image", "image": image})
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2})
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = processor.info.get_hf_config().image_placeholder * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
)
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Nemotron-Parse's device-side normalisation path."""
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ...utils import build_model_context
_MODEL_ID = "nvidia/NVIDIA-Nemotron-Parse-2.0"
def _noise_image(width: int, height: int) -> Image.Image:
rng = np.random.default_rng(0)
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(("width", "height"), [(1240, 1754), (310, 470)])
def test_mm_device_do_normalize(width: int, height: int):
device = current_platform.device_type
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
assert ctx.model_config.multimodal_config.mm_device_do_normalize
images = [_noise_image(width, height)]
prompt = [0]
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
assert raw_values.dtype == torch.uint8
assert raw_values.shape == normalized_values.shape
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
)
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("mm_device_do_normalize", [False, True])
@pytest.mark.parametrize("layout", ["ndarray_hwc", "tensor_hwc", "tensor_chw"])
def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
ctx.model_config.multimodal_config.mm_device_do_normalize = mm_device_do_normalize
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = _noise_image(310, 470)
array = np.asarray(image)
item = {
"ndarray_hwc": array,
"tensor_hwc": torch.from_numpy(array),
"tensor_chw": torch.from_numpy(array).permute(2, 0, 1),
}[layout]
def pixel_values(img: object) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [img]})
return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
@pytest.mark.usefixtures("default_vllm_config")
def test_explicit_kwargs_without_device_normalize():
ctx = build_model_context(
_MODEL_ID,
limit_mm_per_prompt={"image": 1},
mm_processor_kwargs={"do_rescale": False, "do_normalize": False},
)
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
# The remote processor ignores these kwargs and still normalises on the host.
assert values.is_floating_point()
@@ -1,272 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections.abc import Iterable
from contextlib import contextmanager
from functools import partial
from typing import Any, TypeAlias
import numpy as np
import pytest
import torch
import torch.nn as nn
from PIL import Image
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.config.cache import CacheConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.models.interfaces import supports_multimodal
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.multimodal.utils import group_and_batch_mm_kwargs
from vllm.platforms import current_platform
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.utils.collection_utils import is_list_of
from vllm.utils.torch_utils import set_default_torch_dtype
from ....utils import create_new_process_for_each_test
from ...registry import HF_EXAMPLE_MODELS
from ...utils import dummy_hf_overrides
from .test_common import get_model_ids_to_test, get_token_prompt
ImageInput = list[Image.Image]
VideoInput: TypeAlias = (
list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]]
)
AudioInput = list[tuple[np.ndarray, int]]
def _resize_data(
_data: Image.Image | np.ndarray, size_factor: float
) -> Image.Image | np.ndarray:
assert size_factor <= 1, "Size factor must be less than 1"
# Image input
if isinstance(_data, Image.Image):
W, H = _data.width, _data.height
W, H = map(lambda x: int(x * size_factor), (W, H))
return _data.resize((W, H))
# Video input with PIL Images
elif is_list_of(_data, Image.Image):
W, H = next(iter(_data)).width, next(iter(_data)).height
T = len(_data)
T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H))
return [d.resize((W, H)) for d in _data[:T]]
# Video input with numpy arrays
elif isinstance(_data, np.ndarray) and _data.ndim >= 4:
T, H, W, C = _data.shape[-4:]
T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W))
return _data[..., :T, :H, :W, :C]
# Audio input
elif isinstance(_data, np.ndarray) and _data.ndim == 1:
return _data[: int(len(_data) * size_factor)]
raise AssertionError("This line should be unreachable.")
def resize_mm_data(
data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...]
) -> ImageInput | VideoInput | AudioInput:
size_factors = size_factors[: len(data)]
if is_list_of(data, (Image.Image, np.ndarray, list)):
return [_resize_data(d, s) for d, s in zip(data, size_factors)]
elif is_list_of(data, tuple):
return [_resize_data(d, s) for (d, _), s in zip(data, size_factors)]
raise ValueError("Unsupported multimodal data type.")
def create_batched_mm_kwargs(
model_config: ModelConfig,
processor: BaseMultiModalProcessor,
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
processing_info = processor.info
supported_mm_limits = processing_info.get_supported_mm_limits()
mm_counts = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
processor_inputs = processor.get_dummy_inputs(
seq_len=model_config.max_model_len,
mm_counts=mm_counts,
mm_options=MultiModalDummyOptions(),
)
mm_items = processor_inputs.mm_data_items
resized_mm_data = {
modality: resize_mm_data(items.data, size_factors)
for modality, items in mm_items.items()
}
# video metadata will be added back to the resized video data here.
token_prompt = get_token_prompt(processor, resized_mm_data)
mm_kwargs = processor(
prompt=token_prompt,
mm_items=processor.info.parse_mm_data(resized_mm_data),
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
)["mm_kwargs"].require_data()
return group_and_batch_mm_kwargs(
[
(modality, item)
for modality in supported_mm_limits
for item in mm_kwargs[modality]
]
)
# TODO(Isotr0py): Don't initialize model during test
@contextmanager
def initialize_dummy_model(
model_cls: type[nn.Module],
model_config: ModelConfig,
):
temp_file = tempfile.mkstemp()[1]
current_device = torch.get_default_device()
vllm_config = VllmConfig(
model_config=model_config, cache_config=CacheConfig(block_size=16)
)
with set_current_vllm_config(vllm_config=vllm_config):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=1)
with set_default_torch_dtype(model_config.dtype):
torch.set_default_device(current_platform.device_type)
model = model_cls(vllm_config=vllm_config)
torch.set_default_device(current_device)
yield model
del model
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
def test_model_tensor_schema(model_id: str):
if model_id == "moonshotai/Kimi-K2.5":
# FIXME(Isotr0py): Fix Kimi-K2.5's offline inference about vision chunks.
pytest.skip(
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
)
if (
model_id == "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
and not current_platform.is_cuda_alike()
):
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
if model_id == "zai-org/GLM-5.3-Flash" and (current_platform.is_xpu()):
pytest.skip("GLM-5.3-Flash is not supported on XPU")
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_arch = next(
arch for arch, info in HF_EXAMPLE_MODELS.hf_models.items() if info == model_info
)
hf_overrides_fn = partial(
dummy_hf_overrides,
model_arch=model_arch,
exist_overrides=model_info.hf_overrides,
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
)
# ROCm: Detect if model uses AWQ quantization and set appropriate dtype
if "awq" in model_id.lower() and current_platform.is_rocm():
dtype = "float16"
else:
dtype = model_info.dtype
model_config = ModelConfig(
model_id,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
hf_overrides=hf_overrides_fn,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=dtype,
)
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
assert supports_multimodal(model_cls)
factories = model_cls._processor_factory
# Capture helpers return capture-buffer containers (e.g.
# EncoderCudaGraphCaptureInputs), not TensorSchema mm inputs.
capture_helpers = {"prepare_encoder_cudagraph_capture_inputs"}
inputs_parse_methods = []
for attr_name in dir(model_cls):
if attr_name in capture_helpers:
continue
attr = getattr(model_cls, attr_name)
if hasattr(attr, "__annotations__"):
return_type = attr.__annotations__.get("return", None)
if return_type is not None and "Input" in str(return_type):
inputs_parse_methods.append(attr_name)
if not any(inputs_parse_methods):
pytest.skip(f"{model_arch} does not support tensor schema validation.")
ctx = InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
processing_info = factories.info(ctx)
supported_mm_limits = processing_info.get_supported_mm_limits()
limit_mm_per_prompt = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
if modality == "video":
return VideoDummyOptions(count=count)
if modality == "image":
return ImageDummyOptions(count=count)
if modality == "audio":
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
{
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt.items()
}
)
processor = factories.build_processor(ctx)
with initialize_dummy_model(model_cls, model_config) as model:
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
for method_name in inputs_parse_methods:
print(
f"Testing `{method_name}` with modality={modality} "
f"and mm_kwargs{list(mm_kwargs.keys())}"
)
getattr(model, method_name)(modality=modality, **mm_kwargs)