[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,228 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from ....utils import large_gpu_mark
from ...registry import HF_EXAMPLE_MODELS
from ...utils import check_logprobs_close
# Models that require embedding scaling for prompt_embeds test
EMBED_SCALING_MODELS = {
"openbmb/MiniCPM4.1-8B",
}
# This list contains the model that are using AITER kernel.
# Skip model that are not using AITER tests.
# When more AITER kernels are added, this list will not be
# needed as all the models will be calling AITER kernels
# in parts of the operators
AITER_MODEL_LIST = [
"meta-llama/Llama-3.2-1B-Instruct",
"openbmb/MiniCPM3-4B",
"Qwen/Qwen2.5-0.5B-Instruct",
"axolotl-ai-co/tiny-mixtral-30m",
"Qwen/Qwen3-8B",
]
# @maybe_test_rocm_aiter
@pytest.mark.parametrize(
"model",
[
pytest.param(
"bigscience/bloom-560m", # bloom - testing alibi slopes
marks=[
pytest.mark.core_model,
pytest.mark.slow_test,
pytest.mark.cpu_model,
],
),
pytest.param(
"openai-community/gpt2", # gpt2
marks=[pytest.mark.core_model],
),
pytest.param("Milos/slovak-gpt-j-405M"), # gptj
pytest.param("bigcode/tiny_starcoder_py"), # gpt_bigcode
pytest.param("EleutherAI/pythia-70m"), # gpt_neox
pytest.param(
"google/gemma-1.1-2b-it", # gemma
marks=[
pytest.mark.core_model,
pytest.mark.cpu_model,
pytest.mark.slow_test,
],
),
pytest.param(
"google/gemma-2-2b-it", # test hybrid attention
marks=[pytest.mark.cpu_model],
),
pytest.param(
"zai-org/chatglm3-6b", # chatglm (text-only)
),
pytest.param(
"meta-llama/Llama-3.2-1B-Instruct", # llama
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
pytest.param(
"openbmb/MiniCPM4.1-8B", # minicpm
marks=[pytest.mark.core_model, large_gpu_mark(min_gb=48)],
),
pytest.param(
"facebook/opt-125m", # opt
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
),
pytest.param(
"facebook/opt-350m", # opt with projected token embeddings
),
pytest.param(
"microsoft/phi-2", # phi
marks=[pytest.mark.core_model, pytest.mark.slow_test],
),
pytest.param(
"Qwen/Qwen2.5-0.5B-Instruct", # qwen2
marks=[
pytest.mark.core_model,
pytest.mark.cpu_model,
pytest.mark.slow_test,
],
),
pytest.param(
"Qwen/Qwen3-8B", # qwen (text-only)
),
pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
pytest.param("bigcode/starcoder2-3b"), # starcoder2
pytest.param(
"axolotl-ai-co/tiny-mixtral-30m", # mixtral
marks=[pytest.mark.core_model],
),
pytest.param("swiss-ai/Apertus-8B-Instruct-2509"), # apertus
pytest.param(
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", # hyperclovax
marks=[large_gpu_mark(min_gb=32)],
),
],
)
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
@pytest.mark.parametrize("use_prompt_embeds", [True, False])
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
max_tokens: int,
num_logprobs: int,
use_rocm_aiter: bool,
use_prompt_embeds: bool,
monkeypatch,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
if use_rocm_aiter and (model in AITER_MODEL_LIST):
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
elif use_rocm_aiter and model not in AITER_MODEL_LIST:
# Skip model that are not using AITER tests.
# When more AITER kernels are added, this list will not be
# needed as all the models will be calling AITER kernels
# in parts of the operators
pytest.skip(f"Skipping '{model}' model test with AITER kernel.")
if model == "bigcode/starcoder2-3b":
# Replace example.txt's Test1 (an NL prompt) with a code prompt:
# starcoder2-3b is a code model, so NL prompts give near-uniform
# digit logits where HF<->vLLM bf16 drift can reorder top-K.
example_prompts = list(example_prompts)
example_prompts[1] = (
"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - "
)
with hf_runner(
model,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
prompt_embeds: list[torch.Tensor] | None = [] if use_prompt_embeds else None
for prompt in example_prompts:
token_ids = hf_model.tokenizer(prompt, return_tensors="pt").input_ids.to(
hf_model.model.device
)
if prompt_embeds is not None:
# Retained prompt values must not keep reference weights alive.
with torch.no_grad():
embed = hf_model.model.get_input_embeddings()(token_ids)
# MiniCPM models apply scale_emb to embeddings internally.
# vLLM expects pre-scaled embeddings when using inputs_embeds.
if model in EMBED_SCALING_MODELS:
config = hf_model.model.config
embed = embed * config.scale_emb
prompt_embeds.append(embed.squeeze(0))
vllm_kwargs = {}
if (
model == "bigscience/bloom-560m"
and current_platform.is_device_capability_family(90)
):
# On SM90, the metadata builder otherwise selects FA3 AOT scheduling
# before Bloom's ALiBi layers fall back to FA2. Pinning FA2 keeps the
# builder and layer consistent and preserves the L4 test path.
vllm_kwargs["attention_config"] = {"flash_attn_version": 2}
with vllm_runner(
model,
tokenizer_name=model_info.tokenizer or model,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
# Remove the effects of batch variance on ROCm since batch invariance
# is not yet supported.
# See: https://github.com/vllm-project/vllm/issues/27433
max_num_seqs=1 if current_platform.is_rocm() else 2,
enable_prompt_embeds=use_prompt_embeds,
compilation_config={"cudagraph_capture_sizes": [1, 2]},
**vllm_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
if prompt_embeds is not None:
vllm_outputs_from_embeds = vllm_model.generate_greedy_logprobs(
prompt_embeds, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
if prompt_embeds is not None:
check_logprobs_close(
outputs_0_lst=vllm_outputs,
outputs_1_lst=vllm_outputs_from_embeds,
name_0="vllm",
name_1="vllm_from_embeds",
)
if use_rocm_aiter:
# this is to ensure that vllm engine
# has deallocated the memory before running the next
# unit tests. On ROCm, when using AITER
# the memory might not be deallocated completely
# before running the next test case
torch.accelerator.synchronize()
@@ -1,77 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Regression test for hybrid GDN/Mamba models under sleep -> wake.
Hybrid Mamba / gated-delta-net (GDN) models (e.g. Qwen3-Next) keep a
persisted conv + recurrent state cache. With sleep mode (the RLHF reuse
pattern: ``sleep()`` -> weight update -> ``wake_up()``) the state-cache tag is
discarded on sleep and its device memory is re-created on wake. If a *new*
sequence's state slot is consumed before being reset, the gated-delta-rule
kernel faithfully propagates whatever is in that (now non-zeroed) memory; when
it contains NaN/inf the output becomes NaN and ``argmax`` collapses every token
to id 0 (which decodes to ``"!"``), giving ``reward=0`` / NaN log-probs in RL
training.
This test sleeps and wakes a small hybrid GDN model and asserts that
post-wake generation is neither degenerate (single repeated token) nor NaN.
"""
import pytest
from vllm import LLM, SamplingParams
# Small Qwen3-Next (GDN) model already used by the hybrid model test-suite.
MODEL = "tiny-random/qwen3-next-moe"
PROMPTS = [
"The capital of France is",
"Once upon a time,",
"1, 2, 3, 4,",
"Water is made of",
]
@pytest.mark.hybrid_model
def test_gdn_sleep_wake_no_stale_state():
sampling_params = SamplingParams(temperature=0.0, max_tokens=32, logprobs=1)
# Keep the reserved fraction low. On some (notably ROCm/amdgpu) drivers the
# VRAM discarded by ``sleep()`` is not returned to the free pool before
# ``wake_up()`` re-creates it, so the woken allocation must coexist with the
# not-yet-reclaimed one (~2x peak). A high ``gpu_memory_utilization`` then
# OOMs in ``cuMemCreate`` on wake. The model is tiny, so a small fraction
# still leaves ample KV/state cache while keeping the sleep/wake cycle well
# within device memory.
llm = LLM(
model=MODEL,
enable_sleep_mode=True,
enforce_eager=True,
max_model_len=1024,
gpu_memory_utilization=0.4,
trust_remote_code=True,
)
# Warm generation before sleeping.
llm.generate(PROMPTS, sampling_params)
# Default sleep offloads weights and DISCARDS the kv / GDN state cache;
# wake_up re-creates that memory (fresh, not guaranteed zeroed).
llm.sleep()
llm.wake_up()
after = llm.generate(PROMPTS, sampling_params)
for output in after:
completion = output.outputs[0]
token_ids = list(completion.token_ids)
assert token_ids, "empty generation after wake_up"
# The bug collapses every token to a single id (e.g. 0 -> "!").
assert len(set(token_ids)) > 1, (
f"degenerate single-token output after wake_up: {token_ids[:16]}"
)
# NaN logits surface as NaN log-probs.
for step_logprobs in completion.logprobs or []:
for logprob in step_logprobs.values():
assert logprob.logprob == logprob.logprob, (
"NaN log-prob after wake_up (stale GDN state)"
)
@@ -1,928 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from contextlib import contextmanager, nullcontext
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from tests.utils import multi_gpu_test
from vllm import LLM
from vllm.config import CUDAGraphMode
from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.sampling_params import SamplingParams
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
from ...utils import check_logprobs_close, check_outputs_equal
# Mark all tests as hybrid
pytestmark = pytest.mark.hybrid_model
# NOTE: The first model in each list is taken as the primary model,
# meaning that it will be used in all tests in this file
# The rest of the models will only be tested by test_models
APC_MULTIPLY_BY = 300
SSM_MODELS = [
"state-spaces/mamba-130m-hf",
"tiiuae/falcon-mamba-tiny-dev",
# mamba2-codestral in transformers is broken pending:
# https://github.com/huggingface/transformers/pull/40861
# "yujiepan/mamba2-codestral-v0.1-tiny-random",
]
HYBRID_MODELS = [
"ai21labs/Jamba-tiny-dev",
"Zyphra/Zamba2-1.2B-instruct",
"ibm-granite/granite-4.0-tiny-preview",
"tiiuae/Falcon-H1-0.5B-Base",
"LiquidAI/LFM2-1.2B",
"tiny-random/qwen3-next-moe",
]
FULL_CUDA_GRAPH_MODELS = [
"ai21labs/Jamba-tiny-dev",
"Zyphra/Zamba2-1.2B-instruct",
]
FP32_STATE_MODELS = [
"state-spaces/mamba-130m-hf",
"Zyphra/Zamba2-1.2B-instruct",
]
# Avoid OOM
MAX_NUM_SEQS = 4
ATTN_BACKEND = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
def _set_conv_state_layout(monkeypatch, layout: str) -> None:
"""Set conv state layout env var and clear cache to pick up new value."""
from vllm.model_executor.layers.mamba import mamba_utils
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", layout)
mamba_utils.get_conv_state_layout.cache_clear()
@pytest.mark.parametrize("model", SSM_MODELS + HYBRID_MODELS)
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
hf_runner,
vllm_runner,
gpu_memory_cleared,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
num_logprobs: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
with hf_runner(model) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
def test_batching(
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
num_logprobs: int,
conv_state_layout: str,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
_set_conv_state_layout(monkeypatch, conv_state_layout)
for_loop_outputs = []
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
for prompt in example_prompts:
(single_output,) = vllm_model.generate_greedy_logprobs(
[prompt], max_tokens, num_logprobs
)
for_loop_outputs.append(single_output)
batched_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=for_loop_outputs,
outputs_1_lst=batched_outputs,
name_0="for_loop_vllm",
name_1="batched_vllm",
)
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [10])
@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
def test_chunked_prefill_with_parallel_sampling(
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
conv_state_layout: str,
) -> None:
"""Tests chunked prefill in conjunction with n > 1.
In this case, prefill is populated with decoding tokens and
we test that it doesn't fail.
This test might fail if cache is not allocated correctly for n > 1
decoding steps inside a chunked prefill forward pass
(where we have both prefill and decode together)
"""
_set_conv_state_layout(monkeypatch, conv_state_layout)
sampling_params = SamplingParams(n=3, temperature=1, seed=0, max_tokens=max_tokens)
with vllm_runner(
model,
enable_chunked_prefill=True,
# forces prefill chunks with decoding
max_num_batched_tokens=MAX_NUM_SEQS * 3,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
) as vllm_model:
vllm_model.generate(example_prompts, sampling_params)
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [20])
@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
def test_mamba_cache_cg_padding(
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
conv_state_layout: str,
) -> None:
"""This test is for verifying that mamba cache is padded to CG captured
batch size. If it's not, a torch RuntimeError will be raised because
tensor dimensions aren't compatible.
"""
_set_conv_state_layout(monkeypatch, conv_state_layout)
vllm_config = EngineArgs(model=model, trust_remote_code=True).create_engine_config()
cudagraph_dispatcher = CudagraphDispatcher(vllm_config)
cudagraph_dispatcher.initialize_cudagraph_keys(
vllm_config.compilation_config.cudagraph_mode
)
if cudagraph_dispatcher.cudagraph_mode == CUDAGraphMode.NONE:
pytest.skip("CUDA/XPU graph is disabled.Please enable it to run this test. ")
while (
len(example_prompts)
== cudagraph_dispatcher.dispatch(len(example_prompts))[1].num_tokens
):
example_prompts.append(example_prompts[0])
try:
with vllm_runner(model, enable_chunked_prefill=True) as vllm_model:
vllm_model.generate_greedy(example_prompts, max_tokens)
except RuntimeError:
pytest.fail(
"Couldn't run batch size which is not equal to a Cuda Graph "
"captured batch size. "
"Could be related to mamba cache not padded correctly"
)
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
vllm_runner,
example_prompts,
model: str,
) -> None:
"""This test is for verifying that the hybrid inner state management doesn't
collapse in case where the number of incoming requests and
finished_requests_ids is larger than the maximum mamba block capacity.
This could generally happen due to the fact that hybrid does support
statelessness mechanism where it can clean up new incoming requests in
a single step.
"""
try:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
except ValueError:
pytest.fail(
"Hybrid inner state wasn't cleaned up properly between"
"steps finished requests registered unnecessarily "
)
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
def test_state_cleanup(
vllm_runner,
example_prompts,
model: str,
) -> None:
"""This test is for verifying that the Hybrid state is cleaned up between
steps.
If it's not cleaned, an error would be expected.
"""
try:
with vllm_runner(
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
) as vllm_model:
for _ in range(10):
vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
except ValueError:
pytest.fail(
"Hybrid inner state wasn't cleaned up between states, "
"could be related to finished_requests_ids"
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_distributed_correctness(
vllm_runner,
example_prompts,
model: str,
max_tokens: int,
num_logprobs: int,
) -> None:
with vllm_runner(
model,
tensor_parallel_size=1,
max_num_seqs=MAX_NUM_SEQS,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs_tp_1 = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(
model,
tensor_parallel_size=2,
max_num_seqs=MAX_NUM_SEQS,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs_tp_2 = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=vllm_outputs_tp_1,
outputs_1_lst=vllm_outputs_tp_2,
name_0="vllm_tp_1",
name_1="vllm_tp_2",
)
@pytest.mark.parametrize("model", FULL_CUDA_GRAPH_MODELS)
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_full_cuda_graph(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
num_logprobs: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
with hf_runner(model) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
# Leave headroom for the HF model that ran earlier in this process on a
# 32.5 GiB MIG.
with vllm_runner(
model,
max_num_seqs=MAX_NUM_SEQS,
attention_backend=ATTN_BACKEND,
gpu_memory_utilization=0.9,
enable_chunked_prefill=True,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", FP32_STATE_MODELS)
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize(
"cache_dtype_param", ["mamba_ssm_cache_dtype", "mamba_cache_dtype"]
)
def test_fp32_cache_state(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
num_logprobs: int,
cache_dtype_param: str,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
with hf_runner(model) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
# Leave enough headroom for repeated engine initialization on a
# 32.5 GiB MIG.
with vllm_runner(
model,
max_num_seqs=MAX_NUM_SEQS,
gpu_memory_utilization=0.9,
enable_chunked_prefill=True,
**{cache_dtype_param: "float32"},
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
# Helper functions for the APC tests
def _get_vllm_runner_params(
model: str,
max_model_len: int,
tensor_parallel_size: int = 1,
):
return {
"model_name": model,
"enable_chunked_prefill": True,
"enable_prefix_caching": False,
"max_model_len": max_model_len,
"tensor_parallel_size": tensor_parallel_size,
"gpu_memory_utilization": 0.4,
"attention_backend": ATTN_BACKEND,
}
@contextmanager
def _owned_vllm_runner(vllm_runner, kwargs):
with vllm_runner(**kwargs) as runner:
yield runner
def _get_vLLM_output(
vllm_runner,
kwargs,
prompts,
max_tokens,
num_logprobs,
num_repetitions=1,
vllm_model=None,
):
runner_context = (
_owned_vllm_runner(vllm_runner, kwargs)
if vllm_model is None
else nullcontext(vllm_model)
)
with runner_context as runner:
outs = []
for _ in range(num_repetitions):
if num_logprobs < 0:
vllm_output = runner.generate_greedy(prompts, max_tokens)
else:
vllm_output = runner.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs
)
outs.append(vllm_output)
return outs, vllm_model
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("n_repetitions", [2])
# If num_logprobs is set to -1, then the stringent version
# of the test is executed using `check_outputs_equal`
# instead of `check_logprobs_close`
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_apc_single_prompt(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
n_repetitions: int,
num_logprobs: int,
tensor_parallel_size: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
compare_operator: Callable = (
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
)
# Sample prompts.
generated_prompts = [APC_MULTIPLY_BY * example_prompts[0]]
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
vllm_runner_kwargs = _get_vllm_runner_params(
model, max_model_len, tensor_parallel_size=tensor_parallel_size
)
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
vllm_outputs_no_cache, _ = _get_vLLM_output(
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
)
vllm_runner_kwargs["enable_prefix_caching"] = True
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
)
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0],
outputs_1_lst=vllm_outputs_cache_itn,
name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}",
)
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("n_repetitions", [2])
# If num_logprobs is set to -1, then the stringent version
# of the test is executed using `check_outputs_equal`
# instead of `check_logprobs_close`
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_apc_single_prompt_block_align_alignment(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
n_repetitions: int,
num_logprobs: int,
tensor_parallel_size: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
compare_operator: Callable = (
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
)
# Sample prompts. This custom prompt is used, as it causes the most issues
generated_prompts = ["The president of the United States is " * APC_MULTIPLY_BY]
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
vllm_runner_kwargs = _get_vllm_runner_params(
model, max_model_len, tensor_parallel_size=tensor_parallel_size
)
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
vllm_outputs_no_cache, _ = _get_vLLM_output(
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
)
vllm_runner_kwargs["enable_prefix_caching"] = True
with vllm_runner(**vllm_runner_kwargs) as vllm_model:
# Retrieve the default mamba state block size
vllm_config = vllm_model.llm.llm_engine.vllm_config
mamba_block_size = vllm_config.cache_config.mamba_block_size
# In case the hybrid model does not have the
# "mamba_block_size" assume a fixed constant
if mamba_block_size is None:
mamba_block_size = 512
mamba_block_size_multiplier = 10
for offsets in [-3, 3, mamba_block_size // 4 + 3, mamba_block_size // 2 - 3]:
vllm_runner_kwargs["max_num_batched_tokens"] = (
mamba_block_size_multiplier * mamba_block_size - offsets
)
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
)
# Check alignment of the output logits when using APC
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0],
outputs_1_lst=vllm_outputs_cache_itn,
name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}",
)
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("n_repetitions", [2])
# If num_logprobs is set to -1, then the stringent version
# of the test is executed using `check_outputs_equal`
# instead of `check_logprobs_close`
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_apc_multiple_prompts_all_cached_outputs(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
n_repetitions: int,
num_logprobs: int,
tensor_parallel_size: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
compare_operator: Callable = (
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
)
# Sample prompts.
generated_prompts = [APC_MULTIPLY_BY * prompt for prompt in example_prompts]
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
vllm_runner_kwargs = _get_vllm_runner_params(
model, max_model_len, tensor_parallel_size=tensor_parallel_size
)
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
# Reduce the effects of batch variance on ROCm since batch invariance is not
# yet supported. See: https://github.com/vllm-project/vllm/issues/27433
if current_platform.is_rocm():
vllm_runner_kwargs["max_num_seqs"] = 4
vllm_outputs_no_cache, _ = _get_vLLM_output(
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
)
vllm_runner_kwargs["enable_prefix_caching"] = True
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
)
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0],
outputs_1_lst=vllm_outputs_cache_itn,
name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}",
)
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("n_repetitions", [2])
# If num_logprobs is set to -1, then the stringent version
# of the test is executed using `check_outputs_equal`
# instead of `check_logprobs_close`
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_apc_multiple_prompts_block_align_alignment(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
n_repetitions: int,
num_logprobs: int,
tensor_parallel_size: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
compare_operator: Callable = (
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
)
# Sample prompts. This custom prompt is used, as it causes the most issues
prompt_text = "The president of the United States is "
prompt_offsets = [0, 3, 7, 13, 17, 22, 25, 31]
generated_prompts = [
prompt_text[offset:] * APC_MULTIPLY_BY for offset in prompt_offsets
]
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
vllm_runner_kwargs = _get_vllm_runner_params(
model, max_model_len, tensor_parallel_size
)
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
vllm_outputs_no_cache, _ = _get_vLLM_output(
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
)
vllm_runner_kwargs["enable_prefix_caching"] = True
with vllm_runner(**vllm_runner_kwargs) as vllm_model:
# Retrieve the default mamba state block size
vllm_config = vllm_model.llm.llm_engine.vllm_config
mamba_block_size = vllm_config.cache_config.mamba_block_size
# In case the hybrid model does not have the
# "mamba_block_size" assume a fixed constant
if mamba_block_size is None:
mamba_block_size = 512
mamba_block_size_multiplier = 10
for offsets in [-3, 3, mamba_block_size // 4 + 3, mamba_block_size // 2 - 3]:
vllm_runner_kwargs["max_num_batched_tokens"] = (
mamba_block_size_multiplier * mamba_block_size - offsets
)
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
)
# Check alignment of the output logits when using APC
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0],
outputs_1_lst=vllm_outputs_cache_itn,
name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}",
)
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("n_repetitions", [2])
# If num_logprobs is set to -1, then the stringent version
# of the test is executed using `check_outputs_equal`
# instead of `check_logprobs_close`
@pytest.mark.parametrize("num_logprobs", [5])
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_apc_multiple_prompts_partial_cached_outputs(
hf_runner,
vllm_runner,
example_prompts,
monkeypatch,
model: str,
max_tokens: int,
n_repetitions: int,
num_logprobs: int,
tensor_parallel_size: int,
) -> None:
try:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
except ValueError:
pass
compare_operator: Callable = (
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
)
# Sample prompts.
generated_prompts = [APC_MULTIPLY_BY * prompt for prompt in example_prompts]
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
vllm_runner_kwargs = _get_vllm_runner_params(
model, max_model_len, tensor_parallel_size=tensor_parallel_size
)
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
vllm_outputs_no_cache, _ = _get_vLLM_output(
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
)
# Cache only part of all the prompts
vllm_runner_kwargs["enable_prefix_caching"] = True
with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
vllm_outputs_partial_cache, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts[:3],
max_tokens,
num_logprobs,
vllm_model=vllm_model,
)
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0][:3],
outputs_1_lst=vllm_outputs_partial_cache[0],
name_0="vllm_no_cache",
name_1="vllm_partial_cache",
)
vllm_outputs_cache_rep, _ = _get_vLLM_output(
vllm_runner,
vllm_runner_kwargs,
generated_prompts,
max_tokens,
num_logprobs,
n_repetitions,
vllm_model=vllm_model,
)
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
# In the first repetition, the caches are filled
# In the second repetition, these caches are reused
compare_operator(
outputs_0_lst=vllm_outputs_no_cache[0],
outputs_1_lst=vllm_outputs_cache_itn,
name_0="vllm_no_cache",
name_1=f"vllm_cache_it_{r_idx + 1}",
)
# Test that outputs match whether prefix caching is enabled or not for mamba.
@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
def test_same_mamba_output_apc_on_vs_off(
vllm_runner,
model: str,
) -> None:
num_logprobs = 5
prompts = [
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
]
max_tokens = 20
max_model_len = max(len(p) for p in prompts) + max_tokens + 64
base_kwargs = _get_vllm_runner_params(model, max_model_len)
base_kwargs.update(
enforce_eager=True, block_size=16, seed=42, gpu_memory_utilization=0.8
)
# No prefix caching
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
with vllm_runner(**kwargs_no_apc) as vllm_model:
outputs_no_apc, _ = _get_vLLM_output(
vllm_runner,
kwargs_no_apc,
prompts,
max_tokens,
num_logprobs=num_logprobs,
vllm_model=vllm_model,
)
# With prefix caching
kwargs_with_apc = {
**base_kwargs,
"enable_prefix_caching": True,
"mamba_block_size": 16,
}
with vllm_runner(**kwargs_with_apc) as vllm_model:
outputs_with_apc, _ = _get_vLLM_output(
vllm_runner,
kwargs_with_apc,
prompts,
max_tokens,
num_logprobs=num_logprobs,
vllm_model=vllm_model,
)
check_logprobs_close(
outputs_0_lst=outputs_no_apc[0],
outputs_1_lst=outputs_with_apc[0],
name_0="vllm_no_apc",
name_1="vllm_with_apc",
)
# we have to use a real large model to get reasonable results
# the model can't be a hybrid model as we need block_size 16
@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
def test_apc_common_prefix_same_batch(
model: str,
monkeypatch,
) -> None:
# Required to put the two requests in the same batch
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
llm = LLM(
model=model,
enforce_eager=True,
block_size=16,
mamba_block_size=16,
enable_prefix_caching=True,
seed=42,
attention_backend=ATTN_BACKEND,
)
prompts = [
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
]
sampling_params = SamplingParams(temperature=0.0, max_tokens=20)
outputs = llm.generate(prompts, sampling_params)
for output in outputs:
assert "two" in output.outputs[0].text
@@ -1,350 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import json
import pytest
from vllm.parser.mistral import MistralToolCall
from vllm.sampling_params import SamplingParams
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
from ...utils import check_logprobs_close
MODELS = [
"mistralai/Mistral-7B-Instruct-v0.3",
]
MISTRAL_FORMAT_MODELS = [
"mistralai/Mistral-7B-Instruct-v0.3",
# uses the v3-Tekken tokenizer
"mistralai/Ministral-8B-Instruct-2410",
# Mistral-Nemo is too big for CI, but passes locally
# "mistralai/Mistral-Nemo-Instruct-2407"
]
SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
SYMBOLIC_LANG_PROMPTS = [
"勇敢な船乗りについての詩を書く", # japanese
"寫一首關於勇敢的水手的詩", # chinese
"ပုံပြင်လေးပြောပြပါ်:\n", # burmese
"Repeat the phrase 'URGENCY🌶️':\nURGENCY🌶️\nURGENCY🌶️\n", # see https://github.com/vllm-project/vllm/pull/9625
]
# for function calling
TOOLS = [
{
"type": "function",
"function": {
"name": "get_current_weather",
"description": "Get the current weather in a given location",
"parameters": {
"type": "object",
"properties": {
"city": {
"type": "string",
"description": "The city to find the weather for, e.g. "
"'San Francisco'",
},
"state": {
"type": "string",
"description": "the two-letter abbreviation for the state that "
"the city is in, e.g. 'CA' which would mean 'California'",
},
"unit": {
"type": "string",
"description": "The unit to fetch the temperature in",
"enum": ["celsius", "fahrenheit"],
},
},
"required": ["city", "state", "unit"],
},
},
},
{
"type": "function",
"function": {
"name": "rewrite",
"description": "Rewrites text",
"parameters": {
"type": "object",
"required": [],
"properties": {
"text": {
"type": "string",
"description": "The input text to rewrite.",
}
},
},
},
},
]
MSGS = [
{"role": "system", "content": "You are an assistant."},
{
"role": "user",
"content": "Could you please rewrite the below article? \n\n My English needs "
"improving, maybe I make errors.",
},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "bbc5b7ede",
"type": "function",
"function": {
"name": "rewrite",
"arguments": '{"text":"My English needs improving, maybe '
'I make errors."}',
},
}
],
},
{
"role": "tool",
"content": '{"action":"rewrite","outcome":"My English needs improving, maybe '
'I make errors."}',
"tool_call_id": "bbc5b7ede",
"name": "rewrite",
},
{
"role": "assistant",
"content": "---\n\nMy English needs improving, maybe I make errors",
},
{
"role": "user",
"content": (
"Can you tell me what the temperate will be in Dallas, in fahrenheit?"
),
},
]
SAMPLE_JSON_SCHEMA = {
"type": "object",
"properties": {
"name": {"type": "string"},
"age": {"type": "integer"},
"skills": {
"type": "array",
"items": {"type": "string", "maxLength": 10},
"minItems": 3,
},
"work_history": {
"type": "array",
"items": {
"type": "object",
"properties": {
"company": {"type": "string"},
"duration": {"type": "number"},
"position": {"type": "string"},
},
"required": ["company", "position"],
},
},
},
"required": ["name", "age", "skills", "work_history"],
}
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
hf_runner,
vllm_runner,
example_prompts,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
# TODO(sang): Sliding window should be tested separately.
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(model, dtype=dtype, tokenizer_mode="mistral") as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("max_tokens", [64])
@pytest.mark.parametrize("num_logprobs", [5])
def test_mistral_format(
vllm_runner,
example_prompts,
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
with vllm_runner(
model,
dtype=dtype,
tokenizer_mode="mistral",
load_format="mistral",
config_format="mistral",
) as mistral_format_model:
mistral_format_outputs = mistral_format_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
with vllm_runner(
model,
dtype=dtype,
tokenizer_mode="hf",
load_format="safetensors",
config_format="hf",
) as hf_format_model:
hf_format_outputs = hf_format_model.generate_greedy_logprobs(
example_prompts, max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=hf_format_outputs,
outputs_1_lst=mistral_format_outputs,
name_0="hf",
name_1="mistral",
)
@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
def test_mistral_symbolic_languages(vllm_runner, model: str, dtype: str) -> None:
with vllm_runner(
model,
dtype=dtype,
max_model_len=8192,
tokenizer_mode="mistral",
config_format="mistral",
load_format="mistral",
) as vllm_model:
for prompt in SYMBOLIC_LANG_PROMPTS:
msg = {"role": "user", "content": prompt}
outputs = vllm_model.llm.chat([msg], sampling_params=SAMPLING_PARAMS)
assert "�" not in outputs[0].outputs[0].text.strip()
@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
@pytest.mark.parametrize("dtype", ["bfloat16"])
def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
with vllm_runner(
model,
dtype=dtype,
tokenizer_mode="mistral",
config_format="mistral",
load_format="mistral",
) as vllm_model:
msgs = copy.deepcopy(MSGS)
outputs = vllm_model.llm.chat(
msgs, tools=TOOLS, sampling_params=SAMPLING_PARAMS
)
tokenizer = vllm_model.llm.get_tokenizer()
tool_parser = MistralToolParser(tokenizer)
model_output = outputs[0].outputs[0].text.strip()
assert model_output.startswith(tool_parser.bot_token), model_output
parsed_message = tool_parser.extract_tool_calls(model_output, None)
assert parsed_message.tools_called
assert MistralToolCall.is_valid_id(parsed_message.tool_calls[0].id)
assert parsed_message.tool_calls[0].function.name == "get_current_weather"
assert (
parsed_message.tool_calls[0].function.arguments
== '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}'
) # noqa
assert parsed_message.content is None
def test_mistral_function_call_nested_json():
"""Ensure that the function-name regex captures the entire outermost
JSON block, including nested braces."""
# Create a minimal stub tokenizer that provides the few attributes the
# parser accesses (`version` and `get_vocab`).
class _StubMistralTokenizer(MistralTokenizer):
version = 11 # Satisfy the version check
def __init__(self):
pass
@staticmethod
def get_vocab():
# Provide the special TOOL_CALLS token expected by the parser.
return {"[TOOL_CALLS]": 0}
tokenizer = _StubMistralTokenizer()
parser = MistralToolParser(tokenizer)
# Craft a model output featuring nested JSON inside the arguments.
args_dict = {
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
"sub_dict": {"foo": "bar", "inner": {"x": 1, "y": 2}},
}
model_output = f"{parser.bot_token}get_current_weather{json.dumps(args_dict)}"
parsed = parser.extract_tool_calls(model_output, None)
# Assertions: the tool call is detected and the full nested JSON is parsed
# without truncation.
assert parsed.tools_called
assert MistralToolCall.is_valid_id(parsed.tool_calls[0].id)
assert parsed.tool_calls[0].function.name == "get_current_weather"
assert json.loads(parsed.tool_calls[0].function.arguments) == args_dict
# No additional content outside the tool call should be returned.
assert parsed.content is None
# multiple calls
multiple_args_dict = [
{
"city": "Dallas",
"state": "TX",
"unit": "fahrenheit",
"sub_dict": {"foo": "bar", "inner": {"x": 1, "y": 2}},
},
{},
{"a": 0},
{"a": 1, "b": "c"},
]
names = ["get_current_weather", "get_current_weather_2", "random", "random_2"]
model_output = "".join(
[
f"{parser.bot_token}{name}{json.dumps(args)}"
for name, args in zip(names, multiple_args_dict)
]
)
parsed = parser.extract_tool_calls(model_output, None)
# Assertions: the tool call is detected and the full nested JSON is parsed
# without truncation.
assert parsed.tools_called
assert len(parsed.tool_calls) == len(multiple_args_dict)
for i, tool_call in enumerate(parsed.tool_calls):
assert MistralToolCall.is_valid_id(tool_call.id)
assert tool_call.function.name == names[i]
assert json.loads(tool_call.function.arguments) == multiple_args_dict[i]
# No additional content outside the tool call should be returned.
assert parsed.content is None
@@ -1,160 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
from tests.models.utils import (
EmbedModelInfo,
RerankModelInfo,
)
from vllm.platforms import current_platform
from .mteb_embed_utils import mteb_test_embed_models
from .mteb_score_utils import mteb_test_rerank_models
MODELS = [
########## BertModel
EmbedModelInfo(
"thenlper/gte-large",
mteb_score=0.76807651,
mteb_tol=2e-3,
architecture="BertModel",
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo("thenlper/gte-base", architecture="BertModel", enable_test=False),
EmbedModelInfo("thenlper/gte-small", architecture="BertModel", enable_test=False),
EmbedModelInfo(
"thenlper/gte-large-zh", architecture="BertModel", enable_test=False
),
EmbedModelInfo("thenlper/gte-base-zh", architecture="BertModel", enable_test=False),
EmbedModelInfo(
"thenlper/gte-small-zh", architecture="BertModel", enable_test=False
),
########### NewModel
# These three architectures are almost the same, but not exactly the same.
# For example,
# - whether to use token_type_embeddings
# - whether to use context expansion
# So only test one (the most widely used) model
EmbedModelInfo(
"Alibaba-NLP/gte-multilingual-base",
architecture="GteNewModel",
mteb_score=0.775074696,
mteb_tol=2e-3,
hf_overrides={"architectures": ["GteNewModel"]},
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
EmbedModelInfo(
"Alibaba-NLP/gte-base-en-v1.5",
architecture="GteNewModel",
hf_overrides={"architectures": ["GteNewModel"]},
enable_test=False,
),
EmbedModelInfo(
"Alibaba-NLP/gte-large-en-v1.5",
architecture="GteNewModel",
hf_overrides={"architectures": ["GteNewModel"]},
enable_test=False,
),
########### Qwen2ForCausalLM
EmbedModelInfo(
"Alibaba-NLP/gte-Qwen2-1.5B-instruct",
mteb_score=0.758473459018872,
architecture="Qwen2ForCausalLM",
seq_pooling_type="LAST",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
# Skip: numerical regression with transformers v5.
enable_test=False,
),
########## ModernBertModel
EmbedModelInfo(
"Alibaba-NLP/gte-modernbert-base",
mteb_score=0.748193353,
mteb_tol=2e-3,
architecture="ModernBertModel",
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
########## Qwen3ForCausalLM
EmbedModelInfo(
"Qwen/Qwen3-Embedding-0.6B",
mteb_score=0.771163695,
mteb_tol=2e-3,
architecture="Qwen3ForCausalLM",
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
enable_test=True,
),
EmbedModelInfo(
"Qwen/Qwen3-Embedding-4B",
architecture="Qwen3ForCausalLM",
enable_test=False,
),
]
RERANK_MODELS = [
RerankModelInfo(
# classifier_pooling: mean
"Alibaba-NLP/gte-reranker-modernbert-base",
mteb_score=0.33386,
mteb_tol=1e-2,
architecture="ModernBertForSequenceClassification",
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
RerankModelInfo(
"Alibaba-NLP/gte-multilingual-reranker-base",
mteb_score=0.33062,
mteb_tol=1e-2,
architecture="GteNewForSequenceClassification",
hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
),
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
@pytest.mark.parametrize("model_info", MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
vllm_extra_kwargs = {}
if current_platform.is_rocm():
vllm_extra_kwargs["attention_backend"] = "TRITON_ATTN"
mteb_test_rerank_models(
vllm_runner, model_info, vllm_extra_kwargs=vllm_extra_kwargs
)
@@ -1,170 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import pytest
from tests.models.language.pooling.embed_utils import (
check_embeddings_close,
correctness_test_embed_models,
matryoshka_fy,
)
from tests.models.utils import (
EmbedModelInfo,
RerankModelInfo,
)
from vllm import PoolingParams
from vllm.platforms import current_platform
from .mteb_embed_utils import mteb_test_embed_models
from .mteb_score_utils import mteb_test_rerank_models
EMBEDDING_MODELS = [
EmbedModelInfo(
"jinaai/jina-embeddings-v3",
mteb_score=0.824413164,
mteb_tol=2e-3,
architecture="XLMRobertaModel",
is_matryoshka=True,
seq_pooling_type="MEAN",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
EmbedModelInfo(
"jinaai/jina-embeddings-v5-text-small",
mteb_score=0.794535707854956,
mteb_tol=2e-3,
architecture="JinaEmbeddingsV5Model",
seq_pooling_type="LAST",
attn_type="decoder",
is_prefix_caching_supported=True,
is_chunked_prefill_supported=True,
),
EmbedModelInfo(
"jinaai/jina-embeddings-v5-text-nano",
architecture="JinaEmbeddingsV5Model",
mteb_tol=2e-3,
dtype="bfloat16" if current_platform.is_rocm() else "auto",
seq_pooling_type="LAST",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
),
]
RERANK_MODELS = [
RerankModelInfo(
"jinaai/jina-reranker-v2-base-multilingual",
mteb_score=0.33643,
mteb_tol=1e-2,
architecture="XLMRobertaForSequenceClassification",
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
)
]
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
task = "retrieval" if "v5" in model_info.name else "text-matching"
prompt_prefix: str | None = "Document: " if "v5" in model_info.name else None
def hf_model_callback(model):
model.encode = partial(model.encode, task=task)
mteb_test_embed_models(
hf_runner,
vllm_runner,
model_info,
hf_model_callback=hf_model_callback,
prompt_prefix=prompt_prefix,
)
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
def test_embed_models_correctness(
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
) -> None:
task = "retrieval" if "v5" in model_info.name else "text-matching"
def hf_model_callback(model):
model.encode = partial(model.encode, task=task)
correctness_test_embed_models(
hf_runner,
vllm_runner,
model_info,
example_prompts,
hf_model_callback=hf_model_callback,
)
@pytest.mark.flaky(reruns=2)
@pytest.mark.parametrize("model_info", RERANK_MODELS)
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
mteb_test_rerank_models(vllm_runner, model_info)
@pytest.mark.skip(
reason="jinaai/jina-embeddings-v3 custom XLMRobertaLoRA model on HF hub "
"is incompatible with transformers v5 (missing all_tied_weights_keys)"
)
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("dimensions", [16, 32])
def test_matryoshka(
hf_runner,
vllm_runner,
model_info,
dtype: str,
dimensions: int,
example_prompts,
monkeypatch,
) -> None:
if not model_info.is_matryoshka:
pytest.skip("Model is not matryoshka")
# ST will strip the input texts, see test_embedding.py
example_prompts = [str(s).strip() for s in example_prompts]
task = "retrieval" if "v5" in model_info.name else "text-matching"
with hf_runner(
model_info.name,
dtype=dtype,
is_sentence_transformer=True,
) as hf_model:
hf_outputs = hf_model.encode(example_prompts, task=task)
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
with vllm_runner(
model_info.name, runner="pooling", dtype=dtype, max_model_len=None
) as vllm_model:
assert vllm_model.llm.llm_engine.model_config.is_matryoshka
matryoshka_dimensions = (
vllm_model.llm.llm_engine.model_config.matryoshka_dimensions
)
assert matryoshka_dimensions is not None
if dimensions not in matryoshka_dimensions:
with pytest.raises(ValueError):
vllm_model.embed(
example_prompts, pooling_params=PoolingParams(dimensions=dimensions)
)
else:
vllm_outputs = vllm_model.embed(
example_prompts, pooling_params=PoolingParams(dimensions=dimensions)
)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
@@ -1,78 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from tests.models.utils import EmbedModelInfo
from tests.quantization.utils import is_quant_method_supported
from vllm.platforms import current_platform
from .mteb_embed_utils import mteb_test_embed_models
def _fp8_scaled_mm_unsupported() -> bool:
"""Whether the ModernBERT online FP8 test should be skipped.
``is_quant_method_supported("fp8")`` returns True on MI250 (gfx90a), but
serving a per-tensor FP8 linear layer on ROCm requires CDNA3+ or RDNA4,
so treat it as unsupported here.
"""
if not is_quant_method_supported("fp8"):
return True
if current_platform.is_rocm():
from vllm.platforms.rocm import get_cdna_version, on_gfx12x
return get_cdna_version() <= 2 and not on_gfx12x()
return False
MODEL_INFO = EmbedModelInfo(
"Alibaba-NLP/gte-modernbert-base",
mteb_score=0.748193353,
architecture="ModernBertModel",
seq_pooling_type="CLS",
attn_type="encoder_only",
is_prefix_caching_supported=False,
is_chunked_prefill_supported=False,
enable_test=True,
)
def _assert_modernbert_online_fp8(model) -> None:
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
)
layer = model.encoder_layer.layers[0]
linears = {
"attn.Wqkv": layer.attn.Wqkv,
"attn.Wo": layer.attn.Wo,
"mlp.Wi": layer.mlp.Wi,
"mlp.Wo": layer.mlp.Wo,
}
for name, linear in linears.items():
assert isinstance(linear.quant_method, Fp8PerTensorOnlineLinearMethod), (
f"{name} was not quantized with online FP8"
)
@pytest.mark.flaky(reruns=2)
@pytest.mark.skipif(
_fp8_scaled_mm_unsupported(),
reason="No FP8 ScaledMM kernel is available on this GPU type.",
)
def test_modernbert_online_fp8_mteb(hf_runner, vllm_runner, monkeypatch) -> None:
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def check_online_fp8(vllm_model) -> None:
vllm_model.apply_model(_assert_modernbert_online_fp8)
mteb_test_embed_models(
hf_runner,
vllm_runner,
MODEL_INFO,
vllm_extra_kwargs={"quantization": "fp8_per_tensor"},
# Account for Online FP8 kernel variance across GPU architectures.
atol=2e-3,
vllm_model_callback=check_online_fp8,
)
@@ -1,171 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The vLLM team.
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
# reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm import LLM, SamplingParams
MODEL_NAME = "nvidia/audio-flamingo-3-hf"
SINGLE_CONVERSATION = [
{
"role": "user",
"content": [
{
"type": "text",
"text": "What is surprising about the relationship between "
"the barking and the music?",
},
{
"type": "audio_url",
"audio_url": {
"url": "https://huggingface.co/datasets/nvidia/AudioSkills/"
"resolve/main/assets/"
"dogs_barking_in_sync_with_the_music.wav",
},
},
],
}
]
BATCHED_CONVERSATIONS = [
SINGLE_CONVERSATION,
[
{
"role": "user",
"content": [
{
"type": "text",
"text": "Why is the philosopher's name mentioned in the "
"lyrics? (A) To express a sense of nostalgia "
"(B) To indicate that language cannot express clearly, "
"satirizing the inversion of black and white in the world "
"(C) To add depth and complexity to the lyrics "
"(D) To showcase the wisdom and influence of the "
"philosopher",
},
{
"type": "audio_url",
"audio_url": {
"url": "https://huggingface.co/datasets/nvidia/"
"AudioSkills/resolve/main/assets/"
"Ch6Ae9DT6Ko_00-04-03_00-04-31.wav",
},
},
],
}
],
]
def get_fixture_path(filename):
return os.path.join(
os.path.dirname(__file__), "../../fixtures/audioflamingo3", filename
)
def assert_output_matches(output, expected_text, expected_token_ids):
generated = output.outputs[0]
assert generated.text.strip() == expected_text
actual_token_ids = list(generated.token_ids)
assert (
actual_token_ids == expected_token_ids
or actual_token_ids == expected_token_ids[:-1]
or actual_token_ids[:-1] == expected_token_ids
)
@pytest.fixture(scope="module")
def llm():
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
model_info.check_transformers_version(on_fail="skip")
try:
return LLM(
model=MODEL_NAME,
dtype="bfloat16",
enforce_eager=True,
limit_mm_per_prompt={"audio": 1},
)
except Exception as e:
pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
def test_single_generation(llm):
fixture_path = get_fixture_path("expected_results_single.json")
if not os.path.exists(fixture_path):
pytest.skip(f"Fixture not found: {fixture_path}")
with open(fixture_path) as f:
expected = json.load(f)
sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
outputs = llm.chat(
messages=SINGLE_CONVERSATION,
sampling_params=sampling_params,
)
assert_output_matches(
outputs[0],
expected["transcriptions"][0],
expected["token_ids"][0],
)
def test_batched_generation(llm):
fixture_path = get_fixture_path("expected_results_batched.json")
if not os.path.exists(fixture_path):
pytest.skip(f"Fixture not found: {fixture_path}")
with open(fixture_path) as f:
expected = json.load(f)
sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
outputs = llm.chat(
messages=BATCHED_CONVERSATIONS,
sampling_params=sampling_params,
)
for i, output in enumerate(outputs):
assert_output_matches(
output,
expected["transcriptions"][i],
expected["token_ids"][i],
)
def test_single_and_batched_generation_match(llm):
sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
single_output = llm.chat(
messages=SINGLE_CONVERSATION,
sampling_params=sampling_params,
)[0]
batched_output = llm.chat(
messages=BATCHED_CONVERSATIONS,
sampling_params=sampling_params,
)[0]
assert single_output.outputs[0].text == batched_output.outputs[0].text
assert list(single_output.outputs[0].token_ids) == list(
batched_output.outputs[0].token_ids
)
File diff suppressed because it is too large Load Diff
@@ -1,182 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
import pytest
from transformers import AutoModelForSpeechSeq2Seq
from vllm.logprobs import SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.platforms import current_platform
from ....conftest import AudioTestAssets, HfRunner, PromptAudioInput, VllmRunner
from ...registry import HF_EXAMPLE_MODELS
from ...utils import check_logprobs_close
HF_AUDIO_PROMPT = "<|start_of_role|>system<|end_of_role|>Knowledge Cutoff Date: April 2024.\nToday's Date: December 19, 2024.\nYou are Granite, developed by IBM. You are a helpful AI assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|><|audio|>can you transcribe the speech into a written format?<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>" # noqa: E501
def vllm_to_hf_output(
vllm_output: tuple[list[int], str, SampleLogprobs | None],
) -> tuple[list[int], str, SampleLogprobs | None]:
"""Sanitize hf output to be comparable with vllm output."""
output_ids, output_str, out_logprobs = vllm_output
hf_output_str = output_str + "<|end_of_text|>"
return output_ids, hf_output_str, out_logprobs
MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech"
# "plus" variant of granite speech (uses GraniteSpeechPlusForConditionalGeneration).
MODEL_NAME_4_1_PLUS = "ibm-granite/granite-speech-4.1-2b-plus"
# Audio lora co-exists directly in the 3.3 model directory,
# the 4.0 and 4.1-plus models have adapters merged into the weights.
models: dict[str, str | None] = {
MODEL_NAME: MODEL_NAME,
MODEL_NAME_4_0: None,
MODEL_NAME_4_1_PLUS: None,
}
@pytest.fixture
def granite_speech_attention_config():
"""Return attention config for Granite Speech tests on ROCm."""
if current_platform.is_rocm():
from vllm.platforms.rocm import get_cdna_version
# -1 (unknown arch) is truthy; gate on CDNA3+ like other call sites.
if get_cdna_version() > 2:
return {"backend": "ROCM_AITER_FA"}
return {"backend": "TRITON_ATTN"}
return None
def run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
inputs: Sequence[tuple[list[str], PromptAudioInput]],
model: str,
*,
max_model_len: int,
dtype: str,
max_tokens: int,
num_logprobs: int,
tensor_parallel_size: int,
distributed_executor_backend: str | None = None,
attention_config: dict | None = None,
audio_lora_path: str | None = None,
):
"""Inference result should be the same between hf and vllm.
All the audio fixtures for the test are from AUDIO_ASSETS.
For huggingface runner, we provide the audio as input.
For vllm runner, we provide MultiModalDataDict objects
and corresponding MultiModalConfig as input.
Note, the text input is also adjusted to abide by vllm contract.
The text output is sanitized to be able to compare with hf.
"""
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
# max_model_len should be greater than image_feature_size
with vllm_runner(
model,
runner="generate",
max_model_len=max_model_len,
max_num_seqs=1,
dtype=dtype,
limit_mm_per_prompt={"audio": 1},
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enable_lora=audio_lora_path is not None,
max_lora_rank=64,
enforce_eager=True,
attention_config=attention_config,
) as vllm_model:
lora_request = (
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
)
vllm_outputs_per_case = [
vllm_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
audios=audios,
lora_request=lora_request,
)
for prompts, audios in inputs
]
with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
hf_processor = hf_model.processor
eos_token_id = hf_processor.tokenizer.eos_token_id
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
audios=[audios],
eos_token_id=eos_token_id,
)
for prompts, audios in inputs
]
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_logprobs_close(
outputs_0_lst=hf_outputs,
outputs_1_lst=[vllm_to_hf_output(output) for output in vllm_outputs],
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize("model,audio_lora_path", models.items())
@pytest.mark.parametrize(
"dtype", ["float16"] if current_platform.is_rocm() else ["bfloat16"]
)
@pytest.mark.parametrize(
"max_model_len", [512] if current_platform.is_rocm() else [2048]
)
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [10])
def test_models(
hf_runner,
vllm_runner,
model: str,
audio_lora_path: str | None,
audio_assets: AudioTestAssets,
granite_speech_attention_config,
dtype: str,
max_model_len: int,
max_tokens: int,
num_logprobs: int,
) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
audio, sr = audio_assets[0].audio_and_sample_rate
# This model expects 16k sample rate, which our test audio
# already is; if this changes, it may break this test,
# so we check it directly
assert sr == 16000
run_test(
hf_runner,
vllm_runner,
[
([HF_AUDIO_PROMPT], [audio]),
],
model,
dtype=dtype,
max_model_len=max_model_len,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
tensor_parallel_size=1,
attention_config=granite_speech_attention_config,
audio_lora_path=audio_lora_path,
)
@@ -1,76 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import NamedTuple
import pytest
from PIL.Image import Image
from transformers import AutoProcessor
from vllm import LLM, EngineArgs, SamplingParams
from vllm.multimodal.utils import encode_image_url
MODEL_NAME = "Kwai-Keye/Keye-VL-8B-Preview"
QUESTION = "What is the content of each image?"
class ModelRequestData(NamedTuple):
engine_args: EngineArgs
prompt: str
image_data: list[Image]
stop_token_ids: list[int] | None = None
chat_template: str | None = None
sampling_params: SamplingParams | None = None
@pytest.mark.parametrize("question", [QUESTION])
def test_keye_vl(image_assets, question: str):
images = [asset.pil_image for asset in image_assets]
image_urls = [encode_image_url(image) for image in images]
placeholders = [{"type": "image", "image": url} for url in image_urls]
messages = [
{
"role": "user",
"content": [
*placeholders,
{"type": "text", "text": question},
],
},
]
processor = AutoProcessor.from_pretrained(MODEL_NAME, trust_remote_code=True)
prompt = processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True
)
llm = LLM(
model=MODEL_NAME,
trust_remote_code=True,
max_model_len=8192,
max_num_seqs=5,
limit_mm_per_prompt={"image": len(image_urls)},
seed=42,
)
sampling_params = SamplingParams(
temperature=0.0, max_tokens=256, stop_token_ids=None
)
outputs = llm.generate(
{
"prompt": prompt,
"multi_modal_data": {"image": images},
},
sampling_params=sampling_params,
)
print("-" * 50)
for o in outputs:
generated_text = o.outputs[0].text
print(generated_text)
assert len(generated_text) > 10, (
f"Generated text is too short: {generated_text}"
)
print("-" * 50)
@@ -1,713 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Create a reduced-layer version of the Maverick model for testing purposes.
This script creates a new model with fewer layers by:
1. Loading the original Maverick model configuration
2. Creating a reduced configuration
3. Generating compatible safetensors files with appropriate weights
4. Creating the necessary index files for vLLM compatibility
"""
import json
import shutil
from pathlib import Path
from typing import Any
import pytest
import torch
from safetensors.torch import save_file
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, GenerationConfig
from vllm import LLM, SamplingParams
from vllm.v1.executor.abstract import Executor
from vllm.v1.kv_cache_interface import ChunkedLocalAttentionSpec, FullAttentionSpec
from ....utils import multi_gpu_test
# Sample prompts for testing
PROMPTS: list[str] = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
def run_maverick_serving(model: str):
"""Test Llama-4-Maverick model with vLLM LLM class using CLI equivalent
options with reduced layers.
"""
try:
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
llm = LLM(
model=model,
max_model_len=2048,
enforce_eager=True,
tensor_parallel_size=8,
enable_expert_parallel=True,
trust_remote_code=True,
gpu_memory_utilization=0.4,
kv_cache_dtype="fp8",
)
outputs = llm.generate(PROMPTS, sampling_params)
# Print the outputs
print("\nGenerated Outputs:\n" + "-" * 60)
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}")
print(f"Output: {generated_text!r}")
print("-" * 60)
except Exception as e:
print(f"Error initializing or running model: {e}")
raise
def get_rope_layers_config(model_path: str) -> list[int]:
"""Get the interleaved RoPE configuration from HuggingFace config.
Args:
model_path: Path to the local directory containing the reduced
Maverick model checkpoint
Returns:
List of 0 or 1 indicating whether each layer uses RoPE and local attn
0 indicates that RoPE is not used while 1 indicates that RoPE is used.
"""
config_path = Path(model_path) / "config.json"
model_config = json.loads(config_path.read_text())
text_config = model_config["text_config"]
no_rope_layers = text_config["no_rope_layers"]
print(f"Found no_rope_layers: {no_rope_layers}")
return no_rope_layers
def create_reduced_maverick_model(
original_model_name: str = "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
output_dir: str = "/tmp/reduced_maverick",
text_layers: int = 4,
num_experts: int = 4,
vision_layers: int = 2,
force_recreate: bool = False,
) -> str:
"""Create a reduced-layer version of the Maverick model.
Args:
original_model_name: Name of the original Maverick model
output_dir: Directory to save the reduced model
text_layers: Number of text transformer layers
num_experts: Number of experts per layer
vision_layers: Number of vision transformer layers
force_recreate: Whether to recreate if output_dir already exists
Returns:
Path to the created reduced model directory
"""
print(
f"Creating reduced Maverick model with {text_layers} text layers and "
f"{vision_layers} vision layers..."
)
# Create output directory
output_path = Path(output_dir)
if output_path.exists():
if force_recreate:
shutil.rmtree(output_path)
else:
print(
f"Output directory {output_dir} already exists. "
"Use --force-recreate to overwrite."
)
return str(output_path)
output_path.mkdir(parents=True, exist_ok=True)
try:
print("Loading original model configuration...")
original_config = AutoConfig.from_pretrained(
original_model_name, trust_remote_code=True
)
print("Creating reduced configuration...")
reduced_config = create_reduced_config(
original_config, text_layers, num_experts, vision_layers
)
config_path = output_path / "config.json"
with open(config_path, "w") as f:
json.dump(reduced_config, f, indent=2)
print(f"Saved reduced config to {config_path}")
print("Copying tokenizer files...")
copy_tokenizer_files(original_model_name, output_path)
print("Creating reduced safetensors files...")
create_reduced_safetensors(original_config, reduced_config, output_path)
print("Creating preprocessor config...")
create_preprocessor_config(original_config, output_path)
try:
gen_config = GenerationConfig.from_pretrained(original_model_name)
gen_config.save_pretrained(output_path)
print("Copied generation config")
except Exception as e:
print(f"Could not copy generation config: {e}")
print(f"Successfully created reduced Maverick model at {output_path}")
return str(output_path)
except Exception as e:
print(f"Error creating reduced model: {e}")
# Clean up on failure
if output_path.exists():
shutil.rmtree(output_path)
raise
def create_reduced_config(
original_config: Any, text_layers: int, num_experts: int, vision_layers: int
) -> dict[str, Any]:
"""Create a reduced configuration based on the original."""
# Convert config to dictionary
config_dict = original_config.to_dict()
# Reduce text layers
if "text_config" in config_dict:
original_text_layers = config_dict["text_config"]["num_hidden_layers"]
config_dict["text_config"]["num_hidden_layers"] = text_layers
original_layer_types = config_dict["text_config"]["layer_types"]
config_dict["text_config"]["layer_types"] = original_layer_types[:text_layers]
print(f"Reduced text layers from {original_text_layers} to {text_layers}")
original_num_experts = config_dict["text_config"]["num_local_experts"]
config_dict["text_config"]["num_local_experts"] = num_experts
print(f"Reduced num experts from {original_num_experts} to {num_experts}")
hidden_dim_divisor = 4
original_hidden_size = config_dict["text_config"]["hidden_size"]
new_hidden_size = original_hidden_size // hidden_dim_divisor
config_dict["text_config"]["hidden_size"] = new_hidden_size
print(f"Reduced hidden size from {original_hidden_size} to {new_hidden_size}")
original_head_dim = config_dict["text_config"]["head_dim"]
new_head_dim = original_head_dim // hidden_dim_divisor
config_dict["text_config"]["head_dim"] = new_head_dim
print(f"Reduced head dim from {original_head_dim} to {new_head_dim}")
# Reduce vision layers
if "vision_config" in config_dict:
original_vision_layers = config_dict["vision_config"]["num_hidden_layers"]
config_dict["vision_config"]["num_hidden_layers"] = vision_layers
print(f"Reduced vision layers from {original_vision_layers} to {vision_layers}")
# Update model name to indicate it's a reduced version
config_dict["_name_or_path"] = f"reduced_maverick_{text_layers}t_{vision_layers}v"
return config_dict
def copy_tokenizer_files(original_model_name: str, output_path: Path) -> None:
"""Copy tokenizer files from the original model."""
try:
tokenizer = AutoTokenizer.from_pretrained(
original_model_name, trust_remote_code=True
)
tokenizer.save_pretrained(output_path)
print("Tokenizer files copied successfully")
except Exception as e:
print(f"Warning: Could not copy tokenizer files: {e}")
def create_preprocessor_config(original_config: Any, output_path: Path) -> None:
"""Create preprocessor_config.json for multimodal model."""
# Try to load the original preprocessor config
try:
processor = AutoProcessor.from_pretrained(
original_config._name_or_path
or "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
trust_remote_code=True,
)
processor.save_pretrained(output_path)
print("Copied original preprocessor config")
return
except Exception as e:
print(f"Could not copy original preprocessor config: {e}")
raise
def create_reduced_safetensors(
original_config: Any, reduced_config: dict[str, Any], output_path: Path
) -> None:
"""Create safetensors files with weights for the reduced model."""
print("Generating synthetic weights for reduced model...")
text_config = reduced_config["text_config"]
vision_config = reduced_config["vision_config"]
weights = {}
print("Creating text model weights...")
weights.update(create_text_model_weights(text_config))
print("Creating vision model weights...")
weights.update(create_vision_model_weights(vision_config))
print("Creating shared model weights...")
weights.update(create_shared_weights(text_config, vision_config))
print("Saving weights to safetensors files...")
save_weights_to_safetensors(weights, output_path)
def create_text_model_weights(text_config: dict[str, Any]) -> dict[str, torch.Tensor]:
"""Create synthetic weights for the text model with MoE structure."""
weights = {}
vocab_size = text_config["vocab_size"]
hidden_size = text_config["hidden_size"]
intermediate_size = text_config["intermediate_size"]
intermediate_size_mlp = text_config["intermediate_size_mlp"]
num_layers = text_config["num_hidden_layers"]
num_attention_heads = text_config["num_attention_heads"]
num_key_value_heads = text_config.get("num_key_value_heads", num_attention_heads)
# MoE specific parameters
num_experts = text_config.get("num_local_experts")
assert num_experts is not None, "num_local_experts must be specified for MoE"
head_dim = hidden_size // num_attention_heads
# Embedding layers
weights["language_model.model.embed_tokens.weight"] = torch.randn(
vocab_size, hidden_size, dtype=torch.float16
)
# Transformer layers
for layer_idx in range(num_layers):
layer_prefix = f"language_model.model.layers.{layer_idx}"
print(f"Creating weights for layer {layer_prefix}...")
# Self-attention weights (separate q, k, v projections)
weights[f"{layer_prefix}.self_attn.q_proj.weight"] = torch.randn(
num_attention_heads * head_dim, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.k_proj.weight"] = torch.randn(
num_key_value_heads * head_dim, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.v_proj.weight"] = torch.randn(
num_key_value_heads * head_dim, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.o_proj.weight"] = torch.randn(
hidden_size, num_attention_heads * head_dim, dtype=torch.bfloat16
)
print("Self-attention weights created.")
# Feed-forward weights - MoE pattern based on interleave_moe_layer_step
# For interleave_moe_layer_step=2: layers 1,3,5,... are MoE, layers
# 0,2,4,... are dense
interleave_step = text_config.get("interleave_moe_layer_step", 1)
is_moe_layer = interleave_step > 0 and (layer_idx + 1) % interleave_step == 0
if is_moe_layer:
# MoE layer structure
# 1. Router weights
weights[f"{layer_prefix}.feed_forward.router.weight"] = torch.randn(
num_experts, hidden_size, dtype=torch.float16
)
# 2. Individual expert weights (not fused)
for expert_idx in range(num_experts):
expert_prefix = f"{layer_prefix}.feed_forward.experts.{expert_idx}"
weights[f"{expert_prefix}.gate_proj.weight"] = torch.randn(
intermediate_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{expert_prefix}.up_proj.weight"] = torch.randn(
intermediate_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{expert_prefix}.down_proj.weight"] = torch.randn(
hidden_size, intermediate_size, dtype=torch.bfloat16
)
# Expert weight scales (FP8 quantization)
weights[f"{expert_prefix}.gate_proj.weight_scale"] = torch.ones(
intermediate_size, 1, dtype=torch.bfloat16
)
weights[f"{expert_prefix}.up_proj.weight_scale"] = torch.ones(
intermediate_size, 1, dtype=torch.bfloat16
)
weights[f"{expert_prefix}.down_proj.weight_scale"] = torch.ones(
hidden_size, 1, dtype=torch.bfloat16
)
# 3. Shared expert weights
shared_expert_prefix = f"{layer_prefix}.feed_forward.shared_expert"
weights[f"{shared_expert_prefix}.gate_proj.weight"] = torch.randn(
intermediate_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{shared_expert_prefix}.up_proj.weight"] = torch.randn(
intermediate_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{shared_expert_prefix}.down_proj.weight"] = torch.randn(
hidden_size, intermediate_size, dtype=torch.bfloat16
)
print(f"MoE feed-forward weights created for layer {layer_idx}.")
else:
# Dense layer structure
weights[f"{layer_prefix}.feed_forward.gate_proj.weight"] = torch.randn(
intermediate_size_mlp, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.feed_forward.up_proj.weight"] = torch.randn(
intermediate_size_mlp, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.feed_forward.down_proj.weight"] = torch.randn(
hidden_size, intermediate_size_mlp, dtype=torch.bfloat16
)
print(f"Dense feed-forward weights created for layer {layer_idx}.")
# Layer norms
weights[f"{layer_prefix}.input_layernorm.weight"] = torch.ones(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.post_attention_layernorm.weight"] = torch.ones(
hidden_size, dtype=torch.bfloat16
)
print("Layer norms created.")
# Final layer norm and output projection
weights["language_model.model.norm.weight"] = torch.ones(
hidden_size, dtype=torch.bfloat16
)
weights["language_model.lm_head.weight"] = torch.randn(
vocab_size, hidden_size, dtype=torch.bfloat16
)
return weights
def create_vision_model_weights(
vision_config: dict[str, Any],
) -> dict[str, torch.Tensor]:
"""Create synthetic weights for the vision model."""
weights = {}
hidden_size = vision_config["hidden_size"]
intermediate_size = vision_config["intermediate_size"]
num_layers = vision_config["num_hidden_layers"]
# Vision transformer layers
for layer_idx in range(num_layers):
layer_prefix = f"vision_model.model.layers.{layer_idx}"
weights[f"{layer_prefix}.self_attn.q_proj.weight"] = torch.randn(
hidden_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.q_proj.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.k_proj.weight"] = torch.randn(
hidden_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.k_proj.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.v_proj.weight"] = torch.randn(
hidden_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.v_proj.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.o_proj.weight"] = torch.randn(
hidden_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.self_attn.o_proj.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.mlp.fc1.weight"] = torch.randn(
intermediate_size, hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.mlp.fc1.bias"] = torch.zeros(
intermediate_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.mlp.fc2.weight"] = torch.randn(
hidden_size, intermediate_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.mlp.fc2.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.input_layernorm.weight"] = torch.ones(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.input_layernorm.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.post_attention_layernorm.weight"] = torch.ones(
hidden_size, dtype=torch.bfloat16
)
weights[f"{layer_prefix}.post_attention_layernorm.bias"] = torch.zeros(
hidden_size, dtype=torch.bfloat16
)
return weights
def create_shared_weights(
text_config: dict[str, Any], vision_config: dict[str, Any]
) -> dict[str, torch.Tensor]:
"""Create weights for shared components (vision-language connector)."""
weights = {}
text_hidden_size = text_config["hidden_size"]
projector_input_dim = vision_config["projector_input_dim"]
# Vision-language connector (projects vision features to text space)
weights["multi_modal_projector.linear_1.weight"] = torch.randn(
text_hidden_size, projector_input_dim, dtype=torch.bfloat16
)
return weights
def save_weights_to_safetensors(
weights: dict[str, torch.Tensor], output_path: Path
) -> None:
"""Save weights to safetensors files and create index."""
# Determine how to shard the weights
max_shard_size = 5 * 1024 * 1024 * 1024 # 5GB per shard
# Calculate sizes and create shards
shards = []
current_shard: dict[str, torch.Tensor] = {}
current_size = 0
for name, tensor in weights.items():
tensor_size = tensor.numel() * tensor.element_size()
if current_size + tensor_size > max_shard_size and current_shard:
shards.append(current_shard)
current_shard = {}
current_size = 0
current_shard[name] = tensor
current_size += tensor_size
if current_shard:
shards.append(current_shard)
# Save shards and create index
weight_map = {}
if len(shards) == 1:
# Single file
filename = "model.safetensors"
save_file(shards[0], output_path / filename)
weight_map = {name: filename for name in shards[0]}
print(f"Saved weights to single file: {filename}")
else:
# Multiple shards
for i, shard in enumerate(shards):
filename = f"model-{i + 1:05d}-of-{len(shards):05d}.safetensors"
save_file(shard, output_path / filename)
for name in shard:
weight_map[name] = filename
print(f"Saved shard {i + 1}/{len(shards)}: {filename}")
# Create index file
index_data = {
"metadata": {
"total_size": sum(
tensor.numel() * tensor.element_size() for tensor in weights.values()
)
},
"weight_map": weight_map,
}
index_path = output_path / "model.safetensors.index.json"
with open(index_path, "w") as f:
json.dump(index_data, f, indent=2)
print(f"Created index file: {index_path}")
print(
f"Total model size: {index_data['metadata']['total_size'] / (1024**3):.2f} GB"
)
def check_attention_spec_interleaved_rope(
llm: LLM,
num_attention_layers: int,
num_ranks: int,
rope_layers: list[int],
):
"""Check that the attention spec is correct."""
assert isinstance(llm.llm_engine.model_executor, Executor)
kv_cache_specs_per_rank = llm.llm_engine.model_executor.get_kv_cache_specs()
for rank in range(num_ranks):
kv_cache_specs = kv_cache_specs_per_rank[rank]
assert len(kv_cache_specs.keys()) == num_attention_layers
for i in range(num_attention_layers):
if rope_layers[i] == 0:
expected_spec = FullAttentionSpec
else:
expected_spec = ChunkedLocalAttentionSpec
assert isinstance(
kv_cache_specs[f"language_model.model.layers.{i}.self_attn.attn"],
expected_spec,
)
def run_reduced_model(llm: LLM, should_profile: bool = False) -> None:
"""Test the created reduced model with vLLM."""
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=50)
if should_profile:
llm.start_profile()
outputs = llm.generate(PROMPTS, sampling_params)
if should_profile:
llm.stop_profile()
print("Test generation successful!")
for output in outputs:
print(f"Prompt: {output.prompt}")
print(f"Output: {output.outputs[0].text}")
print("-" * 40)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"original_model_name,text_layers,num_experts,vision_layers,",
[("meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", 4, 4, 2)],
)
@pytest.mark.parametrize("enforce_eager", [True, False])
@pytest.mark.parametrize("tp,ep", [(2, True)])
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
def test_dummy_maverick(
monkeypatch,
original_model_name: str,
text_layers: int,
num_experts: int,
vision_layers: int,
enforce_eager: bool,
tp: int,
ep: bool,
output_dir: str = "/tmp/reduced_maverick",
force_recreate: bool = True,
profile: bool = False,
) -> None:
# Disable multiprocessing allows us to access model executor from LLM engine
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
# The forked test process may inherit CUDA state from test collection.
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
model_path = create_reduced_maverick_model(
original_model_name=original_model_name,
output_dir=output_dir,
text_layers=text_layers,
num_experts=num_experts,
vision_layers=vision_layers,
force_recreate=force_recreate,
)
print(f"\nReduced model created successfully at: {model_path}")
rope_layers = get_rope_layers_config(model_path)
llm = LLM(
model=model_path,
trust_remote_code=True,
max_model_len=512, # Small context for testing
gpu_memory_utilization=0.3, # Conservative memory usage
enforce_eager=enforce_eager,
tensor_parallel_size=tp,
enable_expert_parallel=ep,
)
check_attention_spec_interleaved_rope(
llm,
text_layers,
tp,
rope_layers,
)
print(f"\nTesting reduced model at {model_path}...")
run_reduced_model(llm=llm, should_profile=profile)
def main():
"""Main function to create and test the reduced model."""
import argparse
parser = argparse.ArgumentParser(
description="Create a reduced-layer Maverick model"
)
parser.add_argument(
"--output-dir",
default="/tmp/reduced_maverick",
help="Output directory for the reduced model",
)
parser.add_argument(
"--text-layers",
type=int,
default=4,
help="Number of text transformer layers",
)
parser.add_argument("--num-experts", type=int, default=4, help="Number of experts")
parser.add_argument(
"--vision-layers",
type=int,
default=2,
help="Number of vision transformer layers",
)
parser.add_argument(
"--force-recreate",
action="store_true",
help="Force recreation if output directory exists",
)
parser.add_argument(
"--test", action="store_true", help="Test the created model with vLLM"
)
parser.add_argument(
"--profile", action="store_true", help="Profile the created model with vLLM"
)
parser.add_argument(
"--test-original",
action="store_true",
help="Test the original model with vLLM",
)
parser.add_argument(
"--original-model",
default="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
help="Original model name to base the reduction on",
)
args = parser.parse_args()
if args.test:
test_dummy_maverick(
original_model_name=args.original_model,
output_dir=args.output_dir,
text_layers=args.text_layers,
num_experts=args.num_experts,
vision_layers=args.vision_layers,
force_recreate=args.force_recreate,
tp=2,
ep=True,
enforce_eager=True,
profile=args.profile,
)
if args.test_original:
run_maverick_serving(args.original_model)
if __name__ == "__main__":
exit(main())
@@ -1,183 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
import random
import string
import sys
import weakref
import pytest
import torch
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.platforms import current_platform
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
from vllm.utils.mem_utils import format_mib
MODEL_NAME = "Qwen/Qwen3-VL-4B-Instruct"
RANDOM_PREFIX_LEN = 100
TEST_IMAGE_NAMES = [
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
"Grayscale_8bits_palette_sample_image.png",
]
MAX_MODEL_LEN = 8192
REQUESTS_PER_ROUND = 4
WARMUP_ROUNDS = 2
MEASURED_ROUNDS = 16
GPU_GROWTH_THRESHOLD_MIB = 0
CPU_PEAK_GROWTH_THRESHOLD_MIB = 0
SAMPLING_PARAMS = SamplingParams(
temperature=0.0,
max_tokens=16,
)
def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
# Avoid obscuring memory leaks because of prefix caching
random_text = "".join(random.choices(string.ascii_uppercase, k=RANDOM_PREFIX_LEN))
return [
{
"role": "user",
"content": [
{
"type": "text",
"text": f"Ignore this random string: {random_text}",
},
{"type": "image_url", "image_url": {"url": image_url}},
{
"type": "text",
"text": "Describe this image in one short sentence.",
},
],
}
]
def _build_request_batch(
image_urls: list[str],
) -> list[list[ChatCompletionMessageParam]]:
return [
_make_messages(image_urls[i % len(image_urls)])
for i in range(REQUESTS_PER_ROUND)
]
def _ru_maxrss_bytes() -> int | None:
try:
import resource
except ImportError:
return None
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
if rss <= 0:
return 0
# Linux reports kilobytes, macOS reports bytes.
return rss if sys.platform == "darwin" else rss * KiB_bytes
def _gpu_used_bytes() -> int:
torch.accelerator.synchronize()
free_bytes, total_bytes = torch.accelerator.get_memory_info()
return int(total_bytes - free_bytes)
def _format_mib(num_bytes: int | None) -> str:
if num_bytes is None:
return "n/a"
return f"{format_mib(num_bytes)} MiB"
@pytest.fixture(scope="function")
def llm(monkeypatch):
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
# pytest caches the fixture so we use weakref.proxy to
# enable garbage collection
llm_kwargs = dict(
model=MODEL_NAME,
enforce_eager=True,
max_model_len=MAX_MODEL_LEN,
max_num_seqs=REQUESTS_PER_ROUND,
limit_mm_per_prompt={"image": 1},
seed=0,
disable_log_stats=True,
gpu_memory_utilization=0.8,
)
if current_platform.is_rocm():
llm_kwargs["attention_backend"] = "TRITON_ATTN"
llm = LLM(**llm_kwargs)
yield weakref.proxy(llm)
del llm
cleanup_dist_env_and_memory()
@pytest.mark.core_model
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_NAMES], indirect=True)
def test_no_memory_leak(llm, image_urls: list[str]) -> None:
model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
request_batch = _build_request_batch(image_urls)
# Establish a warmup baseline after model load and the first multimodal
# requests complete. Later rounds should remain near this steady state.
for _ in range(WARMUP_ROUNDS):
outputs = llm.chat(request_batch, sampling_params=SAMPLING_PARAMS)
assert len(outputs) == len(request_batch)
assert llm.llm_engine.get_num_unfinished_requests() == 0
del outputs
gc.collect()
warmup_gpu = _gpu_used_bytes()
warmup_cpu_peak = _ru_maxrss_bytes()
post_warmup_gpu_samples: list[int] = []
post_warmup_cpu_peak_samples: list[int] = []
for _ in range(MEASURED_ROUNDS):
outputs = llm.chat(request_batch, sampling_params=SAMPLING_PARAMS)
assert len(outputs) == len(request_batch)
assert llm.llm_engine.get_num_unfinished_requests() == 0
del outputs
gc.collect()
post_warmup_gpu_samples.append(_gpu_used_bytes())
cpu_peak = _ru_maxrss_bytes()
if cpu_peak is not None:
post_warmup_cpu_peak_samples.append(cpu_peak)
gpu_growth = max(post_warmup_gpu_samples) - warmup_gpu
gpu_threshold = GPU_GROWTH_THRESHOLD_MIB * MiB_bytes
assert gpu_growth <= gpu_threshold, (
"Qwen3-VL GPU memory kept growing after warmup. "
f"warmup_baseline={_format_mib(warmup_gpu)}, "
f"post_warmup_samples={[_format_mib(x) for x in post_warmup_gpu_samples]}, "
f"gpu_growth={_format_mib(gpu_growth)}, "
f"gpu_threshold={GPU_GROWTH_THRESHOLD_MIB} MiB"
)
if warmup_cpu_peak is not None and post_warmup_cpu_peak_samples:
cpu_peak_growth = max(post_warmup_cpu_peak_samples) - warmup_cpu_peak
cpu_threshold = CPU_PEAK_GROWTH_THRESHOLD_MIB * MiB_bytes
assert cpu_peak_growth <= cpu_threshold, (
"Qwen3-VL CPU peak RSS kept growing after warmup. "
f"warmup_ru_maxrss={_format_mib(warmup_cpu_peak)}, "
f"post_warmup_ru_maxrss={[_format_mib(x) for x in post_warmup_cpu_peak_samples]}, " # noqa: E501
f"cpu_peak_growth={_format_mib(cpu_peak_growth)}, "
f"cpu_peak_threshold={CPU_PEAK_GROWTH_THRESHOLD_MIB} MiB"
)
@@ -1,122 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
import torch
from transformers import AutoModelForImageTextToText
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from ....conftest import HfRunner, ImageTestAssets, VllmRunner
from .vlm_utils import model_utils
MODEL = "google/gemma-3-4b-it"
PROMPT = (
"<bos><start_of_turn>user\n"
"<start_of_image>What is the content in the center of the image?"
"<end_of_turn>\n<start_of_turn>model\n"
)
def _install_prefill_hidden_capture(model):
model = getattr(model, "module", model)
model._prefill_hidden = None
language_model = model.language_model.model
original_forward = language_model.forward
def forward(*args, **kwargs):
hidden_states = original_forward(*args, **kwargs)
if model._prefill_hidden is None and torch.is_tensor(hidden_states):
# Capturing hidden states for comparison is a deliberate D2H.
with gpu_sync_allowed():
model._prefill_hidden = hidden_states.detach().float().cpu()
return hidden_states
language_model.forward = forward
def _get_prefill_hidden(model):
model = getattr(model, "module", model)
hidden = getattr(model, "_prefill_hidden", None)
assert hidden is not None
return hidden
def _get_hf_prefill_hidden(hf_model: HfRunner, image: Any):
inputs = hf_model.get_inputs([PROMPT], images=[image])[0]
with torch.no_grad():
outputs = hf_model.model.model(
**hf_model.wrap_device(inputs),
use_cache=False,
)
return outputs.last_hidden_state[0].detach().float().cpu()
def _get_vllm_prefill_hidden(
vllm_runner: type[VllmRunner],
image: Any,
vllm_runner_kwargs: dict[str, Any],
):
with vllm_runner(
MODEL,
max_model_len=4096,
max_num_seqs=2,
enforce_eager=True,
limit_mm_per_prompt={"image": 1},
**vllm_runner_kwargs,
) as vllm_model:
vllm_model.apply_model(_install_prefill_hidden_capture)
vllm_model.generate_greedy([PROMPT], max_tokens=1, images=[image])
return vllm_model.apply_model(_get_prefill_hidden)[0]
@pytest.mark.core_model
@pytest.mark.skipif(
current_platform.is_rocm(), reason="ROCm attention has accuracy issue for this test"
)
def test_mm_prefix_lm_e2e(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets,
monkeypatch: pytest.MonkeyPatch,
):
"""Regression: Gemma3 native prefill must apply image prefix-LM mask."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
image = image_assets[0].pil_image
vllm_runner_kwargs: dict[str, Any] = {
"mm_processor_cache_gb": 0,
"mm_processor_kwargs": {"do_pan_and_scan": True},
}
vllm_hidden = _get_vllm_prefill_hidden(vllm_runner, image, vllm_runner_kwargs)
hf_model = hf_runner(
MODEL,
auto_cls=AutoModelForImageTextToText,
)
hf_model = model_utils.gemma3_patch_hf_runner(hf_model)
with hf_model:
hf_hidden = _get_hf_prefill_hidden(hf_model, image)
assert vllm_hidden.shape == hf_hidden.shape
full_cos = torch.nn.functional.cosine_similarity(
vllm_hidden.flatten(), hf_hidden.flatten(), dim=0
)
image_cos = torch.nn.functional.cosine_similarity(
vllm_hidden[1:769].flatten(), hf_hidden[1:769].flatten(), dim=0
)
assert full_cos > 0.9, (
"Gemma3 mm-prefix-LM full prefill hidden states should be close to HF; "
f"got {full_cos=}"
)
assert image_cos > 0.9, (
"Gemma3 mm-prefix-LM image prefill hidden states should be close to HF; "
f"got {image_cos=}"
)
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Generation tests for Moondream3 query and caption support."""
import pytest
from tests.models.registry import HF_EXAMPLE_MODELS
from vllm.platforms import current_platform
from ....conftest import IMAGE_ASSETS, ImageTestAssets
from ....utils import large_gpu_mark, multi_gpu_test
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
MOONDREAM3_TOKENIZER = "moondream/starmie-v1"
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color is the stop sign?<|md_reserved_2|>", # noqa: E501
"cherry_blossom": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color are the flowers?<|md_reserved_2|>", # noqa: E501
}
)
def make_query_prompt(question: str) -> str:
"""Create a direct-answer query prompt for Moondream3."""
return (
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
f"{question}<|md_reserved_2|>"
)
def make_caption_prompt(length: str = "normal") -> str:
"""Create a caption prompt for Moondream3."""
return (
"<|endoftext|><image><|md_reserved_0|>"
f"describe<|md_reserved_1|>{length}<|md_reserved_2|>"
)
@multi_gpu_test(num_gpus=2)
@large_gpu_mark(min_gb=80)
def test_tensor_parallel(image_assets: ImageTestAssets):
import gc
from vllm import LLM, SamplingParams
from vllm.distributed.parallel_state import destroy_model_parallel
destroy_model_parallel()
gc.collect()
current_platform.empty_cache()
llm = LLM(
model=MOONDREAM3_MODEL_ID,
tokenizer=MOONDREAM3_TOKENIZER,
trust_remote_code=True,
dtype="bfloat16",
tensor_parallel_size=2,
max_model_len=1024,
enforce_eager=True,
limit_mm_per_prompt={"image": 1},
gpu_memory_utilization=0.45,
)
image = image_assets[0].pil_image
prompt = make_query_prompt("What color is the stop sign?")
try:
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=20, temperature=0),
)
assert len(outputs) > 0
assert outputs[0].outputs[0].text is not None
finally:
del llm
gc.collect()
current_platform.empty_cache()
@pytest.fixture(scope="module")
def llm():
model_info = HF_EXAMPLE_MODELS.get_hf_info("Moondream3ForCausalLM")
model_info.check_transformers_version(on_fail="skip")
from vllm import LLM
try:
return LLM(
model=MOONDREAM3_MODEL_ID,
tokenizer=MOONDREAM3_TOKENIZER,
trust_remote_code=True,
dtype="bfloat16",
max_model_len=2048,
enforce_eager=True,
limit_mm_per_prompt={"image": 1},
gpu_memory_utilization=0.45,
)
except Exception as exc:
pytest.skip(f"Failed to load {MOONDREAM3_MODEL_ID}: {exc}")
@large_gpu_mark(min_gb=48)
def test_model_loading(llm):
assert llm is not None
@large_gpu_mark(min_gb=48)
def test_query_skill(llm, image_assets: ImageTestAssets):
from vllm import SamplingParams
image = image_assets[0].pil_image
prompt = make_query_prompt("What color is the stop sign?")
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=50, temperature=0),
)
output_text = outputs[0].outputs[0].text
assert output_text is not None
assert len(output_text) > 0
@large_gpu_mark(min_gb=48)
def test_caption_skill(llm, image_assets: ImageTestAssets):
from vllm import SamplingParams
image = image_assets[1].pil_image
prompt = make_caption_prompt()
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=100, temperature=0),
)
output_text = outputs[0].outputs[0].text
assert output_text is not None
assert len(output_text) > 0
@large_gpu_mark(min_gb=48)
def test_batched_inference(llm, image_assets: ImageTestAssets):
from vllm import SamplingParams
images = [asset.pil_image for asset in image_assets]
prompts = [
{"prompt": prompt, "multi_modal_data": {"image": img}}
for img, prompt in zip(images, HF_IMAGE_PROMPTS)
]
outputs = llm.generate(prompts, SamplingParams(max_tokens=50, temperature=0))
assert len(outputs) == len(images)
for output in outputs:
assert output.outputs[0].text is not None
assert len(output.outputs[0].text) > 0
@pytest.mark.parametrize("asset_name", ["stop_sign", "cherry_blossom"])
@large_gpu_mark(min_gb=48)
def test_image_assets(llm, image_assets: ImageTestAssets, asset_name: str):
from vllm import SamplingParams
asset_idx = 0 if asset_name == "stop_sign" else 1
image = image_assets[asset_idx].pil_image
prompt = HF_IMAGE_PROMPTS[asset_idx]
outputs = llm.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(max_tokens=50, temperature=0),
)
output_text = outputs[0].outputs[0].text
assert output_text is not None
assert len(output_text) > 0
@@ -1,132 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Sequence
import pytest
import regex as re
from transformers import AutoModel
from tests.models.utils import check_logprobs_close
from vllm.assets.image import ImageAsset
from vllm.logprobs import Logprob, SampleLogprobs
from vllm.tokenizers import TokenizerLike
from ....conftest import HfRunner, PromptImageInput, VllmRunner
IMAGE = ImageAsset("paper-11").pil_image_ext(ext="png").convert("RGB")
PROMPT = (
"</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"
)
class DummyLogprobs(dict[int, Logprob]):
def __init__(self, vocab_ids: Iterable[int]):
super().__init__(dict.fromkeys(vocab_ids, Logprob(0.0)))
def __repr__(self):
return "DummyLogprobs()"
def mask_bbox_tokens(
output: tuple[list[int], str, SampleLogprobs],
tokenizer: TokenizerLike,
) -> tuple[list[int], str, SampleLogprobs]:
"""Always pass check_logprobs_close check for bounding box tokens
because it is reasonable for them to differ slightly.
"""
ignore_pattern = r"<[xy]_[\d.]+>"
vocab = tokenizer.get_vocab()
output_ids, output_str, out_logprobs = output
masked_logprobs = list[dict[int, Logprob]]()
for token, logprobs in zip(output_ids, out_logprobs):
if re.match(ignore_pattern, tokenizer.decode(token)):
masked_logprobs.append(DummyLogprobs(vocab.values()))
else:
masked_logprobs.append(logprobs)
return output_ids, output_str, masked_logprobs
def run_test(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
inputs: Sequence[tuple[list[str], PromptImageInput]],
model: str,
*,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
"""Verify that the inference result is the same between hf and vllm."""
with vllm_runner(
model,
dtype=dtype,
max_num_seqs=64,
limit_mm_per_prompt={"image": 1},
trust_remote_code=True,
) as vllm_model:
vllm_outputs_per_case = [
vllm_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
)
for prompts, images in inputs
]
tokenizer = vllm_model.llm.get_tokenizer()
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
hf_outputs_per_case = [
hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
images=images,
tokenization_kwargs={"add_special_tokens": False},
)
for prompts, images in inputs
]
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
check_logprobs_close(
outputs_0_lst=[
mask_bbox_tokens(output, tokenizer) for output in hf_outputs
],
outputs_1_lst=[
mask_bbox_tokens(output, tokenizer) for output in vllm_outputs
],
name_0="hf",
name_1="vllm",
)
@pytest.mark.parametrize(
"model",
[
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
# instead of shipping a separate lm_head.weight tensor; this
# exercises that path (see tie_weights() in nemotron_parse.py).
"nvidia/NVIDIA-Nemotron-Parse-2.0",
],
)
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
hf_runner, vllm_runner, model: str, dtype: str, num_logprobs: int
) -> None:
run_test(
hf_runner,
vllm_runner,
inputs=[
([PROMPT] * 10, [IMAGE] * 10),
],
model=model,
dtype=dtype,
max_tokens=100,
num_logprobs=num_logprobs,
)
@@ -1,261 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.image import ImageAsset
from vllm.platforms import current_platform
from ....conftest import VIDEO_ASSETS
from .vlm_utils.builders import sample_frames_with_video_metadata
models = ["Qwen/Qwen2.5-VL-3B-Instruct"]
target_dtype = "bfloat16"
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
def qwen2_5_vl_chat_template(*query):
return f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{''.join(query)}<|im_end|><|im_start|>assistant\n" # noqa: E501
VIDEO_PROMPTS = VIDEO_ASSETS.prompts(
{
"baby_reading": qwen2_5_vl_chat_template(
VIDEO_PLACEHOLDER,
"Describe this video with a short sentence ",
"(no more than 20 words)",
),
}
)
WINDOW_ATTN_IMAGE_PROMPT = qwen2_5_vl_chat_template(
IMAGE_PLACEHOLDER,
"Describe the image.",
)
IMAGE_ONLY_LIMIT_MM_PER_PROMPT = {"image": 1, "video": 0}
def _window_attention_regression_image():
# image from regression issue: https://github.com/vllm-project/vllm/issues/15122
image = ImageAsset("hato").pil_image
return image.resize((image.width // 2, image.height // 2))
def _encoder_cudagraph_config(*, max_vision_items: int) -> dict:
return {
"cudagraph_mm_encoder": True,
"encoder_cudagraph_max_vision_items_per_batch": max_vision_items,
}
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"video_pruning_rate", [0.0] if current_platform.is_cpu() else [0.0, 0.75]
)
@pytest.mark.parametrize("num_frames", [16])
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize(
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
)
def test_qwen2_5_vl_evs_functionality(
vllm_runner,
video_assets,
model,
video_pruning_rate: float,
num_frames: int,
dtype: str,
max_tokens: int,
use_bytecode_hook: bool,
monkeypatch,
) -> None:
"""Test EVS (Efficient Video Sampling) functionality with different
pruning rates.
"""
# Set the environment variable for this test
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
# Sample frames from video assets
sampled_vids = [
sample_frames_with_video_metadata(
(asset.np_ndarrays, asset.metadata), num_frames
)
for asset in video_assets
]
prompts = [VIDEO_PROMPTS[0]]
videos = [sampled_vids[0]]
# Initialize model with EVS configuration
with vllm_runner(
model,
runner="generate",
max_model_len=4000,
dtype=dtype,
limit_mm_per_prompt={"video": 1},
video_pruning_rate=video_pruning_rate,
) as vllm_model:
# Generate output - this should not crash
outputs = vllm_model.generate_greedy(prompts, max_tokens, videos=videos)
# Basic validation that we got a response
assert len(outputs) == 1
output_ids, output_text = outputs[0]
# Ensure we got some output
assert len(output_ids) > 0
assert len(output_text) > 0
# Ensure the output is a string
assert isinstance(output_text, str)
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize(
"video_pruning_rate", [0.0] if current_platform.is_cpu() else [0.0, 0.75]
)
@pytest.mark.parametrize("num_frames", [16])
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize(
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
)
def test_qwen2_5_vl_evs_batched_videos(
vllm_runner,
video_assets,
model,
video_pruning_rate: float,
num_frames: int,
dtype: str,
max_tokens: int,
use_bytecode_hook: bool,
monkeypatch,
) -> None:
"""Test EVS functionality with batched videos.
This test validates that:
1. The model handles batched video inputs correctly with EVS
2. Both pruning configurations work with multiple videos
3. The model doesn't crash when processing multiple videos simultaneously
"""
# Set the environment variable for this test
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
# Sample frames from video assets
sampled_vids = [
sample_frames_with_video_metadata(
(asset.np_ndarrays, asset.metadata), num_frames
)
for asset in video_assets
]
# Test batched videos
prompts = [VIDEO_PROMPTS[0], VIDEO_PROMPTS[0]]
videos = [sampled_vids[0], sampled_vids[0]] # Use same video twice for testing
# Initialize model with EVS configuration
with vllm_runner(
model,
runner="generate",
max_model_len=4000,
max_num_seqs=2,
dtype=dtype,
limit_mm_per_prompt={"video": 2},
tensor_parallel_size=1,
video_pruning_rate=video_pruning_rate,
) as vllm_model:
# Generate output - this should not crash
outputs = vllm_model.generate_greedy(prompts, max_tokens, videos=videos)
# Basic validation that we got responses for both videos
assert len(outputs) == 2
for output_ids, output_text in outputs:
# Ensure we got some output for each video
assert len(output_ids) > 0
assert len(output_text) > 0
# Ensure the output is a string
assert isinstance(output_text, str)
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize(
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
)
def test_qwen2_5_vl_window_attention_image(
vllm_runner,
model,
dtype: str,
max_tokens: int,
use_bytecode_hook: bool,
monkeypatch,
) -> None:
"""Regression test for Qwen2.5 window-attention image path."""
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
prompt = [WINDOW_ATTN_IMAGE_PROMPT]
images = [[_window_attention_regression_image()]]
with vllm_runner(
model,
runner="generate",
max_model_len=4096,
dtype=dtype,
limit_mm_per_prompt=IMAGE_ONLY_LIMIT_MM_PER_PROMPT,
compilation_config=_encoder_cudagraph_config(max_vision_items=1),
) as vllm_model:
outputs = vllm_model.generate_greedy(prompt, max_tokens, images=images)
assert len(outputs) == 1
output_ids, output_text = outputs[0]
assert len(output_ids) > 0
assert len(output_text) > 0
assert isinstance(output_text, str)
@pytest.mark.core_model
@pytest.mark.parametrize("model", models)
@pytest.mark.parametrize("dtype", [target_dtype])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize(
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
)
def test_qwen2_5_vl_window_attention_image_batch(
vllm_runner,
model,
dtype: str,
max_tokens: int,
use_bytecode_hook: bool,
monkeypatch,
) -> None:
"""Regression test window-attention with a small image batch."""
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
image = _window_attention_regression_image()
prompts = [WINDOW_ATTN_IMAGE_PROMPT, WINDOW_ATTN_IMAGE_PROMPT]
images = [[image], [image]]
with vllm_runner(
model,
runner="generate",
max_model_len=4096,
max_num_seqs=2,
dtype=dtype,
limit_mm_per_prompt=IMAGE_ONLY_LIMIT_MM_PER_PROMPT,
compilation_config=_encoder_cudagraph_config(max_vision_items=2),
) as vllm_model:
outputs = vllm_model.generate_greedy(prompts, max_tokens, images=images)
assert len(outputs) == 2
for output_ids, output_text in outputs:
assert len(output_ids) > 0
assert len(output_text) > 0
assert isinstance(output_text, str)
@@ -1,674 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass, field
from functools import partial
import pytest
from vllm.multimodal.video import sample_frames_from_video
from vllm.platforms import current_platform
from ....conftest import IMAGE_ASSETS, VIDEO_ASSETS
from ...utils import dummy_hf_overrides
from .vlm_utils.builders import sample_frames_with_video_metadata
@dataclass
class VitCudagraphTestConfig:
model: str
modalities: list[str] = field(default_factory=lambda: ["image", "video"])
image_prompt: str | None = None
video_prompt: str | None = None
dtype: str = "bfloat16"
max_model_len: int = 4096
max_tokens: int = 64
max_num_seqs: int = 2
num_video_frames: int = 16
needs_video_metadata: bool = False
vllm_runner_kwargs: dict = field(default_factory=dict)
compilation_config_overrides: dict = field(default_factory=dict)
marks: list = field(default_factory=list)
skip: bool = False
def params_with_marks(
configs: dict[str, VitCudagraphTestConfig],
) -> list[pytest.param]:
return [
pytest.param(model_id, marks=cfg.marks) for model_id, cfg in configs.items()
]
def qwen_vl_chat_template(content: str) -> str:
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
def internvl_chat_template(content: str) -> str:
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
def kimi_vl_chat_template(content: str) -> str:
return (
f"<|im_user|>user<|im_middle|>{content}<|im_end|>"
"<|im_assistant|>assistant<|im_middle|>"
)
def deepseek_v41_chat_template(content: str) -> str:
return f"<|begin▁of▁sentence|><|User|>{content}<|Assistant|></think>"
def step3_vl_chat_template(content: str) -> str:
return (
"<|begin▁of▁sentence|> You are a helpful assistant.<|BOT|>user\n "
f"<im_patch>{content} <|EOT|><|BOT|>assistant\n"
)
def gemma3_chat_template(content: str) -> str:
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
def deepseek_vl2_chat_template(content: str) -> str:
return f"<|User|>: {content}\n\n<|Assistant|>:"
def ernie45_vl_chat_template(content: str) -> str:
return (
f"<|begin_of_sentence|>User: {content}"
"Picture 1:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>\n"
"Assistant: <think></think>"
)
def minicpmv_25_chat_template(content: str) -> str:
"""Llama3-style chat template used by MiniCPM-V 2.5."""
return (
f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
f"{content}"
f"<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
)
def minicpmv_chat_template(content: str) -> str:
"""ChatML template used by MiniCPM-V 2.6 / 4.0 / 4.5."""
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
"gemma3": VitCudagraphTestConfig(
model="google/gemma-3-4b-it",
modalities=["image"],
image_prompt=gemma3_chat_template("<start_of_image>What is in this image?"),
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [512],
},
dtype="bfloat16",
max_model_len=4096,
),
"llama4": VitCudagraphTestConfig(
model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
modalities=["image"],
image_prompt=(
"<|begin_of_text|><|header_start|>user<|header_end|>\n\n"
"<|image|>What is in this image?<|eot|>"
"<|header_start|>assistant<|header_end|>\n\n"
),
max_model_len=4096,
max_tokens=32,
max_num_seqs=2,
vllm_runner_kwargs={
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="Llama4ForConditionalGeneration",
),
},
marks=[pytest.mark.core_model],
),
"qwen2_vl": VitCudagraphTestConfig(
model="Qwen/Qwen2-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
),
video_prompt=qwen_vl_chat_template(
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
),
needs_video_metadata=False,
marks=[pytest.mark.core_model],
),
"qwen2_5_vl": VitCudagraphTestConfig(
model="Qwen/Qwen2.5-VL-3B-Instruct",
image_prompt=qwen_vl_chat_template(
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
),
video_prompt=qwen_vl_chat_template(
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
),
needs_video_metadata=True,
marks=[pytest.mark.core_model],
),
"kimi_vl": VitCudagraphTestConfig(
model="moonshotai/Kimi-VL-A3B-Instruct",
modalities=["image"],
image_prompt=kimi_vl_chat_template(
"<|media_start|>image<|media_content|><|media_pad|><|media_end|>"
"What is in this image?"
),
needs_video_metadata=False,
# Single bucket sized to cover the test images' output tokens.
# The default auto-inferred range fans out into multiple power-of-2
# buckets, each holding a full ViT capture pool.
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [1024],
},
# Shrink to 1 text + 1 vision layer with random weights so the
# test runs on any CI GPU (incl. L4) and skips the multi-GiB
# weight download. The test only validates that encoder CG
# capture/replay functions correctly, not output quality.
vllm_runner_kwargs={
"trust_remote_code": True,
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="KimiVLForConditionalGeneration",
),
},
marks=[pytest.mark.core_model],
),
"minimax_m3": VitCudagraphTestConfig(
model="MiniMaxAI/MiniMax-M3",
modalities=["image"],
image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"),
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [512],
},
vllm_runner_kwargs={
"trust_remote_code": True,
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="MiniMaxM3SparseForConditionalGeneration",
),
"mm_processor_kwargs": {"max_long_side_pixel": 336},
},
marks=[pytest.mark.core_model],
),
"qwen3_vl": VitCudagraphTestConfig(
model="Qwen/Qwen3-VL-2B-Instruct",
image_prompt=qwen_vl_chat_template(
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
),
video_prompt=qwen_vl_chat_template(
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
),
needs_video_metadata=True,
marks=[pytest.mark.core_model],
),
"qwen3_5": VitCudagraphTestConfig(
model="Qwen/Qwen3.5-0.8B",
image_prompt=qwen_vl_chat_template(
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
),
video_prompt=qwen_vl_chat_template(
"<|vision_start|><|video_pad|><|vision_end|>"
"Describe this video in one sentence."
),
needs_video_metadata=True,
vllm_runner_kwargs={"enable_chunked_prefill": True},
marks=[pytest.mark.core_model],
),
"internvl": VitCudagraphTestConfig(
model="OpenGVLab/InternVL3-1B",
num_video_frames=8,
image_prompt=internvl_chat_template("<image>\nWhat is in this image?"),
video_prompt=internvl_chat_template(
"<video>\nDescribe this video in one sentence."
),
needs_video_metadata=False,
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"idefics3": VitCudagraphTestConfig(
model="HuggingFaceTB/SmolVLM-256M-Instruct",
modalities=["image"],
image_prompt=(
"<|begin_of_text|>User:<image>What is in this image?"
"<end_of_utterance>\nAssistant:"
),
max_model_len=4096,
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [4096],
},
vllm_runner_kwargs={"gpu_memory_utilization": 0.80},
marks=[pytest.mark.core_model],
),
"deepseek_v41": VitCudagraphTestConfig(
model="deepseek-ai/DeepSeek-V4.1-Flash",
modalities=["image"],
image_prompt=deepseek_v41_chat_template(
"<|deepseek_image|>\n\nWhat is in this image?"
),
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [1024],
},
vllm_runner_kwargs={
"load_format": "dummy",
"attention_backend": (
"ROCM_FLASHMLA_SPARSE_DSV41"
if current_platform.is_rocm()
else "FLASHMLA_SPARSE_DSV41"
),
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="DeepseekV41ForCausalLM",
exist_overrides={"vision_n_layers": 1},
),
},
),
"step3_vl": VitCudagraphTestConfig(
model="stepfun-ai/Step3-VL-10B",
modalities=["image"],
image_prompt=step3_vl_chat_template("What is in this image?"),
# Single bucket sized to cover the largest test image's output
# tokens (1152 > 1141 for cherry_blossom). The default auto-
# inferred range fans out into multiple power-of-2 buckets, each
# holding a full ViT capture pool.
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [1152],
},
# Shrink to 1 text + 1 vision layer with random weights so the
# test runs on any CI GPU (incl. L4) and skips the 20 GiB weight
# download. The test only validates that encoder CG capture/
# replay functions correctly, not output quality.
vllm_runner_kwargs={
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="StepVLForConditionalGeneration",
),
},
),
"ernie45_vl": VitCudagraphTestConfig(
model="baidu/ERNIE-4.5-VL-28B-A3B-PT",
# Image only: Ernie's resampler applies a temporal conv for video
# that changes the output token count, so video uses the eager path.
modalities=["image"],
image_prompt=ernie45_vl_chat_template("What is in this image?"),
# Ernie4_5_VLMoeModel is deliberately not torch-compiled, since its
# split of text and vision experts breaks compilation, so piecewise
# cudagraphs have nothing to partition. Only the encoder graphs this
# test covers are captured.
compilation_config_overrides={
"cudagraph_mode": 2,
},
# Shrink to 1 text + 1 vision layer with random weights so the test
# runs on any CI GPU and skips the ~56 GiB weight download. The test
# only validates encoder CG capture/replay, not output quality.
vllm_runner_kwargs={
"load_format": "dummy",
"trust_remote_code": True,
"revision": "refs/pr/17",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="Ernie4_5_VLMoeForConditionalGeneration",
),
},
),
"glm4_1v": VitCudagraphTestConfig(
model="zai-org/GLM-4.1V-9B-Thinking",
image_prompt=(
"[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
"<|begin_of_image|><|image|><|end_of_image|>"
"What is in this image?<|assistant|>assistant\n"
),
video_prompt=(
"[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
"<|begin_of_video|><|video|><|end_of_video|>"
"Describe this video in one sentence<|assistant|>assistant\n"
),
needs_video_metadata=True,
marks=[pytest.mark.core_model],
vllm_runner_kwargs={
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="Glm4vForConditionalGeneration",
),
},
),
"deepseek_ocr": VitCudagraphTestConfig(
model="deepseek-ai/DeepSeek-OCR",
modalities=["image"],
image_prompt="<image>\nWhat is in this image?",
marks=[pytest.mark.core_model],
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [272],
"mode": 0,
"cudagraph_mode": 2,
},
vllm_runner_kwargs={
"load_format": "dummy",
"hf_overrides": partial(
dummy_hf_overrides,
model_arch="DeepseekOCRForCausalLM",
),
},
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
),
"deepseek_vl2": VitCudagraphTestConfig(
model="deepseek-ai/deepseek-vl2-tiny",
modalities=["image"],
image_prompt=deepseek_vl2_chat_template("<image>\nWhat is in this image?"),
vllm_runner_kwargs={
"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]},
},
marks=[pytest.mark.core_model],
),
"gemma4": VitCudagraphTestConfig(
model="google/gemma-4-E2B-it",
image_prompt=(
"<bos><start_of_turn>user\n<|image|>\nWhat is in this image?<end_of_turn>\n"
"<start_of_turn>model\n"
),
video_prompt=(
"<bos><start_of_turn>user\n<|video|>\nDescribe this video in one sentence."
"<end_of_turn>\n<start_of_turn>model\n"
),
# The 16-frame test video produces 1056 vision tokens. Capture only
# the smallest supported bucket that covers it instead of all default
# buckets through max_model_len, which adds unrelated memory pressure.
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [1120],
},
needs_video_metadata=True,
marks=[pytest.mark.core_model],
),
"minicpmv_25": VitCudagraphTestConfig(
model="openbmb/MiniCPM-Llama3-V-2_5",
modalities=["image"],
image_prompt=minicpmv_25_chat_template(
"(<image>./</image>)\nWhat is in this image?"
),
# CI runs on 35GB MIG slices: every budget captures one graph per
# patch-grid bucket, so the default budgets through max_model_len
# OOM there. A small budget set covers the test images (each item
# is at most (max_slice_num + 1) * query_num = 640 tokens).
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [64, 1024],
},
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"minicpmv_26": VitCudagraphTestConfig(
model="openbmb/MiniCPM-V-2_6",
image_prompt=minicpmv_chat_template(
"(<image>./</image>)\nWhat is in this image?"
),
video_prompt=minicpmv_chat_template(
"(<video>./</video>)\nDescribe this video in one sentence."
),
max_model_len=2048,
# Fewer frames keep the video item within the 1024 budget.
num_video_frames=4,
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [64, 1024],
"encoder_cudagraph_max_frames_per_batch": 4,
},
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
"minicpmv_40": VitCudagraphTestConfig(
model="openbmb/MiniCPM-V-4",
image_prompt=minicpmv_chat_template(
"(<image>./</image>)\nWhat is in this image?"
),
video_prompt=minicpmv_chat_template(
"(<video>./</video>)\nDescribe this video in one sentence."
),
max_model_len=2048,
num_video_frames=4,
compilation_config_overrides={
"encoder_cudagraph_token_budgets": [64, 1024],
"encoder_cudagraph_max_frames_per_batch": 4,
},
vllm_runner_kwargs={"trust_remote_code": True},
marks=[pytest.mark.core_model],
),
}
def get_compilation_config(config: VitCudagraphTestConfig):
return {
"cudagraph_mm_encoder": True,
"encoder_cudagraph_max_vision_items_per_batch": 1,
"encoder_cudagraph_max_frames_per_batch": 16,
**config.compilation_config_overrides,
}
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
)
def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
config = MODEL_CONFIGS[model_id]
if config.skip:
pytest.skip(f"{model_id} is marked to be skipped.")
if "image" not in config.modalities:
pytest.skip(f"{model_id} does not support the image modality.")
image_prompts = IMAGE_ASSETS.prompts(
{
"stop_sign": config.image_prompt, # type: ignore[typeddict-item]
"cherry_blossom": config.image_prompt, # type: ignore[typeddict-item]
}
)
images = [[asset.pil_image] for asset in image_assets]
with vllm_runner(
config.model,
dtype=config.dtype,
max_model_len=config.max_model_len,
max_num_seqs=config.max_num_seqs,
limit_mm_per_prompt={"image": 1},
compilation_config=get_compilation_config(config),
**config.vllm_runner_kwargs,
) as vllm_model:
outputs = vllm_model.generate_greedy(
image_prompts, config.max_tokens, images=images
)
# Basic validation that we got a response
assert len(outputs) == 2
output_ids, output_text = outputs[0]
# Ensure we got some output
assert len(output_ids) > 0
assert len(output_text) > 0
# Ensure the output is a string
assert isinstance(output_text, str)
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
)
def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
config = MODEL_CONFIGS[model_id]
if config.skip:
pytest.skip(f"{model_id} is marked to be skipped.")
if "video" not in config.modalities:
pytest.skip(f"{model_id} does not support the video modality")
video_prompts = VIDEO_ASSETS.prompts(
{
"baby_reading": config.video_prompt, # type: ignore[typeddict-item]
}
)
if config.needs_video_metadata:
sampled_vids = [
sample_frames_with_video_metadata(
(asset.np_ndarrays, asset.metadata), config.num_video_frames
)
for asset in video_assets
]
else:
sampled_vids = [
sample_frames_from_video(asset.np_ndarrays, config.num_video_frames)
for asset in video_assets
]
videos = [sampled_vids[0]]
with vllm_runner(
config.model,
dtype=config.dtype,
max_model_len=config.max_model_len,
max_num_seqs=config.max_num_seqs,
limit_mm_per_prompt={"video": 1},
compilation_config=get_compilation_config(config),
**config.vllm_runner_kwargs,
) as vllm_model:
outputs = vllm_model.generate_greedy(
video_prompts, config.max_tokens, videos=videos
)
# Basic validation that we got a response
assert len(outputs) == 1
output_ids, output_text = outputs[0]
# Ensure we got some output
assert len(output_ids) > 0
assert len(output_text) > 0
# Ensure the output is a string
assert isinstance(output_text, str)
def _check_eonly_encoder_outputs(worker, batches):
"""Compare real E-only replay with eager, retaining outputs across replay."""
import torch
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
runner = worker.model_runner
assert isinstance(runner, MMEncoderModelRunner)
manager = runner.model_state.encoder_runner.cudagraph_manager
assert manager is not None and manager.is_captured()
retained: list[tuple[torch.Tensor, torch.Tensor]] = []
max_error = 0.0
stats = []
with torch.inference_mode():
for batch in batches:
kwargs = {key: value.to(runner.device) for key, value in batch.items()}
actual = manager.execute(kwargs)
expected = runner.model.embed_multimodal(**kwargs)
assert len(actual) == len(expected)
for output, reference in zip(actual, expected):
assert torch.isfinite(output).all()
# Two BF16 ulps at unit scale, fixed before model validation.
torch.testing.assert_close(output, reference, rtol=0.016, atol=0.016)
max_error = max(max_error, (output - reference).abs().max().item())
for output, snapshot in retained:
torch.testing.assert_close(output, snapshot, rtol=0, atol=0)
retained.extend((output, output.clone()) for output in actual)
stats.append(manager.get_cumulative_stats())
assert stats[0]["graph_hits"] > 0
assert stats[1]["graph_hits"] > stats[0]["graph_hits"]
assert stats[2]["graph_misses"] > stats[1]["graph_misses"]
assert stats[3]["graph_hits"] > stats[2]["graph_hits"]
assert stats[4]["graph_hits"] > stats[3]["graph_hits"]
assert stats[5]["graph_hits"] > stats[4]["graph_hits"]
assert stats[6]["graph_hits"] > stats[5]["graph_hits"]
return {"max_abs_error": max_error, "stats": stats}
@pytest.mark.parametrize(
"model_id",
params_with_marks({key: MODEL_CONFIGS[key] for key in ("qwen3_vl", "qwen3_5")}),
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA graphs")
def test_eonly_vit_cudagraph_outputs(
model_id, vllm_runner, image_assets, video_assets, monkeypatch
):
"""Exercise startup capture, image/video replay, fallback and output ownership."""
import numpy as np
from PIL import Image
from transformers import AutoProcessor
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
config = MODEL_CONFIGS[model_id]
token_budgets = [64, 256, 2048]
processor = AutoProcessor.from_pretrained(config.model)
first, second = [asset.pil_image for asset in image_assets]
batches = [
[first.resize((224, 224))],
[second.resize((448, 224)), first.resize((224, 224))],
[second.resize((1792, 1792))],
[second.resize((224, 224))],
[first.resize((1280, 720))],
[first.resize((224, 224))],
]
frames = sample_frames_from_video(video_assets[0].np_ndarrays, 2)
video = np.stack(
[np.asarray(Image.fromarray(frame).resize((224, 224))) for frame in frames]
)
with vllm_runner(
config.model,
dtype=config.dtype,
mm_encoder_only=True,
# FA padding NaNs are tracked separately in #57136.
mm_encoder_attn_backend="FLASHINFER",
enable_prefix_caching=False,
max_model_len=4096,
max_num_seqs=2,
limit_mm_per_prompt={"image": 2, "video": 1},
compilation_config={
"cudagraph_mode": "NONE",
"cudagraph_mm_encoder": True,
"encoder_cudagraph_token_budgets": token_budgets,
"encoder_cudagraph_max_vision_items_per_batch": 2,
"encoder_cudagraph_max_frames_per_batch": 2,
},
) as model:
mm_config = model.llm.llm_engine.vllm_config.model_config.multimodal_config
processor_kwargs = mm_config.merge_mm_processor_kwargs({})
inputs = [
dict(
processor.image_processor(
images=images, return_tensors="pt", **processor_kwargs
)
)
for images in batches
]
inputs.append(
dict(
processor.video_processor(
videos=[video],
do_sample_frames=False,
return_tensors="pt",
**processor_kwargs,
)
)
)
results = model.llm.collective_rpc(
partial(_check_eonly_encoder_outputs, batches=inputs)
)
for result in results:
assert result["stats"][0]["token_budgets"] == token_budgets
print(f"E-only {model_id}: {results}")
@@ -1,207 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Core test implementation to be shared across modalities."""
from collections.abc import Callable
from typing import Any
import torch
from transformers.models.auto.auto_factory import _BaseAutoModelClass
from vllm.config.model import RunnerOption
from vllm.tokenizers import TokenizerLike
from .....conftest import HfRunner, VllmRunner
from ....registry import HF_EXAMPLE_MODELS
from .types import PromptWithMultiModalInput, RunnerOutput
def run_test(
*,
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
inputs: list[PromptWithMultiModalInput],
model: str,
dtype: str,
max_tokens: int,
num_logprobs: int,
enforce_eager: bool,
max_model_len: int,
max_num_seqs: int,
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
auto_cls: type[_BaseAutoModelClass],
use_tokenizer_eos: bool,
comparator: Callable[..., None],
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None,
stop_str: list[str] | None,
limit_mm_per_prompt: dict[str, int],
vllm_runner_kwargs: dict[str, Any] | None,
hf_model_kwargs: dict[str, Any] | None,
hf_processor: Callable[[str], Any] | None,
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
runner: RunnerOption = "auto",
distributed_executor_backend: str | None = None,
tensor_parallel_size: int = 1,
vllm_embeddings: torch.Tensor | None = None,
):
"""Modality agnostic test executor for comparing HF/vLLM outputs."""
# In the case of embeddings, vLLM takes separate input tensors
vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(on_fail="skip")
# Disable other modalities to save memory
default_limits = {"image": 0, "video": 0, "audio": 0}
limit_mm_per_prompt = default_limits | limit_mm_per_prompt
vllm_outputs_per_mm = []
hf_outputs_per_mm = []
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
vllm_runner_kwargs_: dict[str, Any] = {"mm_processor_cache_gb": 0}
if model_info.tokenizer:
vllm_runner_kwargs_["tokenizer_name"] = model_info.tokenizer
if model_info.tokenizer_mode:
vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
if model_info.hf_overrides:
vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
if model_info.require_embed_inputs:
for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
vllm_runner_kwargs_[k] = model_info.require_embed_inputs
if not model_info.enable_prefix_caching:
vllm_runner_kwargs_["enable_prefix_caching"] = False
if vllm_runner_kwargs:
vllm_runner_kwargs_.update(vllm_runner_kwargs)
# Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
# already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
if "limit_mm_per_prompt" in vllm_runner_kwargs_:
limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
with vllm_runner(
model,
max_model_len=max_model_len,
max_num_seqs=max_num_seqs,
dtype=dtype,
limit_mm_per_prompt=limit_mm_per_prompt,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=enforce_eager,
runner=runner,
**vllm_runner_kwargs_,
) as vllm_model:
tokenizer = vllm_model.llm.get_tokenizer()
vllm_kwargs: dict[str, Any] = {}
if get_stop_token_ids is not None:
vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
if stop_str:
vllm_kwargs["stop"] = stop_str
for prompts, image_data, video_data, audio_data in vllm_inputs:
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
vllm_kwargs_with_mm_data = vllm_kwargs | mm_data
vllm_output = vllm_model.generate_greedy_logprobs(
prompts,
max_tokens,
num_logprobs=num_logprobs,
**vllm_kwargs_with_mm_data,
)
vllm_outputs_per_mm.append(vllm_output)
hf_runner_kwargs: dict[str, Any] = {}
if model_info.tokenizer:
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
if hf_processor is not None:
hf_runner_kwargs["processor"] = hf_processor(model)
hf_model = hf_runner(
model,
dtype=dtype,
auto_cls=auto_cls,
model_kwargs=hf_model_kwargs,
**hf_runner_kwargs,
)
# Some models need to patch things like the model processor, e.g., internvl
if patch_hf_runner is not None:
hf_model = patch_hf_runner(hf_model)
with hf_model, torch.no_grad():
tokenizer = hf_model.tokenizer
# Some models need to explicitly pass the eos_token_id off the tokenizer
# or processor for a good comparison;
# currently assume processor/tokenizer agree on the EOS, and pull it off
# the tokenizer if requested.
hf_kwargs = {}
if use_tokenizer_eos:
hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
if stop_str:
hf_kwargs["stop_strings"] = stop_str
for prompts, image_data, video_data, audio_data in inputs:
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
hf_kwargs_with_mm_data = hf_kwargs | mm_data
hf_output = hf_model.generate_greedy_logprobs_limit(
prompts,
max_tokens,
num_logprobs=num_logprobs,
tokenizer=tokenizer,
**hf_kwargs_with_mm_data,
)
hf_outputs_per_mm.append(hf_output)
# Apply output processing / sanitation to the vLLM and HF runner results
hf_outputs_per_mm, vllm_outputs_per_mm = process_runner_outputs(
model,
first_runner_outputs=hf_outputs_per_mm,
second_runner_outputs=vllm_outputs_per_mm,
first_runner_processor=hf_output_post_proc,
second_runner_processor=vllm_output_post_proc,
)
for hf_outputs, vllm_outputs in zip(hf_outputs_per_mm, vllm_outputs_per_mm):
# This is usually check_logprobs_close, but it's passed through to
# allow things like check_outputs_equal where needed
comparator(
outputs_0_lst=hf_outputs,
outputs_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
def process_runner_outputs(
model,
first_runner_outputs,
second_runner_outputs,
first_runner_processor=None,
second_runner_processor=None,
):
"""Applies the runner processor(s) to the runner outputs, if any."""
if first_runner_processor is not None:
first_runner_outputs = process_outputs(
first_runner_processor, model, first_runner_outputs
)
if second_runner_processor is not None:
second_runner_outputs = process_outputs(
second_runner_processor, model, second_runner_outputs
)
return first_runner_outputs, second_runner_outputs
def process_outputs(output_processor, model, outputs_per_image):
"""Applies a model specific post-processor function to a runner's output."""
return [
[output_processor(res, model) for res in outputs]
for outputs in outputs_per_image
]
@@ -1,218 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Types for writing multimodal model tests."""
from collections.abc import Callable, Iterable
from enum import Enum
from pathlib import PosixPath
from typing import Any, NamedTuple
import torch
from pytest import MarkDecorator
from transformers import AutoModelForCausalLM
from transformers.models.auto.auto_factory import _BaseAutoModelClass
from vllm.config.model import RunnerOption
from vllm.logprobs import SampleLogprobs
from vllm.tokenizers import TokenizerLike
from .....conftest import (
AUDIO_ASSETS,
IMAGE_ASSETS,
HfRunner,
ImageAsset,
ImageTestAssets,
PromptAudioInput,
PromptImageInput,
PromptVideoInput,
)
from ....utils import check_logprobs_close
# meta image tag; will be replaced by the appropriate tag for the model
TEST_IMG_PLACEHOLDER = "<vlm_image>"
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
}
)
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
{
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
}
)
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
IMAGE_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
EMBEDDING_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0)]
RunnerOutput = tuple[list[int], str, SampleLogprobs | None]
class PromptWithMultiModalInput(NamedTuple):
"""Holds the multimodal input for a single test case."""
prompts: list[str]
image_data: PromptImageInput | None = None
video_data: PromptVideoInput | None = None
audio_data: PromptAudioInput | None = None
class VLMTestType(Enum):
IMAGE = 1
MULTI_IMAGE = 2
EMBEDDING = 3
VIDEO = 4
AUDIO = 5
CUSTOM_INPUTS = 6
class SizeType(Enum):
SIZE_FACTOR = 1
FIXED_SIZE = 2
class CustomTestOptions(NamedTuple):
inputs: list[PromptWithMultiModalInput]
limit_mm_per_prompt: dict[str, int]
class ImageSizeWrapper(NamedTuple):
type: SizeType
# A size factor is a wrapper of 0+ floats,
# while a fixed size contains an iterable of integer pairs
data: Iterable[float] | Iterable[tuple[int, int]]
class VLMTestInfo(NamedTuple):
"""Holds the configuration for 1+ tests for one model architecture."""
models: list[str]
test_type: VLMTestType | Iterable[VLMTestType]
# Should be None only if this is a CUSTOM_INPUTS test
prompt_formatter: Callable[[str], str] | None = None
img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
audio_idx_to_prompt: Callable[[int], str] = lambda idx: "<audio>\n"
# Most models work on the single / multi-image prompts above, but in some
# cases the log prob check fails, e.g., for paligemma. We allow passing
# an override for the single image prompts / multi-image prompt for this
# reason.
single_image_prompts: Iterable[str] = SINGLE_IMAGE_BASE_PROMPTS
multi_image_prompt: str = MULTI_IMAGE_BASE_PROMPT
# Function for converting ImageAssets to image embeddings;
# We need to define this explicitly for embedding tests
convert_assets_to_embeddings: (
Callable[[ImageTestAssets], list[torch.Tensor]] | None
) = None
# Exposed options for vLLM runner; we change these in a several tests,
# but the defaults are derived from VllmRunner & the engine defaults
# These settings are chosen to avoid OOMs when running in the CI
enforce_eager: bool = True
max_model_len: int = 1024
max_num_seqs: int = 256
runner: RunnerOption = "auto"
tensor_parallel_size: int = 1
vllm_runner_kwargs: dict[str, Any] | None = None
# Optional callable which gets a list of token IDs from the model tokenizer
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None
# Optional list of strings to stop generation, useful when stop tokens are
# not special tokens in the tokenizer
stop_str: list[str] | None = None
# Exposed options for HF runner
hf_model_kwargs: dict[str, Any] | None = None
hf_processor: Callable[[str], Any] | None = None
# Indicates we should explicitly pass the EOS from the tokenizer
use_tokenizer_eos: bool = False
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
patch_hf_runner: Callable[[HfRunner], HfRunner] | None = None
# Post processors that if defined, will run oun the outputs of the
# vLLM and HF runner, respectively (useful for sanitization, etc).
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
# Consumes the output of the callables above and checks if they're equal
comparator: Callable[..., None] = check_logprobs_close
# Default expandable params per test; these defaults can be overridden in
# instances of this object; the complete set of test cases for the model
# is all combinations of .models + all fields below
max_tokens: int = 128
num_logprobs: int = 5
dtype: str = "auto"
distributed_executor_backend: str | None = None
# Only expanded in video tests
num_video_frames: int | tuple[int] = 16
needs_video_metadata: bool = False
# Fixed image sizes / image size factors; most tests use image_size_factors.
# Each inner iterable defines one request batch. All batches are run against
# the same model instance.
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
# Hack for updating a prompt to take into a local path; currently only used
# for Qwen-VL, which requires encoding the image path / url into the prompt
# for HF runner
prompt_path_encoder: (
Callable[[PosixPath, str, list[ImageAsset] | ImageTestAssets], str] | None
) = None # noqa: E501
# Allows configuring a test to run with custom inputs
custom_test_opts: list[CustomTestOptions] | None = None
marks: list[MarkDecorator] | None = None
def get_non_parametrized_runner_kwargs(self):
"""Returns a dictionary of expandable kwargs for items that are used
in all test types, which are NOT used when creating the parametrized
test cases.
"""
return {
"enforce_eager": self.enforce_eager,
"max_model_len": self.max_model_len,
"max_num_seqs": self.max_num_seqs,
"runner": self.runner,
"tensor_parallel_size": self.tensor_parallel_size,
"vllm_runner_kwargs": self.vllm_runner_kwargs,
"hf_output_post_proc": self.hf_output_post_proc,
"vllm_output_post_proc": self.vllm_output_post_proc,
"auto_cls": self.auto_cls,
"use_tokenizer_eos": self.use_tokenizer_eos,
"comparator": self.comparator,
"get_stop_token_ids": self.get_stop_token_ids,
"hf_model_kwargs": self.hf_model_kwargs,
"hf_processor": self.hf_processor,
"stop_str": self.stop_str,
"patch_hf_runner": self.patch_hf_runner,
}
class ExpandableVLMTestArgs(NamedTuple):
"""The expanded kwargs which correspond to a single test case."""
model: str
max_tokens: int
num_logprobs: int
dtype: str
distributed_executor_backend: str | None
# Sizes are used for everything except audio and custom input tests.
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
# Video only
num_video_frames: int | None = None
needs_video_metadata: bool = False
# Custom inputs only
custom_test_opts: CustomTestOptions | None = None
@@ -1,558 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from collections.abc import Set as AbstractSet
from functools import partial
import numpy as np
import pytest
import torch
from PIL import Image
from tests.models.utils import build_model_context
from vllm.config import ModelConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
)
from vllm.inputs import MultiModalDataDict, MultiModalInput
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.platforms import current_platform
from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
from vllm.utils.mistral import is_mistral_tokenizer
from ....multimodal.utils import random_audio, random_image, random_video
from ...registry import (
_MULTIMODAL_EXAMPLE_MODELS,
_TRANSFORMERS_BACKEND_MODELS,
HF_EXAMPLE_MODELS,
)
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""Add metadata to video mm_data."""
def create_metadata(frames: np.ndarray):
num_frames = len(frames)
return {
"total_num_frames": num_frames,
"fps": 2.0,
"duration": num_frames / 2.0,
"video_backend": "opencv",
"frames_indices": list(range(num_frames)),
"do_sample_frames": True,
}
# Ensure video metadata is included
if "video" in mm_data:
video = mm_data["video"]
if isinstance(video, list):
# multiple videos
mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
else:
# single video
mm_data["video"] = (video, create_metadata(video))
return mm_data
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
"""Patch the multimodal data for GLM-ASR model.
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
"""
if "audio" in mm_data:
audio = mm_data["audio"]
if isinstance(audio, list) and len(audio) > 1:
# Limit to single audio to match text requirement
mm_data["audio"] = [audio[0]]
return mm_data
_IGNORE_MM_KEYS = {
# Dithering causes minor divergence
"cohere_asr": {"input_features"},
# In Ultravox, the audio_features can be different depending on padding
# The slight difference should not be a problem though, since
# attention_mask lets us ignore the difference.
"ultravox": {"audio_features"},
}
MM_DATA_PATCHES = {
"glmasr": glmasr_patch_mm_data,
}
_XPU_EXCLUDED_MODEL_IDS = {
"baidu/Unlimited-OCR",
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
"moonshotai/Kimi-K3",
"Qwen/Qwen2.5-Omni-7B-AWQ",
"thinkingmachines/Inkling-NVFP4",
}
_CPU_EXCLUDED_MODEL_IDS = {
# DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now.
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
}
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
for model_arch in model_arch_list:
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
yield model_info.default
for extra_type, extra_model_id in model_info.extras.items():
if "fp" in extra_type:
continue # Redundant to test quantized models
yield extra_model_id
def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
model_ids = list(_iter_model_ids_to_test(model_arch_list))
if current_platform.is_xpu():
for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS:
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
if current_platform.is_cpu():
for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS:
while excluded_model_id in model_ids:
model_ids.remove(excluded_model_id)
return model_ids
def get_model_ids_to_test():
transformers_arch_ids = {
model_id
for info in _TRANSFORMERS_BACKEND_MODELS.values()
for model_id in (info.default, *info.extras.values())
}
vllm_only_archs = {
arch
for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
if not any(
model_id in transformers_arch_ids
for model_id in (info.default, *info.extras.values())
)
}
return _get_model_ids_to_test(vllm_only_archs)
def get_transformers_backend_model_ids_to_test():
return sorted(
{
model_id
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
if "MultiModal" in arch
for model_id in (info.default, *info.extras.values())
}
)
def get_token_prompt(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
) -> list[int]:
tokenizer: TokenizerLike = processor.info.get_tokenizer()
model_config = processor.info.ctx.model_config
if processor.info.data_parser.video_needs_metadata:
mm_data = add_video_metadata(mm_data)
model_type = model_config.hf_config.model_type
if model_type in MM_DATA_PATCHES:
mm_data = MM_DATA_PATCHES[model_type](mm_data)
parsed_data = processor.info.parse_mm_data(mm_data)
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
if is_mistral_tokenizer(tokenizer):
inputs = processor.get_dummy_inputs(
model_config.max_model_len,
mm_counts,
mm_options=MultiModalDummyOptions(),
# Assume all Mistral models define this extra argument
mm_data=mm_data, # type: ignore[call-arg]
)
else:
inputs = processor.get_dummy_inputs(
model_config.max_model_len,
mm_counts,
mm_options=MultiModalDummyOptions(),
)
if not isinstance(inputs.prompt, list):
raise TypeError(type(inputs.prompt))
return inputs.prompt
@pytest.mark.parametrize(
"model_id,durations",
[
("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]),
("nvidia/audio-flamingo-3-hf", [31.0]),
("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]),
("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]),
("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]),
],
)
def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
"""EC metadata preserves the raw-audio prompt without running the HF processor."""
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)})
mm_config = ctx.model_config.multimodal_config
mm_config.enable_mm_embeds = True
mm_config.allow_missing_mm_embeddings = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
sample_rate = processor.info.get_feature_extractor().sampling_rate
audios = [
np.zeros(int(duration * sample_rate), dtype=np.float32)
for duration in durations
]
prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)})
raw = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
features = [
MultiModalFeatureSpec(
data=item,
modality="audio",
identifier=str(index),
mm_position=position,
)
for index, (item, position) in enumerate(
zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"])
)
]
published = collect_ec_item_metadata(
features, PlaceholderMetadataResolver(ctx.model_config)
)
metadata = json.loads(
json.dumps([entry["metadata"] for entry in published.values()])
)
assert len(metadata) == len(audios)
assert all(metadata)
audio_data = _get_embeds_data("audio", metadata, processor)
def unexpected_preprocessing(*args, **kwargs):
pytest.fail("The EC consumer must not preprocess audio")
monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing)
remote = processor(
prompt,
mm_items=processor.info.parse_mm_data({"audio": audio_data}),
hf_processor_mm_kwargs={},
)
assert remote["prompt_token_ids"] == raw["prompt_token_ids"]
assert remote["mm_placeholders"] == raw["mm_placeholders"]
# Omni also consumes these lengths when constructing M-RoPE positions.
if "audio_feature_lengths" in audio_data:
assert torch.equal(
remote["mm_kwargs"].get_data()["audio_feature_lengths"],
raw["mm_kwargs"].get_data()["audio_feature_lengths"],
)
mm_config.allow_missing_mm_embeddings = False
ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
with pytest.raises(ValueError, match="should contain the fields"):
ordinary_processor.info.parse_mm_data({"audio": audio_data})
def random_vision_chunk(
rng: np.random.RandomState,
min_wh: int,
max_wh: int,
min_frames: int,
max_frames: int,
) -> dict:
num_frames = rng.randint(min_frames, max_frames + 1)
if num_frames == 1:
# Single image chunk
wh = rng.randint(min_wh, max_wh + 1)
image = random_image(rng, wh, wh + 1)
return {"type": "image", "image": image}
frames = []
for _ in range(num_frames):
wh = rng.randint(min_wh, max_wh + 1)
frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
frames.append(frame)
video_array = np.stack(frames, axis=0)
return {"type": "video_chunk", "video_chunk": video_array}
def _test_processing_correctness(
model_id_or_arch: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
model_impl: str = "auto",
):
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
# Use model architecture to get the default model id
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
model_id = model_info.default
else:
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
model_id = model_id_or_arch
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_config = ModelConfig(
model_id,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
hf_overrides=model_info.hf_overrides,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=model_info.dtype,
model_impl=model_impl,
)
# Ensure that the cache can fit all of the data
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
model_config.multimodal_config.mm_processor_cache_gb = 2048
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
factories = model_cls._processor_factory
ctx = InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
cache = MultiModalProcessorOnlyCache(model_config)
processing_info = factories.info(ctx)
supported_mm_limits = processing_info.get_supported_mm_limits()
# Keep integer limits for local data generation
limit_mm_per_prompt_ints = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
if modality == "video":
return VideoDummyOptions(count=count)
if modality == "image":
return ImageDummyOptions(count=count)
if modality == "audio":
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
# Assign normalized DummyOptions to the model config
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
{
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt_ints.items()
}
)
processor = factories.build_processor(ctx)
rng = np.random.RandomState(0)
# GLM-ASR requires a minimum audio length of 70ms
min_audio_len = 512 if model_config.hf_config.model_type != "glmasr" else 1120
input_to_hit = {
"image": Image.new("RGB", size=(128, 128)),
"video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
"audio": (np.zeros((min_audio_len,)), 16000),
"vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
}
input_factory = {
"image": partial(random_image, rng, min_wh=128, max_wh=256),
"video": partial(
random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
),
"audio": partial(
random_audio,
rng,
min_len=min_audio_len,
max_len=min_audio_len + 512,
sr=16000,
),
"vision_chunk": partial(
random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
),
}
for batch_idx in range(num_batches):
mm_data = {
k: [
(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
for _ in range(rng.randint(limit + 1))
]
for k, limit in limit_mm_per_prompt_ints.items()
}
# Drop unnecessary keys and test single -> multi conversion
if rng.rand() < simplify_rate:
for k in list(mm_data.keys()):
if not mm_data[k]:
del mm_data[k]
elif len(mm_data[k]) == 1:
mm_data[k] = mm_data[k][0]
_test_processing_correctness_one(
model_config,
mm_data,
processor,
batch_idx,
hit_rate,
num_batches,
simplify_rate,
cache=cache,
)
def _test_processing_correctness_one(
model_config: ModelConfig,
mm_data: MultiModalDataDict,
processor: BaseMultiModalProcessor,
batch_idx: int,
hit_rate: float,
num_batches: int,
simplify_rate: float,
cache: MultiModalProcessorOnlyCache,
):
model_type = model_config.hf_config.model_type
token_prompt = get_token_prompt(processor, mm_data)
mm_items = processor.info.parse_mm_data(mm_data)
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
baseline_tokenized_result = processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
)
cached_tokenized_result = processor(
token_prompt,
mm_items=mm_items,
hf_processor_mm_kwargs={},
cache=cache,
)
_assert_inputs_equal(
baseline_tokenized_result,
cached_tokenized_result,
ignore_mm_keys=ignore_mm_keys,
msg=(
f"Failed ({batch_idx=}, {hit_rate=}, "
f"{num_batches=}, {simplify_rate=}, "
f"{token_prompt=}, {mm_data=})"
),
)
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
if model_id == "openvla/openvla-7b":
pytest.skip(
"OpenVLA uses a custom vLLM processor because its HF remote "
"processor is incompatible with current Transformers."
)
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
pytest.skip(
"Voxtral Realtime doesn't make use of any place-holder "
"tokens and hence cannot pass the processing "
"correctness test as is. Let's revisit adapting this "
"test once more realtime models exist."
)
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
pytest.skip(
"MOSS-Audio uses a custom processor that dynamically expands "
"audio placeholders from processed audio lengths. Its vLLM "
"processor paths are covered by test_moss_audio.py."
)
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
pytest.skip(
"LLaVA-OneVision-2 video processing routes frames through custom "
"video backends (qwen_vl_utils / codec) that require real encoded "
"video bytes and metadata. The synthetic numpy-array videos used by "
"this test yield empty video features, so the generic correctness "
"check cannot exercise the video path. Image processing is covered "
"by registration/inference tests."
)
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
)
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
@pytest.mark.parametrize("num_batches", [32])
@pytest.mark.parametrize("simplify_rate", [1.0])
def test_processing_correctness_transformers(
model_id: str,
hit_rate: float,
num_batches: int,
simplify_rate: float,
):
_test_processing_correctness(
model_id,
hit_rate=hit_rate,
num_batches=num_batches,
simplify_rate=simplify_rate,
model_impl="transformers",
)
def _assert_inputs_equal(
a: MultiModalInput,
b: MultiModalInput,
*,
ignore_mm_keys: set[str] | None = None,
msg: str = "",
):
if ignore_mm_keys is None:
ignore_mm_keys = set()
ignore_prompt_keys = ("prompt", "mm_kwargs")
a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}
b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}
assert a_rest == b_rest, msg
a_data = a["mm_kwargs"].get_data()
b_data = b["mm_kwargs"].get_data()
for key in ignore_mm_keys:
a_data.pop(key, None)
b_data.pop(key, None)
assert batched_tensors_equal(a_data, b_data), msg
@@ -1,241 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Video placeholder accounting for GLM-5.3-Flash.
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
placeholders per timestamp, so the timestamps returned by
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
while ``video_grid_thw`` decides how many rows the vision tower produces. If
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
inside a worker and takes the engine down with it.
The checks below are arithmetic: no weights, no GPU.
"""
import pytest
import torch
from PIL import Image
from transformers.models.glm5_next.video_processing_glm5_next import (
Glm5NextVideoProcessor,
smart_resize,
)
from transformers.video_utils import VideoMetadata
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ...utils import build_model_context
@pytest.fixture(scope="module")
def processor():
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"video": 1},
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
)
def _pixel_path_grid(
video_processor: Glm5NextVideoProcessor,
num_frames: int,
height: int,
width: int,
) -> tuple[int, int, int]:
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
resized_height, resized_width = smart_resize(
num_frames=num_frames,
height=height,
width=width,
temporal_factor=video_processor.temporal_patch_size,
factor=video_processor.patch_size
* video_processor.merge_size
* video_processor.patch_expand_factor,
min_pixels=video_processor.min_image_tokens,
max_pixels=video_processor.max_image_tokens,
)
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
return (
padded_frames // video_processor.temporal_patch_size,
resized_height // video_processor.patch_size,
resized_width // video_processor.patch_size,
)
@pytest.mark.parametrize(
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
[
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
# 1080p, 20 s: same factor of 3 at a full-size canvas.
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
# 1080p, 60 s: the one duration window where the two samplers agree
# anyway -- a regression guard, the count must not move.
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
],
)
def test_video_placeholders_match_encoder_rows(
processor,
total_num_frames: int,
fps: float,
duration: float,
height: int,
width: int,
expected_grid: tuple[int, int, int],
):
info = processor.info
video_processor = info.get_video_processor()
frame_indices = video_processor.sample_frames(
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
)
grid_t, grid_h, grid_w = _pixel_path_grid(
video_processor, len(frame_indices), height, width
)
assert (grid_t, grid_h, grid_w) == expected_grid
timestamps = info._get_video_second_idx_glm46v(
{
"total_num_frames": total_num_frames,
"fps": fps,
"duration": duration,
"do_sample_frames": True,
},
total_num_frames,
)
merge_length = video_processor.merge_size**2
tokens_per_frame = grid_h * grid_w // merge_length
encoder_rows = grid_t * grid_h * grid_w // merge_length
assert len(timestamps) == grid_t
assert len(timestamps) * tokens_per_frame == encoder_rows
assert timestamps == sorted(timestamps)
assert timestamps[0] == 0
assert timestamps[-1] <= duration
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
"""The loader may pre-sample and hand the frames over as they are."""
info = processor.info
video_processor = info.get_video_processor()
num_frames = 32
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
timestamps = info._get_video_second_idx_glm46v(
{
"total_num_frames": 256,
"fps": 8.0,
"duration": 32.0,
"do_sample_frames": False,
"frames_indices": list(range(0, 256, 256 // num_frames)),
},
num_frames,
)
assert len(timestamps) == grid_t
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
"""A clip the sampler cannot pick a single frame from is a bad request."""
with pytest.raises(ValueError, match="selected no frames"):
processor.info._get_video_second_idx_glm46v(
{
"total_num_frames": 2,
"fps": 8.0,
"duration": 0.25,
"do_sample_frames": True,
},
2,
)
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 2},
)
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)
def _image_info(**kwargs):
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 1},
**kwargs,
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
).info
def test_image_encoder_cache_covers_full_token_budget():
"""The most-features probe must reach the processor's token ceiling.
The inherited square probe refits to 89x89 = 7921 tokens under the
max_image_tokens=8000 budget, so the encoder cache came up short and
ordinary non-square images in 7922-8000 tokens were refused with
HTTP 400 (#59539).
"""
info = _image_info()
assert info.get_max_image_tokens() == 8000
# The shapes from the issue, with the token counts the processor
# actually produces; every one must fit the cache.
for width, height, expected_tokens in [
(4032, 3024, 7931), # phone photo, 4:3
(3840, 2160, 7973), # 4K frame, 16:9
(3508, 2480, 7950), # A4 at 300 dpi
(2600, 2400, 7998), # 13:12
(3000, 3000, 7921), # square worst case before the fix
]:
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
assert num_tokens == expected_tokens
assert num_tokens <= info.get_max_image_tokens()
# The profiling dummy covers the real worst case, not the square one.
size = info.get_image_size_with_most_features()
assert size.width * size.height == 2240 * 2800
def test_image_encoder_cache_follows_max_pixels_override():
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
assert info.get_max_image_tokens() == 100
size = info.get_image_size_with_most_features()
assert (
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
== 100
)
@@ -1,81 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
import pytest
import torch
from PIL import Image
from transformers import AutoImageProcessor
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
KimiK25FusedVisionProcessor,
)
from ...utils import build_model_context
_MODEL_ID = "moonshotai/Kimi-K3"
@pytest.fixture(scope="module")
def image_processors():
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
k25_fused = processor.info.image_processor
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
return k25_fused, k3_hf
def _noise_image(width: int, height: int) -> Image.Image:
rng = np.random.default_rng(0)
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
@pytest.mark.parametrize(
("width", "height"),
[(310, 470), (7300, 30), (13, 9)],
)
def test_fused_image_processor_matches_hf(image_processors, width: int, height: int):
fused, hf = image_processors
image = _noise_image(width, height)
expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt")
actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt")
torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"])
torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"])
assert fused.media_tokens_calculator(
{"type": "image", "image": image}
) == hf.media_tokens_calculator({"type": "image", "image": image})
@pytest.mark.usefixtures("default_vllm_config")
def test_mm_device_do_normalize():
device = current_platform.device_type
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2})
assert ctx.model_config.multimodal_config.mm_device_do_normalize
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
images = [
Image.new("RGB", (310, 470), color=(17, 89, 231)),
Image.new("RGB", (480, 320), color=(201, 13, 127)),
]
prompt = processor.info.get_hf_config().image_placeholder * len(images)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_inputs = processor(prompt, mm_items=mm_items)
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
raw_inputs = processor(prompt, mm_items=mm_items)
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
assert raw_values.dtype == torch.uint8
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
)
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Nemotron-Parse's device-side normalisation path."""
import numpy as np
import pytest
import torch
from PIL import Image
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.platforms import current_platform
from ...utils import build_model_context
_MODEL_ID = "nvidia/NVIDIA-Nemotron-Parse-2.0"
def _noise_image(width: int, height: int) -> Image.Image:
rng = np.random.default_rng(0)
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize(("width", "height"), [(1240, 1754), (310, 470)])
def test_mm_device_do_normalize(width: int, height: int):
device = current_platform.device_type
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
assert ctx.model_config.multimodal_config.mm_device_do_normalize
images = [_noise_image(width, height)]
prompt = [0]
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_items = processor.info.parse_mm_data({"image": images})
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
ctx.model_config.multimodal_config.mm_device_do_normalize = True
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
"pixel_values"
]
assert raw_values.dtype == torch.uint8
assert raw_values.shape == normalized_values.shape
input_norm = build_mm_input_norm(ctx.model_config).to(device)
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
torch.testing.assert_close(
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
)
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("mm_device_do_normalize", [False, True])
@pytest.mark.parametrize("layout", ["ndarray_hwc", "tensor_hwc", "tensor_chw"])
def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
ctx.model_config.multimodal_config.mm_device_do_normalize = mm_device_do_normalize
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
image = _noise_image(310, 470)
array = np.asarray(image)
item = {
"ndarray_hwc": array,
"tensor_hwc": torch.from_numpy(array),
"tensor_chw": torch.from_numpy(array).permute(2, 0, 1),
}[layout]
def pixel_values(img: object) -> torch.Tensor:
mm_items = processor.info.parse_mm_data({"image": [img]})
return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
@pytest.mark.usefixtures("default_vllm_config")
def test_explicit_kwargs_without_device_normalize():
ctx = build_model_context(
_MODEL_ID,
limit_mm_per_prompt={"image": 1},
mm_processor_kwargs={"do_rescale": False, "do_normalize": False},
)
ctx.model_config.multimodal_config.mm_device_do_normalize = False
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
# The remote processor ignores these kwargs and still normalises on the host.
assert values.is_floating_point()
@@ -1,272 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from collections.abc import Iterable
from contextlib import contextmanager
from functools import partial
from typing import Any, TypeAlias
import numpy as np
import pytest
import torch
import torch.nn as nn
from PIL import Image
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.config.cache import CacheConfig
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
ImageDummyOptions,
MultiModalDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.models.interfaces import supports_multimodal
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
from vllm.multimodal.utils import group_and_batch_mm_kwargs
from vllm.platforms import current_platform
from vllm.tokenizers import cached_tokenizer_from_config
from vllm.utils.collection_utils import is_list_of
from vllm.utils.torch_utils import set_default_torch_dtype
from ....utils import create_new_process_for_each_test
from ...registry import HF_EXAMPLE_MODELS
from ...utils import dummy_hf_overrides
from .test_common import get_model_ids_to_test, get_token_prompt
ImageInput = list[Image.Image]
VideoInput: TypeAlias = (
list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]]
)
AudioInput = list[tuple[np.ndarray, int]]
def _resize_data(
_data: Image.Image | np.ndarray, size_factor: float
) -> Image.Image | np.ndarray:
assert size_factor <= 1, "Size factor must be less than 1"
# Image input
if isinstance(_data, Image.Image):
W, H = _data.width, _data.height
W, H = map(lambda x: int(x * size_factor), (W, H))
return _data.resize((W, H))
# Video input with PIL Images
elif is_list_of(_data, Image.Image):
W, H = next(iter(_data)).width, next(iter(_data)).height
T = len(_data)
T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H))
return [d.resize((W, H)) for d in _data[:T]]
# Video input with numpy arrays
elif isinstance(_data, np.ndarray) and _data.ndim >= 4:
T, H, W, C = _data.shape[-4:]
T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W))
return _data[..., :T, :H, :W, :C]
# Audio input
elif isinstance(_data, np.ndarray) and _data.ndim == 1:
return _data[: int(len(_data) * size_factor)]
raise AssertionError("This line should be unreachable.")
def resize_mm_data(
data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...]
) -> ImageInput | VideoInput | AudioInput:
size_factors = size_factors[: len(data)]
if is_list_of(data, (Image.Image, np.ndarray, list)):
return [_resize_data(d, s) for d, s in zip(data, size_factors)]
elif is_list_of(data, tuple):
return [_resize_data(d, s) for (d, _), s in zip(data, size_factors)]
raise ValueError("Unsupported multimodal data type.")
def create_batched_mm_kwargs(
model_config: ModelConfig,
processor: BaseMultiModalProcessor,
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
processing_info = processor.info
supported_mm_limits = processing_info.get_supported_mm_limits()
mm_counts = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
processor_inputs = processor.get_dummy_inputs(
seq_len=model_config.max_model_len,
mm_counts=mm_counts,
mm_options=MultiModalDummyOptions(),
)
mm_items = processor_inputs.mm_data_items
resized_mm_data = {
modality: resize_mm_data(items.data, size_factors)
for modality, items in mm_items.items()
}
# video metadata will be added back to the resized video data here.
token_prompt = get_token_prompt(processor, resized_mm_data)
mm_kwargs = processor(
prompt=token_prompt,
mm_items=processor.info.parse_mm_data(resized_mm_data),
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
)["mm_kwargs"].require_data()
return group_and_batch_mm_kwargs(
[
(modality, item)
for modality in supported_mm_limits
for item in mm_kwargs[modality]
]
)
# TODO(Isotr0py): Don't initialize model during test
@contextmanager
def initialize_dummy_model(
model_cls: type[nn.Module],
model_config: ModelConfig,
):
temp_file = tempfile.mkstemp()[1]
current_device = torch.get_default_device()
vllm_config = VllmConfig(
model_config=model_config, cache_config=CacheConfig(block_size=16)
)
with set_current_vllm_config(vllm_config=vllm_config):
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend="nccl",
)
initialize_model_parallel(tensor_model_parallel_size=1)
with set_default_torch_dtype(model_config.dtype):
torch.set_default_device(current_platform.device_type)
model = model_cls(vllm_config=vllm_config)
torch.set_default_device(current_device)
yield model
del model
cleanup_dist_env_and_memory()
@create_new_process_for_each_test()
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
def test_model_tensor_schema(model_id: str):
if model_id == "moonshotai/Kimi-K2.5":
# FIXME(Isotr0py): Fix Kimi-K2.5's offline inference about vision chunks.
pytest.skip(
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
)
if (
model_id == "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
and not current_platform.is_cuda_alike()
):
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
if model_id == "zai-org/GLM-5.3-Flash" and (current_platform.is_xpu()):
pytest.skip("GLM-5.3-Flash is not supported on XPU")
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_arch = next(
arch for arch, info in HF_EXAMPLE_MODELS.hf_models.items() if info == model_info
)
hf_overrides_fn = partial(
dummy_hf_overrides,
model_arch=model_arch,
exist_overrides=model_info.hf_overrides,
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
)
# ROCm: Detect if model uses AWQ quantization and set appropriate dtype
if "awq" in model_id.lower() and current_platform.is_rocm():
dtype = "float16"
else:
dtype = model_info.dtype
model_config = ModelConfig(
model_id,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
hf_overrides=hf_overrides_fn,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
dtype=dtype,
)
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
assert supports_multimodal(model_cls)
factories = model_cls._processor_factory
# Capture helpers return capture-buffer containers (e.g.
# EncoderCudaGraphCaptureInputs), not TensorSchema mm inputs.
capture_helpers = {"prepare_encoder_cudagraph_capture_inputs"}
inputs_parse_methods = []
for attr_name in dir(model_cls):
if attr_name in capture_helpers:
continue
attr = getattr(model_cls, attr_name)
if hasattr(attr, "__annotations__"):
return_type = attr.__annotations__.get("return", None)
if return_type is not None and "Input" in str(return_type):
inputs_parse_methods.append(attr_name)
if not any(inputs_parse_methods):
pytest.skip(f"{model_arch} does not support tensor schema validation.")
ctx = InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
processing_info = factories.info(ctx)
supported_mm_limits = processing_info.get_supported_mm_limits()
limit_mm_per_prompt = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
if modality == "video":
return VideoDummyOptions(count=count)
if modality == "image":
return ImageDummyOptions(count=count)
if modality == "audio":
return AudioDummyOptions(count=count)
return BaseDummyOptions(count=count)
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
{
modality: _to_dummy_options(modality, count)
for modality, count in limit_mm_per_prompt.items()
}
)
processor = factories.build_processor(ctx)
with initialize_dummy_model(model_cls, model_config) as model:
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
for method_name in inputs_parse_methods:
print(
f"Testing `{method_name}` with modality={modality} "
f"and mm_kwargs{list(mm_kwargs.keys())}"
)
getattr(model, method_name)(modality=modality, **mm_kwargs)
-175
View File
@@ -1,175 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# flake8: noqa
"""Tests fp8 models against ground truth generation
Note: these tests will only pass on L4 GPU.
"""
import pytest
from tests.quantization.utils import is_quant_method_supported
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_kv_cache_dtype
from vllm.platforms import current_platform
from ..utils import check_logprobs_close
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported on this GPU type.",
)
@pytest.mark.parametrize(
"kv_cache_dtype,base_model,test_model",
[
# Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
(
"fp8_e4m3",
"meta-llama/Llama-3.2-1B-Instruct",
"nm-testing/Llama-3.2-1B-Instruct-FP8-KV",
),
# Test BF16 checkpoint w. fp8_e5m2 kv-cache.
(
"fp8_e5m2",
"meta-llama/Llama-3.2-1B-Instruct",
"meta-llama/Llama-3.2-1B-Instruct",
),
# Test BF16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
(
"fp8_e4m3",
"meta-llama/Llama-3.2-1B-Instruct",
"meta-llama/Llama-3.2-1B-Instruct",
),
],
)
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
@pytest.mark.parametrize("max_tokens", [4])
@pytest.mark.parametrize("enforce_eager", [True])
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
# NOTE: Increasing this in this suite will fail CI because we currently cannot
# reset distributed env properly. Use a value > 1 just when you test.
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_models(
vllm_runner,
example_prompts,
kv_cache_dtype: str,
base_model: str,
test_model: str,
max_tokens: int,
enforce_eager: bool,
backend: str,
tensor_parallel_size: int,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Only checks log probs match to cover the discrepancy in
numerical sensitive kernels.
"""
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_cuda():
pytest.skip(f"{kv_cache_dtype} is not supported by FLASH_ATTN on CUDA.")
if not flash_attn_supports_kv_cache_dtype(kv_cache_dtype):
pytest.skip(
f"{kv_cache_dtype} is not supported on this GPU type with {backend} attention."
)
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
MAX_MODEL_LEN = 1024
NUM_LOG_PROBS = 8
with vllm_runner(
base_model,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=tensor_parallel_size,
enforce_eager=enforce_eager,
kv_cache_dtype="auto",
attention_config={"backend": backend},
) as vllm_model:
baseline_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, NUM_LOG_PROBS
)
with vllm_runner(
test_model,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=tensor_parallel_size,
enforce_eager=enforce_eager,
kv_cache_dtype=kv_cache_dtype,
attention_config={"backend": backend},
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="fp16_kv_cache",
name_1="fp8_kv_cache",
)
@pytest.mark.cpu_model
@pytest.mark.skipif(not current_platform.is_cpu(), reason="test for the CPU backend.")
@pytest.mark.parametrize(
"kv_cache_dtype,base_model,test_model",
[
# Test BF16 checkpoint w. fp8_e5m2 kv-cache.
(
"fp8_e5m2",
"meta-llama/Llama-3.2-1B-Instruct",
"meta-llama/Llama-3.2-1B-Instruct",
),
],
)
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
@pytest.mark.parametrize("max_tokens", [4])
def test_cpu_models(
vllm_runner,
example_prompts,
kv_cache_dtype: str,
base_model: str,
test_model: str,
max_tokens: int,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""
Only checks log probs match to cover the discrepancy in
numerical sensitive kernels.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
MAX_MODEL_LEN = 1024
NUM_LOG_PROBS = 8
with vllm_runner(
base_model,
max_model_len=MAX_MODEL_LEN,
dtype="bfloat16",
kv_cache_dtype="auto",
) as vllm_model:
baseline_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, NUM_LOG_PROBS
)
with vllm_runner(
test_model,
max_model_len=MAX_MODEL_LEN,
dtype="bfloat16",
kv_cache_dtype=kv_cache_dtype,
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="bf16_kv_cache",
name_1="fp8_kv_cache",
)
-341
View File
@@ -1,341 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end accuracy test for GPT-OSS model quantization.
Config:
Task: gsm8k_platinum
Filter: flexible-extract
n-shot: 5
Metric: exact_match
Run: pytest tests/models/quantization/test_gpt_oss.py
"""
import importlib.metadata
import importlib.util
from dataclasses import dataclass
import huggingface_hub
import lm_eval
import pytest
import torch
from packaging import version
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.config import get_current_vllm_config
from vllm.forward_context import set_forward_context
from vllm.model_executor.layers.fused_moe.config import (
mxfp4_w4a8_moe_quant_config,
mxfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
aiter_triton_kernel_w4a8_moe_forward,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
aiter_triton_kernel_w4a16_moe_forward,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.torch_utils import set_random_seed
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx950, on_gfx1250
else:
def on_gfx950() -> bool:
return False
def on_gfx1250() -> bool:
return False
MODEL_ACCURACIES = {
# Full quantization: attention linears and MoE linears
"amd/gpt-oss-20b-WFP8-AFP8-KVFP8": 0.89,
# MoE linears only quantization
"amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-FP8-KV-FP8": 0.89,
# MoE linears only quantization
# "amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-MXFP4-KV-FP8": 0.90,
}
QUARK_MXFP4_AVAILABLE = importlib.util.find_spec("quark") is not None and version.parse(
importlib.metadata.version("amd-quark")
) >= version.parse("0.9.0")
def has_huggingface_access(repo):
try:
hf_api().list_repo_refs(repo)
return True
except huggingface_hub.errors.RepositoryNotFoundError:
return False
HF_HUB_AMD_ORG_ACCESS = all(
[has_huggingface_access(model_name) for model_name in MODEL_ACCURACIES]
)
@dataclass
class ModelCase:
model_id: str
tp: int
@dataclass
class EvaluationConfig:
model_name: str
def get_model_args(self, tp_size: int):
return {
"pretrained": self.model_name,
"chat_template_args": {"reasoning_effort": "low"},
"enable_thinking": True,
"think_end_token": "200008",
"tensor_parallel_size": tp_size,
"dtype": "auto",
"gpu_memory_utilization": 0.95,
"trust_remote_code": False,
"enable_prefix_caching": False,
"enforce_eager": False,
}
@pytest.mark.skipif(not QUARK_MXFP4_AVAILABLE, reason="amd-quark>=0.9 is not available")
@pytest.mark.skipif(
not HF_HUB_AMD_ORG_ACCESS,
reason="Read access to huggingface.co/amd is required for this test.",
)
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
@pytest.mark.parametrize("model_name, expected_accuracy", MODEL_ACCURACIES.items())
def test_gpt_oss_attention_quantization(
model_name: str,
tp_size: int,
expected_accuracy: float,
monkeypatch: pytest.MonkeyPatch,
):
if tp_size > current_platform.device_count():
pytest.skip("Not enough GPUs to run this test case")
if "amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-FP8-KV-FP8" in model_name and on_gfx950():
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
model_args = EvaluationConfig(model_name).get_model_args(tp_size)
extra_run_kwargs = {
"gen_kwargs": {"max_gen_toks": 8000},
"apply_chat_template": True,
"fewshot_as_multiturn": True,
"num_fewshot": 5,
}
lm_eval_out = lm_eval.simple_evaluate(
model="vllm",
model_args=model_args,
tasks="gsm8k_platinum",
batch_size="auto",
**extra_run_kwargs,
)
measured_accuracy = float(
lm_eval_out["results"]["gsm8k_platinum"]["exact_match,flexible-extract"]
)
rtol = 0.02
assert measured_accuracy >= expected_accuracy - rtol, (
f"Accuracy {measured_accuracy:.4f} is below threshold "
f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})"
)
@pytest.mark.skipif(
not (on_gfx950() or on_gfx1250()),
reason="AITER MXFP4 MoE requires gfx950 or gfx1250",
)
@pytest.mark.skipif(not is_aiter_found_and_supported(), reason="aiter is not installed")
@pytest.mark.skipif(torch.accelerator.device_count() == 0, reason="no gpu available")
@pytest.mark.parametrize(
"mxfp4_backend",
[Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_BF16],
ids=["w4a8", "w4a16"],
)
@pytest.mark.parametrize("num_experts", [32, 64, 128])
def test_aiter_mxfp4_moe_ignores_padded_rows(
mxfp4_backend: Mxfp4MoeBackend,
num_experts: int,
monkeypatch: pytest.MonkeyPatch,
dist_init,
) -> None:
"""Garbage in cudagraph padding rows must not reach the unpadded outputs.
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
-inf/inf/nan.
"""
TOKENS_PADDED = 8
TOKENS_UNPADDED = 7
# `rocm_aiter_ops` snapshots the environment at import time, so setting the
# env var alone is not enough.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
set_random_seed(0)
device = torch.device("cuda")
hidden_size = 2048
intermediate_size = 512
num_experts = 32
topk = 4
is_w4a8 = mxfp4_backend is Mxfp4MoeBackend.AITER_MXFP4_FP8
# TODO: clean this in oracle/mxfp4.
# `convert_gpt_oss_weight_to_mxfp4_moe_kernel_format` maps
# AITER_MXFP4_BF16 onto the CK layout (`shuffle_weight_a16w4`), which the
# triton monolithic cannot consume -- it wants the `_swizzle_mxfp4`
# PrecisionConfig that the TRITON branch of the same converter produces.
weight_backend = (
Mxfp4MoeBackend.AITER_MXFP4_FP8 if is_w4a8 else Mxfp4MoeBackend.TRITON
)
layer = torch.nn.Module()
layer.w13_weight = torch.randint(
0,
256,
(num_experts, 2 * intermediate_size, hidden_size // 2),
dtype=torch.uint8,
device=device,
)
layer.w2_weight = torch.randint(
0,
256,
(num_experts, hidden_size, intermediate_size // 2),
dtype=torch.uint8,
device=device,
)
# Keep the exponents near 127 (2**0) so the dequantized weights stay in a sane
# range and 255 (e8m0 NaN) is never hit.
layer.w13_weight_scale = torch.randint(
124,
131,
(num_experts, 2 * intermediate_size, hidden_size // 32),
dtype=torch.uint8,
device=device,
)
layer.w2_weight_scale = torch.randint(
124,
131,
(num_experts, hidden_size, intermediate_size // 32),
dtype=torch.uint8,
device=device,
)
layer.w13_bias = torch.randn(
(num_experts, 2 * intermediate_size), dtype=torch.float32, device=device
)
layer.w2_bias = torch.randn(
(num_experts, hidden_size), dtype=torch.float32, device=device
)
# Read off `layer`, not off the arguments, by the AITER_MXFP4_FP8 branch.
layer.w13_input_scale = torch.full(
(num_experts,), 0.1, dtype=torch.float32, device=device
)
layer.w2_input_scale = torch.full(
(num_experts,), 0.1, dtype=torch.float32, device=device
)
(
w13_weight,
w2_weight,
w13_scale,
w2_scale,
w13_bias,
w2_bias,
) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
weight_backend,
layer,
layer.w13_weight,
layer.w2_weight,
layer.w13_weight_scale,
layer.w2_weight_scale,
w13_bias=layer.w13_bias,
w2_bias=layer.w2_bias,
)
quant_config_factory = (
mxfp4_w4a8_moe_quant_config if is_w4a8 else mxfp4_w4a16_moe_quant_config
)
quant_config = quant_config_factory(
w1_scale=w13_scale,
w2_scale=w2_scale,
w1_bias=w13_bias,
w2_bias=w2_bias,
)
hidden_states = torch.randn(
(TOKENS_PADDED, hidden_size), dtype=torch.bfloat16, device=device
)
gating_output = torch.randn(
(TOKENS_PADDED, num_experts), dtype=torch.bfloat16, device=device
)
hidden_states[TOKENS_UNPADDED:, 0] = float("inf")
hidden_states[TOKENS_UNPADDED:, 1] = float("-inf")
hidden_states[TOKENS_UNPADDED:, 2] = float("nan")
gating_output[TOKENS_UNPADDED:, :] = float("-inf")
gating_output[TOKENS_UNPADDED:, 0] = float("inf")
gating_output[TOKENS_UNPADDED:, 1] = float("nan")
gating_output[TOKENS_UNPADDED:, 2] = -float("nan")
is_padding = torch.zeros(TOKENS_PADDED, dtype=torch.bool, device=device)
is_padding[TOKENS_UNPADDED:] = True
# Poison memory: after `del blocks`, further `torch.empty` in routing with wrongful
# expert ids may read into these memory sections resulting in
# potential memory access fault.
blocks = []
for numel in (16, 32, 64, 128, 256, 512, 1024, 4096, 1 << 14, 1 << 16, 1 << 20):
blocks.append(torch.full((numel,), 100e7, dtype=torch.int32, device=device))
# return blocks
del blocks
torch.accelerator.synchronize()
with set_forward_context(None, get_current_vllm_config(), is_padding=is_padding):
if is_w4a8:
# AITER_MXFP4_FP8 routes through the triton monolithic expert, which
# does its own top-k inside `aiter.ops.triton.moe.moe_routing`.
output = aiter_triton_kernel_w4a8_moe_forward(
hidden_states=hidden_states,
w1=w13_weight,
w2=w2_weight,
gating_output=gating_output,
topk=topk,
renormalize=True,
quant_config=quant_config,
global_num_experts=num_experts,
unpadded_N_w1=2 * intermediate_size,
unpadded_K_w1=hidden_size,
unpadded_N_w2=hidden_size,
unpadded_K_w2=intermediate_size,
)
else:
output = aiter_triton_kernel_w4a16_moe_forward(
hidden_states=hidden_states,
w1=w13_weight,
w2=w2_weight,
gating_output=gating_output,
topk=topk,
renormalize=True,
quant_config=quant_config,
global_num_experts=num_experts,
unpadded_N_w1=2 * intermediate_size,
unpadded_K_w1=hidden_size,
unpadded_N_w2=hidden_size,
unpadded_K_w2=intermediate_size,
)
assert torch.isfinite(output[:TOKENS_UNPADDED]).all()
@@ -1,88 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests AutoGPTQ (GPTQ with Marlin kernels) output correctness.
Note: Marlin internally uses locks to synchronize the threads. This can
result in very slight nondeterminism for Marlin. As a result, we re-run the test
up to 3 times to see if we pass.
"""
import os
import pytest
from tests.quantization.utils import is_quant_method_supported
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
from vllm.platforms import current_platform
from ..utils import check_logprobs_close
os.environ["TOKENIZERS_PARALLELISM"] = "true"
MAX_MODEL_LEN = 1024
MODELS = [
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "main"),
# desc_act=True is a no-op for channelwise quantization.
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
("TechxGenus/gemma-2b-it-GPTQ", "main"),
]
@pytest.mark.flaky(reruns=3)
@pytest.mark.skipif(
not is_quant_method_supported("auto_gptq")
or current_platform.is_rocm()
or not current_platform.is_cuda(),
reason="auto_gptq is not supported on this GPU type.",
)
@pytest.mark.parametrize("model", MODELS)
@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
vllm_runner,
example_prompts,
model,
dtype: str,
max_tokens: int,
num_logprobs: int,
) -> None:
model_name, revision = model
# Run marlin.
with vllm_runner(
model_name=model_name,
revision=revision,
dtype=dtype,
quantization="marlin",
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=1,
) as gptq_marlin_model:
gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
example_prompts[:-1], max_tokens, num_logprobs
)
_ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
# Run gptq.
# The naive gptq kernel doesn't support bf16 yet.
# Here we always compare fp16/bf16 gpt marlin kernel
# to fp16 gptq kernel.
with vllm_runner(
model_name=model_name,
revision=revision,
dtype="half",
quantization="gptq",
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=1,
) as gptq_model:
gptq_outputs = gptq_model.generate_greedy_logprobs(
example_prompts[:-1], max_tokens, num_logprobs
)
check_logprobs_close(
outputs_0_lst=gptq_outputs,
outputs_1_lst=gptq_marlin_outputs,
name_0="gptq",
name_1="gptq_marlin",
)
@@ -1,84 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# flake8: noqa
"""Tests Model Optimizer fp8 models against ground truth generation
Note: these tests will only pass on H100
"""
import os
import pytest
from transformers import AutoTokenizer
from tests.quantization.utils import is_quant_method_supported
from vllm import LLM, SamplingParams
os.environ["TOKENIZERS_PARALLELISM"] = "true"
MAX_MODEL_LEN = 1024
MODELS = ["nvidia/Llama-3.1-8B-Instruct-FP8"]
EXPECTED_STRS_MAP = {
"nvidia/Llama-3.1-8B-Instruct-FP8": [
"You're referring to VLLM, a high-performance Large Language Model (LLM) inference and",
"Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ",
"The comparison between artificial intelligence (AI) and human intelligence in terms of processing information is a complex and",
'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
"**The Spark of Imagination**\n\nZeta-5, a sleek and efficient robot, whir",
"The COVID-19 pandemic has had a profound impact on global economic structures and business models, leading to",
"The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of",
"Here are the translations:\n\n**Japanese:** 「早起きは早く獲物をとる",
]
}
# This test compares against golden strings for exact match since
# there is no baseline implementation to compare against
# and is unstable w.r.t specifics of the fp8 implementation or
# the hardware being run on.
# Disabled to prevent it from breaking the build
@pytest.mark.skip(
reason="Prevent unstable test based on golden strings from breaking the build."
)
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported on this GPU type.",
)
@pytest.mark.parametrize("model_name", MODELS)
def test_models(example_prompts, model_name) -> None:
llm = LLM(
model=model_name,
max_model_len=MAX_MODEL_LEN,
trust_remote_code=True,
enforce_eager=True,
quantization="modelopt",
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
formatted_prompts = [
tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
tokenize=False,
add_generation_prompt=True,
)
for prompt in example_prompts
]
params = SamplingParams(max_tokens=20, temperature=0)
generations: list[str] = []
# Note: these need to be run 1 at a time due to numerical precision,
# since the expected strs were generated this way.
for prompt in formatted_prompts:
outputs = llm.generate(prompt, params)
generations.append(outputs[0].outputs[0].text)
del llm
print(model_name, generations)
expected_strs = EXPECTED_STRS_MAP[model_name]
for i in range(len(example_prompts)):
generated_str = generations[i]
expected_str = expected_strs[i]
assert expected_str == generated_str, (
f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}"
)
-42
View File
@@ -1,42 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# flake8: noqa
"""Tests Quark mxfp4 models against ground truth generation"""
import pytest
from vllm import LLM, SamplingParams
MODELS = ["amd/Llama-2-7b-chat-hf-wmxfp4-amxfp4-kvfp8-scale-uint8"]
EXPECTED_STRS_MAP = {
"amd/Llama-2-7b-chat-hf-wmxfp4-amxfp4-kvfp8-scale-uint8": [
"\n### Key Features\n\n* **High-throughput Inference**: vLL",
"\nArtificial intelligence (AI) has evolved significantly since its inception in the 1",
"Artificial intelligence (AI) and human intelligence (HI) are two distinct concepts that have been",
"A neural network is a machine learning model inspired by the structure of the human brain. It consists of",
"\nTitle: The Dreaming Robot\n\nAs the sun set on the bustling metropol",
"\nThe COVID-19 pandemic has had a profound impact on global economic structures and business",
"The Mona Lisa painting, created by Leonardo da Vinci in the early 16th",
" everybody knows this proverbial saying, but did you know that it's not entirely accurate?",
]
}
@pytest.mark.skip(reason="Model to be released in the future")
@pytest.mark.quant_model
@pytest.mark.parametrize("model_name", MODELS)
def test_models(example_prompts, model_name) -> None:
sampling_params = SamplingParams(max_tokens=20, temperature=0)
llm = LLM(
model=model_name,
kv_cache_dtype="fp8",
quantization="quark",
)
outputs = llm.generate(example_prompts, sampling_params)
for i, output in enumerate(outputs):
output_str = output.outputs[0].text
expected_str = EXPECTED_STRS_MAP[model_name][i]
assert expected_str == output_str, (
f"Expected: {expected_str!r}\nvLLM: {output_str!r}"
)
-255
View File
@@ -1,255 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for online MXFP8 quantization.
Loads a BF16 model with ``--quantization mxfp8`` (online quantization) and
compares log-probabilities against the same model served in BF16 without
quantization. This exercises the full pipeline: config parsing,
``Mxfp8OnlineLinearMethod``, ``Mxfp8OnlineMoEMethod``, weight loading,
online quantization / shuffling, and inference through ``apply_monolithic``.
Layer skipping (``modules_to_not_convert``) is configured in the model's
``config.json`` under ``quantization_config`` and is not tested here.
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
diverse prompts from ``tests/prompts/example.txt``.
"""
import pytest
import torch
from tests.quantization.utils import is_quant_method_supported
from vllm.platforms import current_platform
from ..utils import check_logprobs_close
# A small MoE model that fits on a single GPU and has both linear + MoE layers.
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
# A small dense model (no MoE) to validate the linear-only path.
DENSE_MODEL = "Qwen/Qwen3-0.6B"
MAX_MODEL_LEN = 1024
MAX_TOKENS = 4
NUM_LOG_PROBS = 8
@pytest.mark.skipif(
not is_quant_method_supported("mxfp8"),
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
)
@pytest.mark.quant_model
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_mxfp8_logprobs(
vllm_runner,
quant_baseline_logprobs,
example_prompts,
model: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Compare BF16 baseline logprobs against online MXFP8-quantized model.
Runs the same model twice -- once in BF16 (baseline) and once with
online MXFP8 quantization -- then checks that the top log-probabilities
are close. Only 4 tokens are generated to keep the test fast while
still catching numerical divergence.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
baseline_outputs = quant_baseline_logprobs(
model,
example_prompts,
max_model_len=MAX_MODEL_LEN,
max_tokens=MAX_TOKENS,
num_logprobs=NUM_LOG_PROBS,
)
with vllm_runner(
model,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
quantization="mxfp8",
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="bf16",
name_1="mxfp8",
)
@pytest.mark.skipif(
not is_quant_method_supported("mxfp8"),
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AITER MXFP8 MoE backend is ROCm-only.",
)
@pytest.mark.quant_model
def test_mxfp8_aiter_requires_swigluoai_activation(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
monkeypatch.setattr(
aiter_mxfp8_moe.AiterMxfp8Experts,
"_supports_current_device",
staticmethod(lambda: True),
)
config = FusedMoEConfig(
num_experts=8,
experts_per_token=2,
hidden_dim=256,
intermediate_size=256,
num_local_experts=8,
num_logical_experts=8,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
moe_backend="aiter",
)
with pytest.raises(ValueError, match="requires activation=swigluoai_uninterleave"):
select_mxfp8_moe_backend(config)
@pytest.mark.skipif(
not is_quant_method_supported("mxfp8"),
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AITER MXFP8 MoE backend is ROCm-only.",
)
@pytest.mark.quant_model
def test_mxfp8_aiter_requires_swigluoai_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
monkeypatch.setattr(
aiter_mxfp8_moe.AiterMxfp8Experts,
"_supports_current_device",
staticmethod(lambda: True),
)
config = FusedMoEConfig(
num_experts=8,
experts_per_token=2,
hidden_dim=256,
intermediate_size=256,
num_local_experts=8,
num_logical_experts=8,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
moe_backend="aiter",
)
with pytest.raises(ValueError, match="hardcodes SwiGLU-OAI"):
select_mxfp8_moe_backend(config)
@pytest.mark.skipif(
not is_quant_method_supported("mxfp8"),
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="AITER MXFP8 MoE backend is ROCm-only.",
)
@pytest.mark.quant_model
def test_mxfp8_aiter_accepts_swigluoai_params(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
select_mxfp8_moe_backend,
)
monkeypatch.setattr(
aiter_mxfp8_moe.AiterMxfp8Experts,
"_supports_current_device",
staticmethod(lambda: True),
)
config = FusedMoEConfig(
num_experts=8,
experts_per_token=2,
hidden_dim=256,
intermediate_size=256,
num_local_experts=8,
num_logical_experts=8,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
in_dtype=torch.bfloat16,
device="cuda",
routing_method=RoutingMethodType.Renormalize,
moe_backend="aiter",
swiglu_alpha=aiter_mxfp8_moe._AITER_SWIGLU_ALPHA,
swiglu_beta=aiter_mxfp8_moe._AITER_SWIGLU_BETA,
)
backend, experts_cls = select_mxfp8_moe_backend(config)
assert backend == Fp8MoeBackend.AITER_MXFP8
assert experts_cls is aiter_mxfp8_moe.AiterMxfp8Experts
@pytest.mark.skipif(
not is_quant_method_supported("mxfp8"),
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
)
@pytest.mark.quant_model
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_mxfp8_generation(vllm_runner, model: str) -> None:
"""Smoke test: verify online MXFP8 model generates coherent text."""
prompt = "1 2 3 4 5"
with vllm_runner(
model,
enforce_eager=True,
quantization="mxfp8",
max_model_len=MAX_MODEL_LEN,
) as vllm_model:
output = vllm_model.generate_greedy([prompt], max_tokens=5)
generated = output[0][1]
assert len(generated) > len(prompt), (
f"MXFP8 model produced no new tokens. Output: {generated!r}"
)
-153
View File
@@ -1,153 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# flake8: noqa
"""Tests Model Optimizer nvfp4 models against ground truth generation
Note: these tests will only pass on B200
"""
import os
from typing import List
import pytest
from transformers import AutoTokenizer
from tests.quantization.utils import is_quant_method_supported
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
os.environ["TOKENIZERS_PARALLELISM"] = "true"
MAX_MODEL_LEN = 1024
MODELS = ["nvidia/Llama-3.3-70B-Instruct-FP4"]
EXPECTED_STRS_MAP = {
"nvidia/Llama-3.3-70B-Instruct-FP4": [
"vLLM (Vectorized Large Language Model) is indeed a high-throughput and memory-efficient inference",
"Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ",
"Artificial intelligence (AI) and human intelligence (HI) are two distinct forms of intelligence that process",
"A neural network is a type of machine learning model inspired by the structure and function of the human brain",
"In the heart of a cutting-edge robotics lab, a team of engineers had been working tirelessly to push",
"The COVID-19 pandemic has had a profound impact on global economic structures and future business models, leading",
"The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of",
"Here are the translations:\n\n* Japanese: (Sasuga no tori ga miwa o ts",
]
}
# This test compares against golden strings for exact match since
# there is no baseline implementation to compare against
# and is unstable w.r.t specifics of the fp4 implementation or
# the hardware being run on.
# Disabled to prevent it from breaking the build
@pytest.mark.skip(
reason="Prevent unstable test based on golden strings from breaking the build "
" and test input model being too large and hanging the system."
)
@pytest.mark.skipif(
not is_quant_method_supported("modelopt_fp4"),
reason="modelopt_fp4 is not supported on this GPU type.",
)
@pytest.mark.parametrize("model_name", MODELS)
def test_models(example_prompts, model_name) -> None:
llm = LLM(
model=model_name,
max_model_len=MAX_MODEL_LEN,
trust_remote_code=True,
enforce_eager=True,
quantization="modelopt_fp4",
)
tokenizer = AutoTokenizer.from_pretrained(model_name)
formatted_prompts = [
tokenizer.apply_chat_template(
[{"role": "user", "content": prompt}],
tokenize=False,
add_generation_prompt=True,
)
for prompt in example_prompts
]
params = SamplingParams(max_tokens=20, temperature=0)
generations: List[str] = []
# Note: these need to be run 1 at a time due to numerical precision,
# since the expected strs were generated this way.
for prompt in formatted_prompts:
outputs = llm.generate(prompt, params)
generations.append(outputs[0].outputs[0].text)
del llm
print(model_name, generations)
expected_strs = EXPECTED_STRS_MAP[model_name]
for i in range(len(example_prompts)):
generated_str = generations[i]
expected_str = expected_strs[i]
assert expected_str == generated_str, (
f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}"
)
EAGER = [True, False]
SM_100_NVFP4_BACKENDS = [
"flashinfer_cutedsl",
"flashinfer_cudnn",
"flashinfer_trtllm",
"flashinfer_cutlass",
"torch",
]
@pytest.mark.parametrize("model", ["nvidia/Llama-3.1-8B-Instruct-NVFP4"])
@pytest.mark.parametrize("eager", EAGER)
@pytest.mark.parametrize(
"backend",
[
"emulation",
"flashinfer_cutedsl",
"flashinfer_cudnn",
"flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
"flashinfer_cutlass",
"torch",
],
)
def test_nvfp4(vllm_runner, model, eager, backend):
if backend == "flashinfer_cutedsl" and not (
current_platform.is_device_capability_family(100)
):
pytest.skip("The flashinfer_cutedsl backend is only supported on SM10x")
if (
not current_platform.has_device_capability(100)
and backend in SM_100_NVFP4_BACKENDS
):
pytest.skip(
f"The backend {backend} is not supported with current_platform.has_device_capability(100) == False"
)
with vllm_runner(model, enforce_eager=eager, linear_backend=backend) as llm:
output = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
assert output[0][1] == "1 2 3 4 5 6"
@pytest.mark.parametrize(
"model",
[
"nvidia/Qwen3-30B-A3B-NVFP4",
"RedHatAI/Qwen3-30B-A3B-NVFP4",
],
)
@pytest.mark.parametrize("backend", ["emulation"])
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
)
def test_nvfp4_moe(vllm_runner, model, backend):
with vllm_runner(
model,
moe_backend=backend,
linear_backend=backend,
load_format="dummy",
hf_overrides={"num_hidden_layers": 2},
) as llm:
_ = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end accuracy tests for per-token-head KV cache quantization.
Compares logprobs between a baseline bf16 model and the same model with
per-token-head quantized KV cache (int8 or fp8) using the Triton attention
backend.
Run: pytest tests/models/quantization/test_per_token_kv_cache.py -v -s
"""
import pytest
from vllm.platforms import current_platform
from ..utils import check_logprobs_close
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Per-token-head KV cache requires CUDA or ROCm GPU.",
)
@pytest.mark.parametrize(
"base_model,test_model",
[
(
"meta-llama/Llama-3.2-1B-Instruct",
"meta-llama/Llama-3.2-1B-Instruct",
),
],
)
@pytest.mark.parametrize(
"kv_cache_dtype",
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
)
@pytest.mark.parametrize("max_tokens", [4])
@pytest.mark.parametrize("enforce_eager", [True])
@pytest.mark.parametrize("backend", ["TRITON_ATTN"])
@pytest.mark.parametrize("tensor_parallel_size", [1])
def test_per_token_head_kv_cache_accuracy(
vllm_runner,
example_prompts,
base_model: str,
test_model: str,
kv_cache_dtype: str,
max_tokens: int,
enforce_eager: bool,
backend: str,
tensor_parallel_size: int,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Compare logprobs between bf16 baseline and per-token-head quantized KV
cache.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
MAX_MODEL_LEN = 1024
NUM_LOG_PROBS = 8
with vllm_runner(
base_model,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=tensor_parallel_size,
enforce_eager=enforce_eager,
kv_cache_dtype="auto",
attention_config={"backend": backend},
) as vllm_model:
baseline_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, NUM_LOG_PROBS
)
with vllm_runner(
test_model,
max_model_len=MAX_MODEL_LEN,
tensor_parallel_size=tensor_parallel_size,
enforce_eager=enforce_eager,
kv_cache_dtype=kv_cache_dtype,
attention_config={"backend": backend},
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="bf16_kv_cache",
name_1=f"{kv_cache_dtype}_kv_cache",
)
-407
View File
@@ -1,407 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for the fused QSA prepare kernel."""
import pytest
import torch
from vllm.models.qwen4_exp.common.qsa_cache import (
canonical_qsa_rope_positions,
circular_qsa_slot_mapping,
compressed_qsa_slot_mapping,
)
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
qsa_compress_groups_with_ratio,
qsa_store_cache_rows,
)
from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
requires_qsa_kernels = pytest.mark.skipif(
not current_platform.is_cuda() or not HAS_TRITON,
reason="QSA kernels require CUDA and Triton",
)
HQ, D = 4, 128
CR = 4
MROPE_SECTION = (11, 11, 10)
EPS = 1e-6
BLOCK_SIZE = 16
COMP_PAGE = BLOCK_SIZE // CR
ROPE_POS_OFFSET = D
RTOL = 1.6e-2
ATOL = 1e-2
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
def _make_block_table(block_counts):
num_blocks = sum(block_counts)
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
physical_blocks = torch.randperm(num_blocks)
offset = 0
for request, count in enumerate(block_counts):
table[request, :count] = physical_blocks[offset : offset + count]
offset += count
return table, num_blocks
def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None:
# e4m3 is sign-magnitude, so within a sign the uint8 code order matches the
# value order and one ulp is one code step. The two paths' intermediates
# differ in the pooling accumulation order, which at denormal magnitudes
# (absolute grid step 2^-9) shows up as up to 2 code steps.
code_diff = (
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
).abs()
abs_diff = (actual.float() - expected.float()).abs()
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
"""Random main-attention arguments with a paged cache in vLLM's layout."""
num_blocks = num_tokens // MAIN_PAGE + 2
cache = torch.zeros(
num_blocks,
MAIN_HK,
MAIN_PAGE,
2 * MAIN_D,
dtype=torch.uint8 if fp8_cache else torch.bfloat16,
device="cuda",
).transpose(1, 2)
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
slots[-1] = -1
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
qkv_width = 2 * (MAIN_HQ + MAIN_HK) * MAIN_D
return dict(
main_qkv=torch.randn(
num_tokens,
qkv_width + (HQ + 1) * D,
dtype=torch.bfloat16,
device="cuda",
)[:, :qkv_width],
main_q_norm_weight=norm_weights[0],
main_k_norm_weight=norm_weights[1],
main_eps=EPS,
main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache,
main_slot_mapping=slots,
main_k_scale=0.5,
main_v_scale=2.0,
)
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
"""Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash."""
from vllm._custom_ops import reshape_and_cache_flash
from vllm.model_executor.layers.fused_qk_norm_rope import (
fused_qk_rmsnorm_rope_gate,
)
q_gate, k, v = main["main_qkv"].split(
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
)
q, k, gate = fused_qk_rmsnorm_rope_gate(
q_gate,
k,
main["main_q_norm_weight"],
main["main_k_norm_weight"],
rope.cos_sin_cache,
positions,
EPS,
MAIN_HQ,
MAIN_HK,
MAIN_D,
rope.rotary_dim,
mrope_section=MROPE_SECTION if positions.ndim == 2 else None,
norm_beta=1.0,
)
kv_cache = main["main_kv_cache"]
fp8_cache = kv_cache.dtype == torch.float8_e4m3fn
cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache)
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
reshape_and_cache_flash(
k.view(-1, MAIN_HK, MAIN_D),
v.view(-1, MAIN_HK, MAIN_D),
key_cache,
value_cache,
main["main_slot_mapping"],
"fp8" if fp8_cache else "auto",
torch.tensor(main["main_k_scale"], device="cuda"),
torch.tensor(main["main_v_scale"], device="cuda"),
)
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
assert torch.equal(gate_out.flatten(1), gate)
if fp8_cache:
assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn))
else:
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
@requires_qsa_kernels
@pytest.mark.usefixtures("default_vllm_config")
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize(
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
[
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
pytest.param(
True,
False,
False,
8,
*MIXED_BATCH,
id="mrope-model-1d-no-position-cache",
),
pytest.param(
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
),
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
],
)
def test_qsa_fused_prepare_matches_unfused(
indexer_dtype,
mrope,
is_2d_positions,
cache_rope_positions,
state_size,
seq_lens,
query_lens,
history_lens,
) -> None:
from flashinfer.norm import gemma_rmsnorm
from vllm.model_executor.layers.rotary_embedding import get_rope
device = "cuda"
rope_params = {
"partial_rotary_factor": 0.25,
"rope_theta": 10000000,
"rope_type": "default",
}
if mrope:
rope_params["mrope_interleaved"] = True
rope_params["mrope_section"] = list(MROPE_SECTION)
with torch.device(device):
rope = get_rope(
head_size=256,
max_position=32768,
rope_parameters=rope_params,
dtype=torch.bfloat16,
)
token_to_req = torch.cat(
[
torch.full((length,), request, dtype=torch.int32)
for request, length in enumerate(query_lens)
]
).to(device)
logical_positions = torch.cat(
[
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
for seq_len, query_len in zip(seq_lens, query_lens)
]
).to(device)
num_tokens = logical_positions.numel()
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
).to(device)
positions = (
torch.stack(
[
logical_positions,
logical_positions // 7 + 3,
logical_positions // 13 + 11,
]
)
if is_2d_positions
else logical_positions
)
position_rows = (
canonical_qsa_rope_positions(positions)
if cache_rope_positions
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
)
compressed_block_counts = [
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
]
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
compressed_block_table, num_compressed_blocks = _make_block_table(
compressed_block_counts
)
raw_block_table = raw_block_table.to(device)
raw_slots = circular_qsa_slot_mapping(
raw_block_table,
token_to_req,
logical_positions,
state_size,
query_start_loc,
)
compressed_slots = compressed_qsa_slot_mapping(
compressed_block_table.to(device),
token_to_req,
logical_positions,
COMP_PAGE,
CR,
)
group_counts = torch.tensor(
[
seq_len // CR - (seq_len - query_len) // CR
for seq_len, query_len in zip(seq_lens, query_lens)
],
dtype=torch.int32,
device=device,
)
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
work_requests = torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32, device=device),
k_work_counts,
)
local_work = torch.arange(
int(k_start_loc[-1]), dtype=torch.int32, device=device
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
k_work_metadata[: work_requests.numel()] = torch.stack(
(work_requests, local_work), dim=1
)
raw_width = D + 12 if cache_rope_positions else D
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
# blocks have a larger stride than their logical contents.
raw_page_elements = state_size * raw_width
fused_raw_storage = torch.zeros(
num_raw_blocks,
raw_page_elements + 16,
dtype=torch.bfloat16,
device=device,
)
fused_raw = torch.as_strided(
fused_raw_storage,
(num_raw_blocks, state_size, 1, raw_width),
(raw_page_elements + 16, raw_width, raw_width, 1),
)
for request, history_len in enumerate(history_lens):
history_end = seq_lens[request] - query_lens[request]
for position in range(history_end - history_len, history_end):
block = int(raw_block_table[request, 0])
row = fused_raw[block, position % state_size, 0]
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
if cache_rope_positions:
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
torch.tensor(
[position, position // 7 + 3, position // 13 + 11],
dtype=torch.int64,
device=device,
)
)
unfused_raw = fused_raw.clone()
compressed_page_elements = COMP_PAGE * D
fused_compressed_storage = torch.zeros(
num_compressed_blocks,
compressed_page_elements + 16,
dtype=indexer_dtype,
device=device,
)
fused_compressed = torch.as_strided(
fused_compressed_storage,
(num_compressed_blocks, COMP_PAGE, 1, D),
(compressed_page_elements + 16, D, D, 1),
)
unfused_compressed = fused_compressed.clone()
# TP=4 QKVG width makes the indexer slice row-strided, as in the merged
# projection. The exact width is unimportant; the larger stride is tested.
main_qkvg_width = 3584
projected_qk = torch.randn(
num_tokens, main_qkvg_width + (HQ + 1) * D, dtype=torch.bfloat16, device=device
)[:, main_qkvg_width:]
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
# The main-attention cache follows the recipe's pairing with the indexer.
main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn)
main_q_out, main_gate_out = qsa_prepare(
projected_qk[:, : HQ * D],
projected_qk[:, HQ * D :],
positions,
rope.cos_sin_cache,
q_weight,
k_weight,
EPS,
fused_query,
fused_raw,
raw_slots,
raw_block_table,
query_start_loc,
logical_positions,
fused_compressed,
compressed_slots,
k_work_metadata,
compress_ratio=CR,
mrope_section=MROPE_SECTION if mrope else None,
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
**main,
)
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
unfused_query = gemma_rmsnorm(
unfused_query.reshape(-1, D), q_weight, EPS
).reshape_as(unfused_query)
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
raw_keys = unfused_raw[..., :D]
rope_positions = (
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
if cache_rope_positions
else None
)
pooled, first_positions = qsa_compress_groups_with_ratio(
projected_qk[:, HQ * D :].reshape(-1, 1, D),
position_rows,
raw_keys,
raw_block_table,
token_to_req,
query_start_loc,
logical_positions,
compressed_slots,
CR,
rope_positions,
)
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
-1, 1, D
)
group_positions = (
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
)
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
if rope_positions is not None:
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
if indexer_dtype == torch.float8_e4m3fn:
# Both paths round the same ~bf16 intermediates to e4m3. Bitwise
# equality (the dsv4 indexer test's bar) does not hold here: the 1D
# reference rope (forward_cuda) differs from _norm_rope by up to 1
# bf16 ulp, and even the MRoPE pair flips a code occasionally.
unfused_query = unfused_query.to(indexer_dtype)
assert_fp8_within_one_ulp(fused_query, unfused_query)
else:
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
if indexer_dtype == torch.float8_e4m3fn:
assert_fp8_within_one_ulp(fused_compressed, unfused_compressed)
else:
torch.testing.assert_close(
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
)
-185
View File
@@ -1,185 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import pytest
from tests.utils import multi_gpu_marks
from vllm import SamplingParams
from vllm.config import CompilationConfig
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.metrics.reader import Counter
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
def _small_hyv4_config(config):
# Apply to both target and MTP draft while retaining MLA/indexer dimensions.
config.update(
{
"hidden_size": 512,
"num_hidden_layers": 2,
"num_attention_heads": 64,
"q_lora_rank": 128,
"intermediate_size": 512,
"moe_intermediate_size": 256,
"n_routed_experts": 4,
"num_experts_per_tok": 2,
"n_shared_experts": 1,
"enable_ihc": True,
"index_n_heads": 32,
"index_topk": 2048,
"layer_types": ["deepseek_sparse_attention"] * 2,
"indexer_types": ["full", "shared"],
"num_nextn_predict_layers": 1,
"vocab_size": 256,
"bos_token_id": 1,
"eos_token_id": 2,
"pad_token_id": 0,
}
)
return config
def _num_captured_graphs(_worker):
from vllm.compilation.counter import compilation_counter
return compilation_counter.num_cudagraph_captured
def _piecewise_compile_state(worker):
from vllm.compilation.counter import compilation_counter
# `cudagraph_mode` is returned by name: the enum does not survive the
# collective_rpc round trip, it arrives as its bare value.
return (
compilation_counter.num_models_seen,
compilation_counter.num_cudagraph_captured,
worker.vllm_config.compilation_config.cudagraph_mode.name,
)
@pytest.mark.parametrize(
"tp,enforce_eager,mtp,dtype,kv_cache_dtype",
[
pytest.param(1, True, False, "bfloat16", "auto", id="eager-tp1"),
pytest.param(1, True, False, "float16", "auto", id="eager-fp16"),
pytest.param(1, True, False, "bfloat16", "fp8", id="eager-fp8-kv"),
pytest.param(
2,
False,
False,
"bfloat16",
"auto",
id="graphs-tp2",
marks=multi_gpu_marks(num_gpus=2),
),
pytest.param(1, True, True, "bfloat16", "auto", id="eager-mtp"),
],
)
def test_hyv4_dummy_generation(
vllm_runner,
enable_pickle,
tp: int,
enforce_eager: bool,
mtp: bool,
dtype: str,
kv_cache_dtype: str,
) -> None:
"""Exercise prefill, decode, and batch consistency with dummy HY weights."""
with vllm_runner(
"tencent/Hy4-preview",
trust_remote_code=True,
skip_tokenizer_init=True,
load_format="dummy",
disable_log_stats=not mtp,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
hf_overrides=_small_hyv4_config,
max_model_len=256,
max_num_batched_tokens=128,
max_num_seqs=8,
enable_chunked_prefill=True,
enable_prefix_caching=False,
kv_cache_memory_bytes=128 * 1024 * 1024,
enforce_eager=enforce_eager,
tensor_parallel_size=tp,
speculative_config={"method": "mtp", "num_speculative_tokens": 2}
if mtp
else None,
seed=17,
# Exercise supported decode graphs independently of default graph policy.
compilation_config=CompilationConfig(
mode=CompilationMode.NONE,
cudagraph_mode=CUDAGraphMode.NONE
if enforce_eager
else CUDAGraphMode.FULL_DECODE_ONLY,
cudagraph_capture_sizes=[] if enforce_eager else [1, 2, 4],
),
) as model:
if not enforce_eager:
assert model.collective_rpc(_num_captured_graphs) == [3] * tp
prompts = [
{"prompt_token_ids": [11, 12, 13, 14]},
{"prompt_token_ids": list(range(20, 84))},
]
params = SamplingParams(
temperature=0, max_tokens=4, ignore_eos=True, logprobs=1
)
outputs = model.llm.generate(prompts, params)
assert len(outputs) == len(prompts)
for prompt, output in zip(prompts, outputs):
completion = output.outputs[0]
assert len(completion.token_ids) == 4
assert completion.cumulative_logprob is not None
assert math.isfinite(completion.cumulative_logprob)
single = model.llm.generate([prompt], params)[0].outputs[0]
assert single.token_ids == completion.token_ids
if mtp:
assert any(
isinstance(metric, Counter)
and metric.name == "vllm:spec_decode_num_drafts"
and metric.value > 0
for metric in model.llm.get_metrics()
), "MTP drafter did not run"
def test_hyv4_backbone_reaches_the_compiler(vllm_runner, enable_pickle) -> None:
"""Guard against the backbone losing `@support_torch_compile`.
An undecorated backbone is not a hard failure: vLLM downgrades
`cudagraph_mode` to NONE and serves the whole model eager, which on TP8
gfx942 cost ~4x decode latency.
"""
with vllm_runner(
"tencent/Hy4-preview",
trust_remote_code=True,
skip_tokenizer_init=True,
load_format="dummy",
dtype="bfloat16",
hf_overrides=_small_hyv4_config,
max_model_len=256,
max_num_batched_tokens=128,
max_num_seqs=8,
enable_chunked_prefill=True,
enable_prefix_caching=False,
kv_cache_memory_bytes=128 * 1024 * 1024,
enforce_eager=False,
seed=17,
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_mode=CUDAGraphMode.PIECEWISE,
cudagraph_capture_sizes=[1, 2, 4],
),
) as model:
for models_seen, captured, mode in model.collective_rpc(
_piecewise_compile_state
):
assert models_seen > 0, "backbone was never handed to the compiler"
assert mode == CUDAGraphMode.PIECEWISE.name
assert captured > 0
-264
View File
@@ -1,264 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import gc
from collections.abc import Generator
from functools import partial
from unittest.mock import patch
import pytest
import torch
from vllm import LLM
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
from vllm.v1.core.kv_cache_planning import get_kv_cache_configs
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
from vllm.v1.engine.core import EngineCore as V1EngineCore
from ..utils import (
create_new_process_for_each_test,
requires_spawn_multiprocessing,
wait_for_gpu_memory_to_clear,
wait_for_memory_to_settle,
)
from .registry import (
_TRANSFORMERS_BACKEND_MODELS,
AUTO_EXAMPLE_MODELS,
HF_EXAMPLE_MODELS,
HfExampleModels,
)
from .utils import dummy_hf_overrides
logger = init_logger(__name__)
# This minimal list of model architectures is smaller than the total list of
# supported models. The intention is that in the "typical" regression testing
# scenario, we only test initializing these models. This subset was chosen
# to include representative examples of model varieties/workloads (conditional
# generation, sequence classification, causal LM, ranking, chat, reward model,
# multimodal, geospatial, voice, embedding, MTP)
MINIMAL_MODEL_ARCH_LIST = [
"LlavaForConditionalGeneration",
"Llama4ForConditionalGeneration",
"BertForSequenceClassification",
"Gemma3nForCausalLM",
"JinaVLForRanking",
"InternVLChatModel",
"InternLM2ForRewardModel",
"TransformersMultiModalForCausalLM",
"Terratorch",
"UltravoxModel",
"DeepSeekMTPModel",
"XLMRobertaModel",
]
# This list is the complement of the minimal list above. The intention is that
# this list of models is only tested in a "special case" i.e. most PRs should
# not test these models
OTHER_MODEL_ARCH_LIST = set(HF_EXAMPLE_MODELS.get_supported_archs()) - set(
MINIMAL_MODEL_ARCH_LIST
)
@pytest.fixture(autouse=True)
def cleanup_gpu_memory_between_tests() -> Generator[None, None, None]:
"""Reclaim GPU memory left over by each test's forked subprocess.
These tests are known to leave behind GPU memory (see the comment at
``gpu_memory_utilization=0.80`` below). Each parametrized case builds an
LLM in a forked subprocess whose VRAM is only reclaimed asynchronously
when the subprocess exits, so the leftovers can OOM a later test in this
module (observed with the heavier multimodal archs, e.g.
InternVLChatModel). Mirrors ``clean_gpu_memory_between_tests`` in the
top-level conftest, but unconditionally for this module.
"""
yield
gc.collect()
if not torch.cuda.is_available():
return
torch.accelerator.empty_cache()
if current_platform.is_rocm() or current_platform.is_xpu():
wait_for_memory_to_settle()
return
try:
wait_for_gpu_memory_to_clear(
devices=list(range(torch.accelerator.device_count())),
threshold_ratio=0.1,
)
except Exception as e:
# NVML is not usable on MIG slices (raises NVMLError_NoPermission);
# cleanup is best-effort, so never fail a test over it.
logger.info("Failed to clean GPU memory: %s", e)
@create_new_process_for_each_test()
def can_initialize(
model_arch: str, monkeypatch: pytest.MonkeyPatch, EXAMPLE_MODELS: HfExampleModels
):
"""The reason for using create_new_process_for_each_test is to avoid
the WARNING:
"We must use the 'spawn' multiprocessing start method. Overriding
VLLM_WORKER_MULTIPROC_METHOD to 'spawn'."
The spawn process causes the _initialize_kv_caches_v1 function below to
become ineffective.
"""
model_info = EXAMPLE_MODELS.get_hf_info(model_arch)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
hf_overrides_fn = partial(
dummy_hf_overrides,
model_arch=model_arch,
exist_overrides=model_info.hf_overrides,
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
)
# Avoid calling model.forward()
def _initialize_kv_caches_v1(self, vllm_config):
kv_cache_specs = self.model_executor.get_kv_cache_specs()
layout = resolve_kv_cache_layout(
vllm_config,
self.model_executor.get_supported_kv_cache_layouts(),
[spec for worker_specs in kv_cache_specs for spec in worker_specs.values()],
)
self.model_executor.set_kv_cache_layout(layout.name)
kv_cache_configs = get_kv_cache_configs(
vllm_config,
kv_cache_specs,
[10 * GiB_bytes],
)
scheduler_kv_cache_config = generate_scheduler_kv_cache_config(kv_cache_configs)
vllm_config.cache_config.num_gpu_blocks = scheduler_kv_cache_config.num_blocks
kv_cache_groups = scheduler_kv_cache_config.kv_cache_groups
if kv_cache_groups:
vllm_config.cache_config.block_size = min(
g.kv_cache_spec.block_size for g in kv_cache_groups
)
vllm_config.validate_block_size()
return scheduler_kv_cache_config
if model_arch == "MoonshotKimiaForCausalLM":
pytest.skip(
"Kimi-Audio requires SpeechToTextConfig "
"which is not configured in test environment"
)
if model_arch == "Terratorch":
import importlib.util
if importlib.util.find_spec("terratorch") is None:
pytest.skip(
"terratorch is not installed; "
"temporarily skipped while PyPI has `lightning` quarantined "
"(see #41376)"
)
if model_arch in ["DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"]:
from vllm.platforms import current_platform
capability = current_platform.get_device_capability()
if capability and capability.major < 9:
pytest.skip(
f"DeepseekV32 requires Hopper (9.0+) or Blackwell (10.0+) "
f"for FLASHMLA_SPARSE backend. Current device has compute "
f"capability {capability.major}.{capability.minor}"
)
if model_arch == "DeepseekV4ForConditionalGeneration":
from vllm.platforms import current_platform
if not (current_platform.is_cuda() or current_platform.is_rocm()):
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
with (
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
monkeypatch.context() as m,
):
if requires_spawn_multiprocessing() or model_arch == "Glm5NextForCausalLM":
# The EngineCore subprocess re-imports the class and does not
# inherit the KV-cache patch above, so it OOMs. Run in-process
# so the patch applies.
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
# FIXME: A hack to bypass FA3 assertion because our CI's L4 GPU
# has cc==8.9 which hasn't supported FA3 yet. Remove this hack when
# L4 supports FA3.
# Step1ForCausalLM requires TRITON_ATTN for use_alibi_sqrt support.
attention_config = (
{"backend": "TRITON_ATTN"}
if model_arch in ("GptOssForCausalLM", "Step1ForCausalLM")
else None
)
if model_arch == "WhisperForConditionalGeneration":
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
kwargs = {}
if not model_info.enable_prefix_caching:
kwargs["enable_prefix_caching"] = False
LLM(
model_info.default,
tokenizer=model_info.tokenizer,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
enforce_eager=model_info.enforce_eager,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
dtype=model_info.dtype,
speculative_config={
"model": model_info.speculative_model,
"method": model_info.speculative_method,
"num_speculative_tokens": 1,
}
if model_info.speculative_model
else None,
trust_remote_code=model_info.trust_remote_code,
max_model_len=model_info.max_model_len,
max_num_batched_tokens=model_info.max_num_batched_tokens,
# these tests seem to produce leftover memory
gpu_memory_utilization=0.80,
load_format="dummy",
model_impl="transformers"
if model_arch in _TRANSFORMERS_BACKEND_MODELS
else "vllm",
hf_overrides=hf_overrides_fn,
max_num_seqs=model_info.max_num_seqs,
attention_config=attention_config,
**kwargs,
)
@pytest.mark.parametrize("model_arch", MINIMAL_MODEL_ARCH_LIST)
def test_can_initialize_small_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
"""Test initializing small subset of supported models."""
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
@pytest.mark.parametrize("model_arch", OTHER_MODEL_ARCH_LIST)
def test_can_initialize_large_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
"""Test initializing large subset of supported models.
This test covers the complement of the tests covered in the "small subset"
test.
"""
if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel"):
from vllm.platforms import current_platform
if current_platform.is_rocm():
pytest.skip("HY V4 ROCm initialization requires #54405")
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
@pytest.mark.parametrize("model_arch", AUTO_EXAMPLE_MODELS.get_supported_archs())
def test_implicit_converted_models(model_arch: str, monkeypatch: pytest.MonkeyPatch):
can_initialize(model_arch, monkeypatch, AUTO_EXAMPLE_MODELS)
-237
View File
@@ -1,237 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import warnings
import pytest
import torch.cuda
from vllm.model_executor.models import (
is_pooling_model,
is_text_generation_model,
supports_multimodal,
)
from vllm.model_executor.models.adapters import (
as_embedding_model,
as_seq_cls_model,
)
from vllm.model_executor.models.registry import (
_MULTIMODAL_MODELS,
_SPECULATIVE_DECODING_MODELS,
_TEXT_GENERATION_MODELS,
ModelRegistry,
_LazyRegisteredModel,
)
from vllm.platforms import current_platform
from ..utils import create_new_process_for_each_test
from .registry import HF_EXAMPLE_MODELS
@pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
def test_registry_imports(model_arch):
# Skip if transformers version is incompatible
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
if model_arch == "Terratorch":
import importlib.util
if importlib.util.find_spec("terratorch") is None:
pytest.skip(
"terratorch is not installed; "
"temporarily skipped while PyPI has `lightning` quarantined "
"(see #41376)"
)
# DSpark draft model is supported on CUDA and ROCm; stubbed to None on XPU.
if model_arch == "DSparkDraftModel" and not (
current_platform.is_cuda() or current_platform.is_rocm()
):
pytest.skip("DSparkDraftModel is only supported on CUDA and ROCm")
if model_arch in ("Dots3NoteForCausalLM", "Dots3NoteMTPModel") and not (
current_platform.is_cuda()
):
pytest.skip("Dots3 NOTE is only supported on CUDA")
if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel") and not (
current_platform.is_cuda() or current_platform.is_rocm()
):
pytest.skip("HY V4 is only supported on CUDA and ROCm")
if (
model_arch == "DeepseekV4ForConditionalGeneration"
and not current_platform.is_cuda_alike()
):
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
# Ensure all model classes can be imported successfully
model_cls = ModelRegistry._try_load_model_cls(model_arch)
assert model_cls is not None
if model_arch in _SPECULATIVE_DECODING_MODELS:
return # Ignore these models which do not have a unified format
if model_arch in _TEXT_GENERATION_MODELS or model_arch in _MULTIMODAL_MODELS:
assert is_text_generation_model(model_cls)
# All vLLM models should be convertible to a pooling model
assert is_pooling_model(as_seq_cls_model(model_cls))
assert is_pooling_model(as_embedding_model(model_cls))
if model_arch in _MULTIMODAL_MODELS:
assert supports_multimodal(model_cls)
@pytest.mark.parametrize(
"model_arch,is_mm,score_type",
[
("LlamaForCausalLM", False, "bi-encoder"),
("LlavaForConditionalGeneration", True, "bi-encoder"),
("DeepseekV41ForCausalLM", True, "bi-encoder"),
("BertForSequenceClassification", False, "cross-encoder"),
("RobertaForSequenceClassification", False, "cross-encoder"),
("XLMRobertaForSequenceClassification", False, "cross-encoder"),
("GteNewModel", False, "bi-encoder"),
("GteNewForSequenceClassification", False, "cross-encoder"),
("HF_ColBERT", False, "late-interaction"),
],
)
def test_registry_model_property(model_arch, is_mm, score_type):
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
assert model_info is not None
assert model_info.supports_multimodal is is_mm
assert model_info.score_type == score_type
@pytest.mark.parametrize(
"model_arch,is_pp",
[
("DeepseekV2ForCausalLM", True),
("Qwen2VLForConditionalGeneration", True),
],
)
def test_registry_is_pp(model_arch, is_pp):
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
assert model_info is not None
assert model_info.supports_pp is is_pp
@pytest.mark.parametrize(
"model_arch,supported",
[
("Qwen4ExpForCausalLM", True),
("Qwen4ExpForConditionalGeneration", True),
("Qwen4ExpMTP", True),
("LlamaForCausalLM", False),
],
)
def test_registry_supports_hc_sp(model_arch, supported):
"""HC SP is opt-in for each model implementation, including MTP."""
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
assert model_info is not None
assert model_info.supports_hc_sp is supported
@create_new_process_for_each_test()
@pytest.mark.parametrize(
"model_arch",
["LlavaForConditionalGeneration", "Qwen2VLForConditionalGeneration"],
)
def test_registry_inspect_does_not_init_cuda(model_arch):
"""Inspecting a model must not import it: these architectures initialize
CUDA at import time, so the inspection path has to stay out-of-process."""
if not current_platform.is_cuda_alike():
pytest.skip("Requires a CUDA-like platform")
assert ModelRegistry._try_inspect_model_cls(model_arch) is not None
assert not torch.cuda.is_initialized()
ModelRegistry._try_load_model_cls(model_arch)
if not torch.cuda.is_initialized():
warnings.warn(
"This model no longer initializes CUDA on import. "
"Please test using a different one.",
stacklevel=2,
)
@pytest.mark.parametrize(
"model_arch,supported",
[
# ReplaySSM is opt-in per model.
("NemotronHForCausalLM", True),
("KimiLinearForCausalLM", not current_platform.is_rocm()),
("KimiK3ForConditionalGeneration", not current_platform.is_rocm()),
("Mamba2ForCausalLM", False),
("Zamba2ForCausalLM", False),
],
)
def test_registry_supports_replayssm(model_arch, supported):
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
assert model_info is not None
assert model_info.supports_replayssm is supported
def test_lazy_modelinfo_package_hash_includes_submodules(tmp_path):
package_dir = tmp_path / "model_package"
package_dir.mkdir()
init_file = package_dir / "__init__.py"
init_file.write_text("from .model import Model\n", encoding="utf-8")
model_file = package_dir / "model.py"
model_file.write_text("class Model: pass\n", encoding="utf-8")
first_hash = _LazyRegisteredModel._get_modelinfo_module_hash(init_file)
model_file.write_text("class Model:\n supports_pp = True\n", encoding="utf-8")
second_hash = _LazyRegisteredModel._get_modelinfo_module_hash(init_file)
assert first_hash != second_hash
def test_lazy_modelinfo_package_attempts_cache_load(monkeypatch):
cached_model_info = object()
loaded_hashes = []
def fake_load_cache(self, module_hash):
loaded_hashes.append(module_hash)
return cached_model_info
monkeypatch.setattr(
_LazyRegisteredModel,
"_load_modelinfo_from_cache",
fake_load_cache,
)
monkeypatch.setattr(
"vllm.model_executor.models.registry._run_in_subprocess",
lambda _: pytest.fail("Package-backed model should use the cache path"),
)
registered_model = _LazyRegisteredModel(
module_name="vllm.model_executor.models.transformers",
class_name="TransformersForCausalLM",
)
result = registered_model.inspect_model_cls()
assert result is cached_model_info
assert len(loaded_hashes) == 1
assert loaded_hashes[0]
def test_hf_registry_coverage():
untested_archs = (
ModelRegistry.get_supported_archs() - HF_EXAMPLE_MODELS.get_supported_archs()
)
assert not untested_archs, (
"Please add the following architectures to "
f"`tests/models/registry.py`: {untested_archs}"
)
-241
View File
@@ -1,241 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import regex as re
import torch
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
_merge_multimodal_embeddings,
)
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
class ModuleWithBatchNorm(torch.nn.Module):
def __init__(self):
super().__init__()
self.bn = torch.nn.BatchNorm1d(2)
def forward(self, x):
return self.bn(x)
class ModuleWithNestedBatchNorm(torch.nn.Module):
def __init__(self):
super().__init__()
self.nested_mod = ModuleWithBatchNorm()
def forward(self, x):
return self.nested_mod(x)
@pytest.mark.cpu_test
def test_module_with_batchnorm_can_load():
"""Ensure the auto weight loader can load batchnorm stats."""
mod = ModuleWithBatchNorm()
# Run some data through the module with batchnorm
mod(torch.Tensor([[1, 2], [3, 4]]))
# Try to load the weights to a new instance
def weight_generator():
yield from mod.state_dict().items()
new_mod = ModuleWithBatchNorm()
assert not torch.all(new_mod.bn.running_mean == mod.bn.running_mean)
assert not torch.all(new_mod.bn.running_var == mod.bn.running_var)
assert new_mod.bn.num_batches_tracked.item() == 0
loader = AutoWeightsLoader(new_mod)
loader.load_weights(weight_generator())
# Ensure the stats are updated
assert torch.all(new_mod.bn.running_mean == mod.bn.running_mean)
assert torch.all(new_mod.bn.running_var == mod.bn.running_var)
assert new_mod.bn.num_batches_tracked.item() == 1
@pytest.mark.cpu_test
def test_module_with_child_containing_batchnorm_can_autoload():
"""Ensure the auto weight loader can load nested modules batchnorm stats."""
mod = ModuleWithNestedBatchNorm()
# Run some data through the module with batchnorm
mod(torch.Tensor([[1, 2], [3, 4]]))
# Try to load the weights to a new instance
def weight_generator():
yield from mod.state_dict().items()
new_mod = ModuleWithNestedBatchNorm()
assert not torch.all(
new_mod.nested_mod.bn.running_mean == mod.nested_mod.bn.running_mean
)
assert not torch.all(
new_mod.nested_mod.bn.running_var == mod.nested_mod.bn.running_var
)
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 0
loader = AutoWeightsLoader(new_mod)
loader.load_weights(weight_generator())
# Ensure the stats are updated
assert torch.all(
new_mod.nested_mod.bn.running_mean == mod.nested_mod.bn.running_mean
)
assert torch.all(new_mod.nested_mod.bn.running_var == mod.nested_mod.bn.running_var)
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 1
VOCAB_SIZE = 16
HIDDEN_SIZE = 2
class ModuleWithTiedWeights(torch.nn.Module):
"""Mimics how models tie `lm_head` to the input embeddings."""
def __init__(self, tie: bool):
super().__init__()
self.model = torch.nn.Module()
self.model.embed_tokens = VocabParallelEmbedding(VOCAB_SIZE, HIDDEN_SIZE)
self.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
if tie:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
def make_embedding_weights(value: float) -> torch.Tensor:
return torch.full((VOCAB_SIZE, HIDDEN_SIZE), value)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
@pytest.mark.parametrize("tie", [True, False])
def test_module_skip_tied_weights(tie: bool):
"""Tied weights must be loaded once, under the first of their names."""
mod = ModuleWithTiedWeights(tie)
weights = [
("model.embed_tokens.weight", make_embedding_weights(1.0)),
("lm_head.weight", make_embedding_weights(2.0)),
]
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
if tie:
assert loaded == {"model.embed_tokens.weight"}
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 1.0)
else:
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 2.0)
@pytest.mark.cpu_test
@pytest.mark.usefixtures("dist_init")
def test_module_skip_tied_weights_without_canonical():
"""Skipping a tied weight must not leave the shared weight uninitialized."""
mod = ModuleWithTiedWeights(tie=True)
weights = [("lm_head.weight", make_embedding_weights(2.0))]
with pytest.raises(ValueError, match="model.embed_tokens.weight"):
AutoWeightsLoader(mod).load_weights(iter(weights))
class ModuleWithSharedParam(torch.nn.Module):
"""Mimics an MoE router shared between the MLP and its fused experts."""
def __init__(self):
super().__init__()
self.experts = torch.nn.Module()
self.experts.gate = torch.nn.Linear(2, 2, bias=False)
self.gate = self.experts.gate
@pytest.mark.cpu_test
def test_module_load_shared_params_that_are_not_tied_embeddings():
"""Only tied embeddings are skipped; other shared params must still load."""
mod = ModuleWithSharedParam()
weights = [("gate.weight", torch.Tensor([[1, 2], [3, 4]]))]
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
assert loaded == {"gate.weight"}
assert torch.all(mod.gate.weight == torch.Tensor([[1, 2], [3, 4]]))
class raise_if_cuda_sync:
def __enter__(self):
self.previous_debug_mode = torch.cuda.get_sync_debug_mode()
torch.cuda.set_sync_debug_mode("error")
def __exit__(self, exception_type, exception_value, traceback):
torch.cuda.set_sync_debug_mode(self.previous_debug_mode)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
def test_merge_multimodal_embeddings_no_sync():
inputs_embeds = torch.zeros(
[5, 10], dtype=torch.bfloat16, device=f"{DEVICE_TYPE}:0"
)
multimodal_embeddings = [
torch.ones([3, 10], dtype=torch.bfloat16, device=f"{DEVICE_TYPE}:0")
]
is_multimodal = torch.tensor([True, False, True, True, False], device="cpu")
with raise_if_cuda_sync():
_merge_multimodal_embeddings(
inputs_embeds, multimodal_embeddings, is_multimodal
)
@pytest.mark.cpu_test
def test_get_rename_mapper_keeps_only_renames():
"""`None` means "do not load", which is meaningless to the consumers of
this mapper (LoRA name parsing, quantization config layer lists), and
applying it would silently shrink their lists."""
mapper = WeightsMapper(
orig_to_new_regex={re.compile(r"^drop_regex\."): None},
orig_to_new_substr={"drop_substr": None, "keep_substr": "kept"},
orig_to_new_stacked={".q_proj": (".qkv_proj", "q")},
orig_to_new_prefix={"drop_prefix.": None, "keep_prefix.": "kept."},
orig_to_new_suffix={".drop_suffix": None},
)
renames = mapper.get_rename_mapper()
assert renames.orig_to_new_regex == {}
assert renames.orig_to_new_substr == {"keep_substr": "kept"}
assert renames.orig_to_new_stacked == {}
assert renames.orig_to_new_prefix == {"keep_prefix.": "kept."}
assert renames.orig_to_new_suffix == {}
# Names the full mapper drops now survive unchanged.
for name in ("drop_regex.w", "drop_substr.w", "drop_prefix.w", "w.drop_suffix"):
assert mapper._map_name(name) is None
assert renames._map_name(name) == name
def test_weights_mapper_stacks_one_weight_into_several_shards():
"""One checkpoint tensor can feed more than one shard of a stacked
parameter, and a dropped name stays dropped rather than reappearing as a
shard of the stacked one."""
mapper = WeightsMapper(
orig_to_new_substr={"layers.1.k_proj.": None},
orig_to_new_stacked={".k_proj.": (".qkv_proj.", ["k", "v"])},
)
weight = torch.ones(2)
weights = [(f"layers.{i}.k_proj.weight", weight) for i in (0, 1)]
mapped = list(mapper.apply(weights))
assert [(name, w.shard_id) for name, w in mapped] == [
("layers.0.qkv_proj.weight", "k"),
("layers.0.qkv_proj.weight", "v"),
]
# Each shard needs its own tensor object, but they alias one allocation.
assert mapped[0][1] is not mapped[1][1]
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr() == weight.data_ptr()
File diff suppressed because it is too large Load Diff
-786
View File
@@ -1,786 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Test the functionality of the Transformers modeling backend."""
import ast
import contextlib
import os
import tempfile
from functools import partial
from types import SimpleNamespace
from typing import Any
import pytest
import torch
import torch.nn as nn
from transformers import AutoConfig, AutoModel, PreTrainedConfig
from vllm.config import ModelConfig, VllmConfig
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.fusers import AttentionFuser
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
from vllm.model_executor.models.utils import StageMissingLayer
from ...conftest import HfRunner, VllmRunner
from ...utils import multi_gpu_test, prep_prompts
from ..registry import HF_EXAMPLE_MODELS
from ..utils import check_embeddings_close, check_logprobs_close
@pytest.fixture(scope="function", autouse=True)
def enable_pickle(monkeypatch):
"""`LLM.apply_model` requires pickling a function."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
def get_model(arch: str) -> str:
model_info = HF_EXAMPLE_MODELS.get_hf_info(arch)
model_info.check_transformers_version(on_fail="skip")
return model_info.default
def count_modules(model, cls: type) -> int:
return sum(isinstance(m, cls) for m in model.modules())
def get_num_fused(model) -> tuple[int, int]:
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
)
glu = count_modules(model, MergedColumnParallelLinear)
qkv = count_modules(model, QKVParallelLinear)
return glu, qkv
def check_implementation(
runner_ref: type[HfRunner | VllmRunner],
runner_test: type[VllmRunner],
example_prompts: list[str],
model: str,
kwargs_ref: dict[str, Any] | None = None,
kwargs_test: dict[str, Any] | None = None,
num_fused: tuple[int, int] = (1, 1),
**kwargs,
):
if kwargs_ref is None:
kwargs_ref = {}
if kwargs_test is None:
kwargs_test = {}
max_tokens = 32
num_logprobs = 5
args = (example_prompts, max_tokens, num_logprobs)
with runner_test(model, **kwargs_test, **kwargs) as model_test:
model_config = model_test.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
num_layers = model_config.hf_config.get_text_config().num_hidden_layers
expected_glu, expected_qkv = num_fused
for num_glu, num_qkv in model_test.apply_model(get_num_fused):
assert num_glu == expected_glu * num_layers
assert num_qkv == expected_qkv * num_layers
tp_size = kwargs_test.get("tensor_parallel_size", 1)
from vllm.model_executor.layers.attention import Attention
counts = model_test.apply_model(partial(count_modules, cls=Attention))
assert counts == [num_layers] * tp_size
outputs_test = model_test.generate_greedy_logprobs(*args)
with runner_ref(model, **kwargs_ref) as model_ref:
if isinstance(model_ref, VllmRunner):
outputs_ref = model_ref.generate_greedy_logprobs(*args)
else:
outputs_ref = model_ref.generate_greedy_logprobs_limit(*args)
check_logprobs_close(
outputs_0_lst=outputs_ref,
outputs_1_lst=outputs_test,
name_0="ref",
name_1="test",
)
@pytest.mark.parametrize(
"model,model_impl,num_fused",
[
("meta-llama/Llama-3.2-1B-Instruct", "transformers", (1, 1)),
("hmellor/Ilama-3.2-1B", "auto", (1, 1)), # CUSTOM CODE
("allenai/OLMoE-1B-7B-0924", "transformers", (0, 1)), # MoE
],
) # trust_remote_code=True by default
def test_models(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: str,
model_impl: str,
num_fused: tuple[int, int],
) -> None:
check_implementation(
hf_runner,
vllm_runner,
example_prompts,
model,
num_fused=num_fused,
model_impl=model_impl,
)
def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None:
prompts, _, _ = prep_prompts(4, (800, 801))
kwargs_ref = {"max_model_len": 8192, "enforce_eager": True}
kwargs_test = {"model_impl": "transformers", **kwargs_ref}
check_implementation(
vllm_runner,
vllm_runner,
prompts,
model="hmellor/tiny-random-Gemma2ForCausalLM",
kwargs_ref=kwargs_ref,
kwargs_test=kwargs_test,
)
def get_sinks(model) -> dict[int, torch.Tensor]:
"""The sink tensor each attention layer was handed, keyed by layer index."""
sinks = {}
for i, (prefix, fuser) in model.attention_fusers.items():
if (sink := fuser.sinks(model.get_submodule(prefix))) is not None:
sinks[i] = sink.float().cpu()
return sinks
def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None:
"""Learnable attention sinks must reach the attention layers.
Only the attention impl can apply them, so if they are not passed to
`Attention` they are dropped and every softmax is subtly wrong.
"""
from vllm.platforms import current_platform
if not current_platform.has_device_capability(90):
pytest.skip("Attention sinks need FlashAttention 3 or TRTLLM attention")
model = "tiny-random/gpt-oss-bf16"
with vllm_runner(
model, model_impl="transformers", max_model_len=1024, enforce_eager=True
) as model_test:
assert model_test.llm.llm_engine.model_config.using_transformers_backend()
sinks = model_test.apply_model(get_sinks)[0]
with hf_runner(model, dtype="bfloat16") as model_ref:
layers = model_ref.model.model.get_decoder().layers
expected = {i: layer.self_attn.sinks.float() for i, layer in enumerate(layers)}
assert sinks.keys() == expected.keys()
for i, sink in sinks.items():
torch.testing.assert_close(sink, expected[i].cpu())
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
args = (example_prompts, 32, 5)
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
with vllm_runner(
model, model_impl="transformers", trust_remote_code=False, **kwargs
) as model_test:
model_config = model_test.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
num_layers = model_config.hf_config.get_text_config().num_hidden_layers
from vllm.model_executor.layers.attention import MLAAttention
counts = model_test.apply_model(partial(count_modules, cls=MLAAttention))
assert counts == [num_layers]
outputs_test = model_test.generate_greedy_logprobs(*args)
with vllm_runner(model, model_impl="auto") as model_ref:
outputs_ref = model_ref.generate_greedy_logprobs(*args)
check_logprobs_close(
outputs_0_lst=outputs_ref,
outputs_1_lst=outputs_test,
name_0="native",
name_1="transformers",
)
@multi_gpu_test(num_gpus=2)
def test_distributed(
hf_runner: type[HfRunner],
vllm_runner: type[VllmRunner],
example_prompts,
):
kwargs = {"model_impl": "transformers", "tensor_parallel_size": 2}
check_implementation(
hf_runner,
vllm_runner,
example_prompts,
"meta-llama/Llama-3.2-1B-Instruct",
kwargs_test=kwargs,
)
@pytest.mark.parametrize(
"model, quantization_kwargs",
[
("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}),
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", {}),
],
)
@pytest.mark.parametrize("max_tokens", [32])
@pytest.mark.parametrize("num_logprobs", [5])
def test_quantization(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: str,
quantization_kwargs: dict[str, str],
max_tokens: int,
num_logprobs: int,
) -> None:
with vllm_runner(
model,
model_impl="auto",
enforce_eager=True,
**quantization_kwargs,
) as vllm_model:
vllm_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
)
with vllm_runner(
model,
model_impl="transformers",
enforce_eager=True,
**quantization_kwargs,
) as vllm_model:
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
transformers_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
)
check_logprobs_close(
outputs_0_lst=transformers_outputs,
outputs_1_lst=vllm_outputs,
name_0="transformers",
name_1="vllm",
)
@pytest.mark.parametrize(
"model",
[
# Layers live in `layers`
"Qwen/Qwen3-Embedding-0.6B",
# Layers live in `model.layers`
"meta-llama/Llama-3.2-1B-Instruct",
],
)
def test_embed_loading(vllm_runner, model):
with vllm_runner(
model,
max_model_len=1024,
enforce_eager=True,
runner="pooling",
model_impl="transformers",
) as model_test:
model_config = model_test.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
@pytest.mark.parametrize(
"arch", ["TransformersEmbeddingModel", "TransformersForSequenceClassification"]
)
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_pooling(
hf_runner,
vllm_runner,
example_prompts,
arch,
monkeypatch,
use_v2_model_runner,
):
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
model = get_model(arch)
vllm_kwargs = dict(max_model_len=None, model_impl="transformers")
hf_kwargs = dict()
if arch == "TransformersEmbeddingModel":
hf_kwargs["is_sentence_transformer"] = True
elif arch == "TransformersForSequenceClassification":
from transformers import AutoModelForSequenceClassification
hf_kwargs["auto_cls"] = AutoModelForSequenceClassification
# The example_prompts has ending "\n", for example:
# "Write a short story about a robot that dreams for the first time.\n"
# sentence_transformers will strip the input texts, see:
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
# This makes the input_ids different between hf_model and vllm_model.
# So we need to strip the input texts to avoid test failing.
example_prompts = [str(s).strip() for s in example_prompts]
with (
vllm_runner(model, **vllm_kwargs) as vllm_model,
hf_runner(model, **hf_kwargs) as hf_model,
):
model_config = vllm_model.llm.llm_engine.model_config
assert model_config.using_transformers_backend()
if arch == "TransformersEmbeddingModel":
vllm_outputs = vllm_model.embed(example_prompts)
hf_outputs = hf_model.encode(example_prompts)
elif arch == "TransformersForSequenceClassification":
vllm_outputs = vllm_model.classify(example_prompts)
hf_outputs = hf_model.classify(example_prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
)
VOCAB_SIZE = 64
PER_LAYER_VOCAB_SIZE = 32
NUM_POSITIONS = 16
HIDDEN_SIZE = 8
EMBED_SCALE = 3.0
class ScaledWordEmbedding(nn.Embedding):
"""Mirrors Transformers' `*ScaledWordEmbedding` classes."""
def __init__(
self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0
):
super().__init__(num_embeddings, embedding_dim, padding_idx)
self.scalar_embed_scale = embed_scale
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
def forward(self, input_ids):
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
class ComposedWordEmbedding(nn.Module):
"""Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it."""
def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0):
super().__init__()
self.embed = nn.Embedding(num_embeddings, embedding_dim)
self.embed_scale = embed_scale
def forward(self, input_ids):
return self.embed(input_ids) * self.embed_scale
@pytest.fixture
def tp_init():
"""Single rank tensor parallel state, so vLLM layers can be constructed."""
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from ...utils import ensure_current_vllm_config
fd, temp_file = tempfile.mkstemp()
os.close(fd)
try:
with ensure_current_vllm_config():
init_distributed_environment(
world_size=1,
rank=0,
distributed_init_method=f"file://{temp_file}",
local_rank=0,
backend=current_platform.dist_backend,
)
initialize_model_parallel(1, 1)
yield
cleanup_dist_env_and_memory()
finally:
with contextlib.suppress(OSError):
os.unlink(temp_file)
@pytest.fixture
def vpe(tp_init):
"""`VocabParallelEmbedding`, imported late so collection does not import vLLM."""
from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
return VocabParallelEmbedding
def replace(embedding):
"""Replace `embedding` and fill the new weights with recognisable values."""
from vllm.model_executor.models.transformers.utils import replace_embedding_class
new_embedding = replace_embedding_class(embedding)
for _, param in new_embedding.named_parameters():
param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape)
return new_embedding
def assert_scaled(vpe, module):
"""`module`'s output is its own unscaled output times `EMBED_SCALE`."""
input_ids = torch.arange(VOCAB_SIZE)
unscaled = vpe.forward(module, input_ids)
torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE)
def test_replace_plain_embedding(vpe):
"""A plain `nn.Embedding` is replaced outright, leaving no subclass behind."""
assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe
def test_replace_infers_shape_and_dtype(tp_init):
"""Shape and dtype come from the replaced module, not from the config."""
embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16)
new_embedding = replace(embedding)
assert new_embedding.num_embeddings == VOCAB_SIZE * 2
assert new_embedding.org_vocab_size == VOCAB_SIZE * 2
assert new_embedding.embedding_dim == HIDDEN_SIZE + 1
assert new_embedding.weight.dtype == torch.float16
def test_replace_inherited_embedding(vpe):
"""Subclasses keep their extra state and their scaled `forward`."""
new_embedding = replace(
ScaledWordEmbedding(
VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE
)
)
assert isinstance(new_embedding, vpe)
assert new_embedding.scalar_embed_scale == EMBED_SCALE
assert "embed_scale" in new_embedding._non_persistent_buffers_set
assert_scaled(vpe, new_embedding)
def test_replace_is_idempotent(tp_init):
"""A tied table is reached twice by the recursion, and must not be rebased twice."""
from vllm.model_executor.models.transformers.utils import replace_embedding_class
new_embedding = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=2))
again = replace_embedding_class(new_embedding)
assert again is new_embedding
assert type(again) is type(new_embedding)
class FakeVocabModel(nn.Module):
"""The embedding tables a model can hold, only one of which is a vocab table.
`embed_tokens_per_layer` mirrors Gemma 3n, whose per-layer table is sized by
`vocab_size_per_layer_input` rather than `vocab_size`.
"""
def __init__(self, per_layer: bool = False, composed: bool = False):
super().__init__()
embed_cls = ComposedWordEmbedding if composed else ScaledWordEmbedding
self.embed_tokens = embed_cls(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
if per_layer:
self.embed_tokens_per_layer = ScaledWordEmbedding(
PER_LAYER_VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE
)
# Not a vocab table: indexed by position, and read as a whole by models
# which interpolate it (e.g. `SiglipVisionEmbeddings`)
self.position_embedding = nn.Embedding(NUM_POSITIONS, HIDDEN_SIZE)
def get_input_embeddings(self):
return self.embed_tokens
def replace_vocab_embeddings(model, **config_kwargs):
"""Replace `model`'s vocab tables as `recursive_replace` would, and return them."""
from vllm.model_executor.models.transformers.utils import attrsetter
stub = nn.Module()
stub.model = model
stub.config = PreTrainedConfig(
vocab_size=VOCAB_SIZE, num_positions=NUM_POSITIONS, **config_kwargs
)
embeddings = Base._vocab_embeddings(stub)
replaced = []
for name, module in list(model.named_modules()):
if module in embeddings:
replaced.append(replace(module))
attrsetter(name)(model, replaced[-1])
return replaced
def test_only_vocab_tables_are_replaced(vpe):
"""Position embeddings share the `nn.Embedding` type, but not the treatment."""
model = FakeVocabModel()
assert replace_vocab_embeddings(model) == [model.embed_tokens]
assert isinstance(model.embed_tokens, vpe)
assert type(model.position_embedding) is nn.Embedding
def test_per_layer_vocab_table_is_replaced(vpe):
"""A second vocab table is found by its size, without a bespoke accessor."""
model = FakeVocabModel(per_layer=True)
replaced = replace_vocab_embeddings(
model, vocab_size_per_layer_input=PER_LAYER_VOCAB_SIZE
)
assert replaced == [model.embed_tokens, model.embed_tokens_per_layer]
assert all(isinstance(module, vpe) for module in replaced)
def test_composed_input_embeddings_are_replaced(vpe):
"""A wrapper is left alone; only the `nn.Embedding` it composes is replaced.
`CausalMixin` ties `lm_head` to whichever `VocabParallelEmbedding` it finds under
`get_input_embeddings()`, reading a `.weight` the wrapper does not have.
"""
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
model = FakeVocabModel(composed=True)
wrapper = model.embed_tokens
replaced = replace_vocab_embeddings(model)
assert model.embed_tokens is wrapper
assert replaced == [m for m in wrapper.modules() if isinstance(m, vpe)]
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
assert lm_head.tie_weights(replaced[0]).weight is replaced[0].weight
def test_missing_input_embeddings_are_skipped(tp_init):
"""Pipeline ranks without the embeddings have a `PPMissingLayer` in their place."""
from vllm.model_executor.models.utils import PPMissingLayer
model = FakeVocabModel()
model.embed_tokens = PPMissingLayer()
assert replace_vocab_embeddings(model) == []
MULTIMODAL_MODEL = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
class MarkingStub(SupportsMultiModal, nn.Module):
"""Just enough of the backend to exercise component marking."""
_mark_model_components = MultiModalMixin._mark_model_components
_find_encoder_classes = MultiModalMixin._find_encoder_classes
_from_config_kwargs = Base._from_config_kwargs
_pre_trained_model_classes = Base._pre_trained_model_classes
def build_marked_model(
image_limit: int, video_limit: int = 0, skip_tokenizer_init: bool = False
):
"""Build the HF model inside the marking context and return it."""
model_config = ModelConfig(
model=MULTIMODAL_MODEL,
model_impl="transformers",
limit_mm_per_prompt={"image": image_limit, "video": video_limit},
)
# Set after construction: building the config itself needs the tokenizer
model_config.skip_tokenizer_init = skip_tokenizer_init
stub = MarkingStub()
stub.config = AutoConfig.from_pretrained(MULTIMODAL_MODEL)
stub.model_config = model_config
vllm_config = VllmConfig(model_config=model_config)
with stub._mark_model_components(vllm_config), torch.device("meta"):
stub.model = AutoModel.from_config(**stub._from_config_kwargs)
return stub.model
@pytest.mark.parametrize(
("image_limit", "video_limit", "skipped"),
[(0, 0, True), (4, 0, False), (0, 4, False)],
)
def test_tower_weights_skipped_when_modality_disabled(
image_limit, video_limit, skipped
):
"""Only `--limit-mm-per-prompt image=0,video=0` drops the tower they share."""
vision_tower = build_marked_model(image_limit, video_limit).vision_tower
assert isinstance(vision_tower, StageMissingLayer) is skipped
def test_marking_skipped_without_tokenizer():
"""Marking needs the HF processor, which needs a tokenizer, so it is skipped.
The tower is built as normal rather than the model failing to load.
"""
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
assert not isinstance(vision_tower, StageMissingLayer)
NUM_LAYERS = 4
def build_model(model_type: str, **overrides) -> nn.Module:
"""A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
try:
config = AutoConfig.for_model(
model_type,
num_hidden_layers=NUM_LAYERS,
hidden_size=64,
intermediate_size=128,
num_attention_heads=4,
num_key_value_heads=2,
head_dim=16,
vocab_size=VOCAB_SIZE,
**overrides,
)
except ValueError:
pytest.skip(f"The installed transformers has no {model_type!r} model")
with torch.device("meta"):
return AutoModel.from_config(config)
ATTENTION_MODEL_TYPES = [
# The default `head_size**-0.5` happens to be right here
"llama",
# Scales the query by a learnable per-dim weight, so it declares 1.0
"gemma4_text",
# MLA, and declares yarn's `mscale`
"deepseek_v3",
]
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_dispatch_is_matched(model_type: str):
"""Exactly the decoder layers' attention modules match an `AttentionFuser`.
That match is what `recursive_replace` records, and so what
`_create_attention_instances` attaches vLLM's attention layer to.
"""
model = build_model(model_type)
matched = {
name
for name, module in model.named_modules()
if AttentionFuser.match(None, module) is not None
}
assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
def test_attention_layer_index_is_the_modules_own():
"""The layer served comes from the module, not from its position in the stack.
LongCat Flash gives each decoder layer two attention sublayers numbered
`2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
and the enclosing layer's position is not the index the KV cache is keyed by.
A vision tower is excluded separately, by `validate`: only the text config is
patched to dispatch to vLLM, so a tower's attention is left to Transformers.
"""
model = build_model("longcat_flash", num_layers=2)
text_config = model.config.get_text_config()
assert len(model.layers) * 2 == text_config.num_hidden_layers
attentions = [
model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
]
fusers = [AttentionFuser.match(None, attn) for attn in attentions]
assert all(fuser is not None for fuser in fusers)
assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
# Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
# config is not consulted, only the one the module was built with.
assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
text_config._attn_implementation = VLLM_ATTN_IMPL
assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
"""Models selected by the permissive registry fail clearly without dispatch."""
monkeypatch.setattr(
"vllm.model_executor.models.transformers.base.get_pp_indices",
lambda *_: (0, 1),
)
model = SimpleNamespace(
text_config=SimpleNamespace(num_hidden_layers=1),
pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
attention_fusers={},
_get_attn_cls=lambda: None,
)
with pytest.raises(
ValueError,
match="Layer 0 does not dispatch through the Transformers attention interface",
):
Base._create_attention_instances(model)
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
def test_attention_scale_is_the_declared_one(model_type: str):
"""The scale comes off the module, not from the `head_size**-0.5` default.
Two of the three model types declare something the default would silently
replace, so they are what makes this more than a tautology.
"""
attention = build_model(model_type).layers[0].self_attn
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == attention.scaling
def test_attention_scale_is_the_argument_not_the_attribute():
"""OPT applies the scale to the query itself, then declares a literal 1.0.
Its `self.scaling` is the `head_size**-0.5` it has already applied, so
reading the attribute rather than the argument the module hands the
interface would scale twice.
"""
model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
attention = model.get_submodule("decoder.layers.0.self_attn")
fuser = AttentionFuser.match(None, attention)
assert fuser is not None
assert fuser.scale(attention) == 1.0 != attention.scaling
def test_attention_scale_rejects_unresolvable_expression():
"""Unresolvable declared scales must not silently use the Llama default."""
class Attention(nn.Module):
head_dim = 16
fuser = AttentionFuser(
source_cls="Attention",
scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
)
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
fuser.scale(Attention())
class _ImageOnlyMRoPEModel:
"""`get_rope_index` without `video_grid_thw` or `**kwargs` (e.g. HunYuanVL)."""
def get_rope_index(self, input_ids, image_grid_thw):
seq_len = input_ids.shape[-1]
positions = torch.arange(seq_len).view(1, 1, -1).expand(4, 1, -1)
return positions, torch.tensor([0])
def test_get_mrope_input_positions_omits_unsupported_grid_kwargs():
"""Optional grids the model can't accept must not be passed when empty."""
mixin = SimpleNamespace(model=_ImageOnlyMRoPEModel())
positions, delta = MultiModalMixin.get_mrope_input_positions(mixin, [1, 2, 3], [])
assert positions.shape == (4, 3)
assert delta == 0
-611
View File
@@ -1,611 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import warnings
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any
import torch
import torch.nn.functional as F
from transformers import PreTrainedConfig
from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption
from vllm.config.pooler import SequencePoolingType, TokenPoolingType
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers import cached_tokenizer_from_config
from .. import ci_envs
from .registry import HF_EXAMPLE_MODELS
TokensText = tuple[list[int], str]
def check_outputs_equal(
*,
outputs_0_lst: Sequence[TokensText],
outputs_1_lst: Sequence[TokensText],
name_0: str,
name_1: str,
):
"""Compare the two sequences generated by different models,
which should be equal.
"""
assert len(outputs_0_lst) == len(outputs_1_lst)
for prompt_idx, (outputs_0, outputs_1) in enumerate(
zip(outputs_0_lst, outputs_1_lst)
):
output_ids_0, output_str_0 = outputs_0
output_ids_1, output_str_1 = outputs_1
# The text and token outputs should exactly match
fail_msg = (
f"Test{prompt_idx}:"
f"\n{name_0}:\t{output_str_0!r}"
f"\n{name_1}:\t{output_str_1!r}"
)
assert output_str_0 == output_str_1, fail_msg
assert output_ids_0 == output_ids_1, fail_msg
# Representation of generated sequence as a tuple of
# * Token ID list
# * String
# * List of top sample logprobs for each sampled token
#
# Assumes prompt logprobs were not requested.
TokensTextLogprobs = tuple[
list[int], str, list[dict[int, float]] | SampleLogprobs | None
]
# Allow for tokens to be represented as str's rather than IDs;
# tuple of
# * Token string representations list
# * String
# * Optional list of top sample logprobs for each sampled token
#
# Assumes prompt logprobs were not requested.
TextTextLogprobs = tuple[
list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None
]
# Representation of generated sequence as a tuple of
# * Token ID list
# * String
# * Optional list of top sample logprobs for each sampled token
# * Optional list of top prompt logprobs for each prompt token
#
# Allows prompt logprobs to be requested.
TokensTextLogprobsPromptLogprobs = tuple[
list[int],
str,
list[dict[int, float]] | SampleLogprobs | None,
list[dict[int, float] | None] | PromptLogprobs | None,
]
def check_logprobs_close(
*,
outputs_0_lst: Sequence[
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
],
outputs_1_lst: Sequence[
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
],
name_0: str,
name_1: str,
num_outputs_0_skip_tokens: int = 0,
warn_on_mismatch: bool = True,
always_check_logprobs: bool = False,
) -> None:
"""Compare the logprobs of two sequences generated by different models,
which should be similar but not necessarily equal.
How sample logprobs are compared:
* `always_check_logprobs == True`: set of highest-logprob token ids
must match between seq0 and seq1 at all sampled token offsets
* `always_check_logprobs == False`: highest-logprob token ids are
only compared at sampled token offsets for which generated token
ids don't match
Prompt logprobs must be provided either for both input sequences, or
for neither. If prompt logprobs are provided, then highest-logprob
prompt token ids must match between seq0 and seq1 at all prompt token
offsets.
Args:
outputs_0_lst: First sequence to compare
outputs_0_lst: Second sequence to compare
name_0: sequence #0 name
name_1: sequence #1 name
num_outputs_0_skip_tokens: If > 0, specifies the number of initial
sequence #0 tokens & logprobs to discard
before comparison, i.e. all
of sequence #1 will be compared to
sequence #0 beginning at index
num_outputs_0_skip_tokens
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
mismatch between the two sequences
always_check_logprobs: If true, check logprobs even when tokens match
"""
assert len(outputs_0_lst) == len(outputs_1_lst)
# Loop through responses to each prompt.
for prompt_idx, (outputs_0, outputs_1) in enumerate(
zip(outputs_0_lst, outputs_1_lst)
):
assert len(outputs_0) == len(outputs_1)
if len(outputs_0) == 3:
assert len(outputs_1) == 3
# Break out tokens, text & sample logprobs
# (prompt logprobs were not provided)
output_ids_0, output_str_0, logprobs_0 = outputs_0
output_ids_1, output_str_1, logprobs_1 = outputs_1
elif len(outputs_0) == 4:
assert len(outputs_1) == 4
# Break out tokens, text, sample logprobs & prompt logprobs
(
output_ids_0,
output_str_0,
logprobs_0,
prompt_logprobs_0,
) = outputs_0
(
output_ids_1,
output_str_1,
logprobs_1,
prompt_logprobs_1,
) = outputs_1
# Test prompt logprobs closeness
if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:
# Both sequences' prompt logprobs lists are not `None`
# (although individual list elements may be `None`);
# for each token's logprobs:
for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
zip(prompt_logprobs_0, prompt_logprobs_1)
):
fail_msg = (
f"Prompt logprobs test:"
f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}"
)
if logprobs_elem_0 is None:
# If the seq 0 token's logprobs are `None`,
# the seq 1 token's logprobs must be `None`
assert logprobs_elem_1 is None, fail_msg
else:
# If the seq 0 token's logprobs are not `None`,
# the seq 1 token's logprobs must not be `None`
assert logprobs_elem_1 is not None, fail_msg
# Logprobs check: top-k token choices must be the same
assert set(logprobs_elem_0.keys()) == set(
logprobs_elem_1.keys()
), fail_msg
else:
# Both sequence logprobs lists must be `None`
fail_msg = (
f"Prompt logprobs test:"
f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}"
)
assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg
else:
raise ValueError(
f"Outputs tuple must have 3 or 4 elements but "
f"{len(outputs_0)} elements were provided: "
f"{outputs_0}"
)
if logprobs_0 is None:
logprobs_0 = [None] * len(output_ids_0)
if logprobs_1 is None:
logprobs_1 = [None] * len(output_ids_1)
# Skip specified number of initial sequence #0 tokens
# & logprobs, leaving output text as-is for simplicity
# (text mismatches may generate warnings but do not
# cause the test to fail.)
if num_outputs_0_skip_tokens < 0:
raise ValueError("num_outputs_0_skip_tokens must be non-negative")
output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]
# Loop through generated tokens.
for idx, (output_id_0, output_id_1) in enumerate(
zip(output_ids_0, output_ids_1)
):
is_tok_mismatch = output_id_0 != output_id_1
# If generated tokens don't match
# or it is desired to always check logprobs,
# then
if is_tok_mismatch or always_check_logprobs:
logprobs_elem_0 = logprobs_0[idx]
logprobs_elem_1 = logprobs_1[idx]
# Each predicted token must be in top N logprobs of the other
fail_msg = (
f"Test{prompt_idx}:"
f"\nMatched tokens:\t{output_ids_0[:idx]}"
f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}"
f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}"
)
assert logprobs_elem_0 is not None, fail_msg
assert logprobs_elem_1 is not None, fail_msg
assert output_id_0 in logprobs_elem_1, fail_msg
assert output_id_1 in logprobs_elem_0, fail_msg
if warn_on_mismatch and is_tok_mismatch:
with warnings.catch_warnings():
# This ensures that repeated warnings are shown
# in the output, not just the first occurrence
warnings.simplefilter("always")
warnings.warn(fail_msg, stacklevel=2)
# Break out since sequences will now diverge.
break
else:
if output_str_0 != output_str_1 and warn_on_mismatch:
# The token outputs exactly match,
# so the text outputs should exactly match as well
fail_msg = (
f"Test{prompt_idx}:"
f"\n{name_0}:\t{output_str_0!r}"
f"\n{name_1}:\t{output_str_1!r}"
)
with warnings.catch_warnings():
# This ensures that repeated warnings are shown
# in the output, not just the first occurrence
warnings.simplefilter("always")
warnings.warn(fail_msg, stacklevel=2)
def build_model_context(
model_id: str,
runner: RunnerOption = "auto",
dtype: ModelDType = "auto",
model_config_kwargs: dict[str, Any] | None = None,
mm_processor_kwargs: dict[str, Any] | None = None,
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None,
mm_processor_cache_gb: int = 0,
):
"""Creates an InputProcessingContext for a given model.
Args:
model_id: ID of the model being considered.
mm_processor_kwargs: optional processor kwargs for to be leveraged
in the input processor, mapper, dummy data creation, etc.
limit_mm_per_prompt: Multimodal limits.
Returns:
InputProcessingContext for the model being considered.
"""
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
model_info.check_available_online(on_fail="skip")
model_info.check_transformers_version(
on_fail="skip",
check_max_version=False,
check_version_reason="vllm",
)
model_config_kwargs = model_config_kwargs or {}
limit_mm_per_prompt = {
modality: dict(limit) if isinstance(limit, Mapping) else limit
for modality, limit in (limit_mm_per_prompt or {}).items()
}
model_config = ModelConfig(
model_id,
runner=runner,
tokenizer=model_info.tokenizer or model_id,
tokenizer_mode=model_info.tokenizer_mode,
revision=model_info.revision,
trust_remote_code=model_info.trust_remote_code,
dtype=dtype,
seed=0,
mm_processor_kwargs=mm_processor_kwargs,
limit_mm_per_prompt=limit_mm_per_prompt,
mm_processor_cache_gb=mm_processor_cache_gb,
hf_overrides=model_info.hf_overrides,
skip_tokenizer_init=model_info.require_embed_inputs,
enable_prompt_embeds=model_info.require_embed_inputs,
enable_mm_embeds=model_info.require_embed_inputs,
enforce_eager=model_info.enforce_eager,
**model_config_kwargs,
)
return InputProcessingContext(
model_config,
tokenizer=cached_tokenizer_from_config(model_config),
)
def check_embeddings_close(
*,
embeddings_0_lst: Sequence[list[float]],
embeddings_1_lst: Sequence[list[float]],
name_0: str,
name_1: str,
tol: float = 1e-3,
) -> None:
assert len(embeddings_0_lst) == len(embeddings_1_lst)
for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
zip(embeddings_0_lst, embeddings_1_lst)
):
assert len(embeddings_0) == len(embeddings_1), (
f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}"
)
sim = F.cosine_similarity(
torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0
)
fail_msg = (
f"Test{prompt_idx}:"
f"\nCosine similarity: \t{sim:.4f}"
f"\n{name_0}:\t{embeddings_0[:16]!r}"
f"\n{name_1}:\t{embeddings_1[:16]!r}"
)
assert sim >= 1 - tol, fail_msg
def matryoshka_fy(tensor: torch.Tensor, dimensions: int):
tensor = torch.tensor(tensor)
tensor = tensor[..., :dimensions]
tensor = F.normalize(tensor, p=2, dim=1)
return tensor
def softmax(data):
if data.shape[-1] == 1:
return F.sigmoid(data)
else:
return F.softmax(data, dim=-1)
@dataclass
class ModelInfo:
name: str
revision: str | None = None
architecture: str = ""
dtype: str = "auto"
max_model_len: int | None = None
hf_dtype: str = "float32"
hf_overrides: dict[str, Any] | None = None
seq_pooling_type: SequencePoolingType | None = None
tok_pooling_type: TokenPoolingType | None = None
attn_type: AttnTypeStr | None = None
is_prefix_caching_supported: bool | None = None
is_chunked_prefill_supported: bool | None = None
enable_test: bool = True
@dataclass
class EmbedModelInfo(ModelInfo):
mteb_score: float | None = None
# Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL.
mteb_tol: float | None = None
is_matryoshka: bool = False
matryoshka_dimensions: list[int] | None = None
@dataclass
class RerankModelInfo(ModelInfo):
mteb_score: float | None = None
# Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL.
mteb_tol: float | None = None
chat_template_name: str | None = None
@dataclass
class GenerateModelInfo(ModelInfo):
hf_dtype: str = "auto"
hf_ppl: float | None = None
def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs):
# A model family has many models with the same architecture,
# and we don't need to test each one.
if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test:
import pytest
pytest.skip("Skipping test.")
# Allow vllm to test using the given dtype, such as float32
vllm_extra_kwargs = vllm_extra_kwargs or {}
vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype
# Allow vllm to test using hf_overrides
if model_info.hf_overrides is not None:
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
# Allow changing the head dtype used by vllm in tests
if ci_envs.VLLM_CI_HEAD_DTYPE is not None:
if "hf_overrides" not in vllm_extra_kwargs:
vllm_extra_kwargs["hf_overrides"] = {}
vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE
# Allow control over whether tests use enforce_eager
if ci_envs.VLLM_CI_ENFORCE_EAGER is not None:
vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER
return vllm_extra_kwargs
def dummy_hf_overrides(
hf_config: PreTrainedConfig,
*,
model_arch: str = "",
exist_overrides: dict[str, Any] | None = None,
use_original_num_layers: bool = False,
) -> PreTrainedConfig:
"""Dummy HF overrides function used to create dummy model
with only minimum nums of layer.
"""
# Copy because this helper is called more than once
# while loading config, and we `.pop()`
exist_overrides = (exist_overrides or {}).copy()
text_config_override = exist_overrides.pop("text_config", None)
hf_config.update(exist_overrides)
text_config = hf_config.get_text_config()
if text_config_override is not None:
# multimodal test models may override *some* text-model fields
text_config.update(text_config_override)
# Ensure at least 2 expert per group
# Since `grouped_topk` assumes top-2
n_group = getattr(text_config, "n_group", None)
# Kimi uses `num_expert_group` instead of `n_group`.
if n_group is None:
n_group = getattr(text_config, "num_expert_group", None)
# InternS1Pro uses `router_n_groups` instead of `n_group`.
if n_group is None:
n_group = getattr(text_config, "router_n_groups", None)
num_experts = n_group * 2 if n_group is not None else 2
# we use three layers for Gemma-3n to check
# both normal layer and kv_shared_layer
if use_original_num_layers:
# Use the original number of layers from the config
num_layers = getattr(text_config, "num_layers", 1)
num_hidden_layers = getattr(text_config, "num_hidden_layers", 1)
else:
# Use minimal layers for testing
num_layers = 1
num_hidden_layers = (
3
if model_arch
in (
"Gemma3nForConditionalGeneration",
"Gemma4ForCausalLM",
"Gemma4ForConditionalGeneration",
"Gemma4MTPModel",
"DiffusionGemmaForBlockDiffusion",
)
else 1
)
update_dict = {
"num_layers": num_layers,
# For Gemma-3n
"num_kv_shared_layers": 1,
}
_hf_config = hf_config
class DummyConfig:
hf_config = _hf_config
hf_text_config = text_config
# Keep architecture conversion on the default multimodal path; this
# helper only needs HF-derived fields, not deployment MM limits.
@staticmethod
def _supports_multimodal_for_mm_prefix() -> bool:
return True
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
# Only set MoE related config when the model has MoE layers.
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
if model_arch_config.num_experts > 0:
num_experts_per_tok = 2
if model_arch in (
"Llama4ForConditionalGeneration",
"Llama4ForCausalLM",
"EagleLlama4ForCausalLM",
):
num_experts_per_tok = 1
elif model_arch == "InternS1ProForConditionalGeneration":
assert n_group is not None
num_experts_per_tok = n_group
update_dict.update(
{
"num_experts": num_experts,
"num_experts_per_tok": num_experts_per_tok,
# Kimi uses `num_experts_per_token`.
"num_experts_per_token": num_experts_per_tok,
"num_local_experts": num_experts,
# Otherwise there will not be any expert layers
"first_k_dense_replace": 0,
# To avoid OOM on DeepSeek-V3
"n_routed_experts": num_experts,
}
)
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
# from num_layers for its dual-attention layers).
if model_arch not in (
"LongcatFlashForCausalLM",
"LongCatFlashMTPModel",
"LongcatFlashNgramForCausalLM",
):
update_dict["num_hidden_layers"] = num_hidden_layers
text_config.update(update_dict)
# Keep per-layer metadata consistent with the reduced layer count.
# HY V4 uses this to decide which sparse-attention indexers are shared.
if indexer_types := getattr(text_config, "indexer_types", None):
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
# Update n_layers and moe configs for Moondream3 model
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
text_config.update(
{
"n_layers": num_hidden_layers,
"moe_num_experts": num_experts,
"moe_experts_per_token": 2,
"moe_start_layer": num_hidden_layers,
}
)
if hasattr(hf_config, "vision_config"):
vision_config = hf_config.vision_config
vision_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
}
)
# Keep per-layer metadata consistent with the reduced layer count.
if layer_types := getattr(vision_config, "layer_types", None):
vision_config.update({"layer_types": layer_types[:1]})
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
vision_config.update({"enc_n_layers": 1})
# e.g.: ibm-granite/granite-speech-3.3-2b
if hasattr(hf_config, "encoder_config"):
hf_config.encoder_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
}
)
# e.g.: Qwen/Qwen2-Audio-7B-Instruct
# audio_config may exist but be None (e.g. audio-less Gemma4 variants).
if getattr(hf_config, "audio_config", None) is not None:
hf_config.audio_config.update(
{
"num_layers": 1,
"num_hidden_layers": 1,
"encoder_layers": 1,
}
)
return hf_config