[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,228 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import large_gpu_mark
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
# Models that require embedding scaling for prompt_embeds test
|
||||
EMBED_SCALING_MODELS = {
|
||||
"openbmb/MiniCPM4.1-8B",
|
||||
}
|
||||
|
||||
# This list contains the model that are using AITER kernel.
|
||||
# Skip model that are not using AITER tests.
|
||||
# When more AITER kernels are added, this list will not be
|
||||
# needed as all the models will be calling AITER kernels
|
||||
# in parts of the operators
|
||||
AITER_MODEL_LIST = [
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"openbmb/MiniCPM3-4B",
|
||||
"Qwen/Qwen2.5-0.5B-Instruct",
|
||||
"axolotl-ai-co/tiny-mixtral-30m",
|
||||
"Qwen/Qwen3-8B",
|
||||
]
|
||||
|
||||
|
||||
# @maybe_test_rocm_aiter
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
pytest.param(
|
||||
"bigscience/bloom-560m", # bloom - testing alibi slopes
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.slow_test,
|
||||
pytest.mark.cpu_model,
|
||||
],
|
||||
),
|
||||
pytest.param(
|
||||
"openai-community/gpt2", # gpt2
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
pytest.param("Milos/slovak-gpt-j-405M"), # gptj
|
||||
pytest.param("bigcode/tiny_starcoder_py"), # gpt_bigcode
|
||||
pytest.param("EleutherAI/pythia-70m"), # gpt_neox
|
||||
pytest.param(
|
||||
"google/gemma-1.1-2b-it", # gemma
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu_model,
|
||||
pytest.mark.slow_test,
|
||||
],
|
||||
),
|
||||
pytest.param(
|
||||
"google/gemma-2-2b-it", # test hybrid attention
|
||||
marks=[pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"zai-org/chatglm3-6b", # chatglm (text-only)
|
||||
),
|
||||
pytest.param(
|
||||
"meta-llama/Llama-3.2-1B-Instruct", # llama
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"openbmb/MiniCPM4.1-8B", # minicpm
|
||||
marks=[pytest.mark.core_model, large_gpu_mark(min_gb=48)],
|
||||
),
|
||||
pytest.param(
|
||||
"facebook/opt-125m", # opt
|
||||
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
|
||||
),
|
||||
pytest.param(
|
||||
"facebook/opt-350m", # opt with projected token embeddings
|
||||
),
|
||||
pytest.param(
|
||||
"microsoft/phi-2", # phi
|
||||
marks=[pytest.mark.core_model, pytest.mark.slow_test],
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen2.5-0.5B-Instruct", # qwen2
|
||||
marks=[
|
||||
pytest.mark.core_model,
|
||||
pytest.mark.cpu_model,
|
||||
pytest.mark.slow_test,
|
||||
],
|
||||
),
|
||||
pytest.param(
|
||||
"Qwen/Qwen3-8B", # qwen (text-only)
|
||||
),
|
||||
pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
|
||||
pytest.param("bigcode/starcoder2-3b"), # starcoder2
|
||||
pytest.param(
|
||||
"axolotl-ai-co/tiny-mixtral-30m", # mixtral
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
pytest.param("swiss-ai/Apertus-8B-Instruct-2509"), # apertus
|
||||
pytest.param(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", # hyperclovax
|
||||
marks=[large_gpu_mark(min_gb=32)],
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
@pytest.mark.parametrize("use_prompt_embeds", [True, False])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
use_rocm_aiter: bool,
|
||||
use_prompt_embeds: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
if use_rocm_aiter and (model in AITER_MODEL_LIST):
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
elif use_rocm_aiter and model not in AITER_MODEL_LIST:
|
||||
# Skip model that are not using AITER tests.
|
||||
# When more AITER kernels are added, this list will not be
|
||||
# needed as all the models will be calling AITER kernels
|
||||
# in parts of the operators
|
||||
pytest.skip(f"Skipping '{model}' model test with AITER kernel.")
|
||||
|
||||
if model == "bigcode/starcoder2-3b":
|
||||
# Replace example.txt's Test1 (an NL prompt) with a code prompt:
|
||||
# starcoder2-3b is a code model, so NL prompts give near-uniform
|
||||
# digit logits where HF<->vLLM bf16 drift can reorder top-K.
|
||||
example_prompts = list(example_prompts)
|
||||
example_prompts[1] = (
|
||||
"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - "
|
||||
)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
prompt_embeds: list[torch.Tensor] | None = [] if use_prompt_embeds else None
|
||||
|
||||
for prompt in example_prompts:
|
||||
token_ids = hf_model.tokenizer(prompt, return_tensors="pt").input_ids.to(
|
||||
hf_model.model.device
|
||||
)
|
||||
if prompt_embeds is not None:
|
||||
# Retained prompt values must not keep reference weights alive.
|
||||
with torch.no_grad():
|
||||
embed = hf_model.model.get_input_embeddings()(token_ids)
|
||||
|
||||
# MiniCPM models apply scale_emb to embeddings internally.
|
||||
# vLLM expects pre-scaled embeddings when using inputs_embeds.
|
||||
if model in EMBED_SCALING_MODELS:
|
||||
config = hf_model.model.config
|
||||
embed = embed * config.scale_emb
|
||||
|
||||
prompt_embeds.append(embed.squeeze(0))
|
||||
|
||||
vllm_kwargs = {}
|
||||
if (
|
||||
model == "bigscience/bloom-560m"
|
||||
and current_platform.is_device_capability_family(90)
|
||||
):
|
||||
# On SM90, the metadata builder otherwise selects FA3 AOT scheduling
|
||||
# before Bloom's ALiBi layers fall back to FA2. Pinning FA2 keeps the
|
||||
# builder and layer consistent and preserves the L4 test path.
|
||||
vllm_kwargs["attention_config"] = {"flash_attn_version": 2}
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
tokenizer_name=model_info.tokenizer or model,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
# Remove the effects of batch variance on ROCm since batch invariance
|
||||
# is not yet supported.
|
||||
# See: https://github.com/vllm-project/vllm/issues/27433
|
||||
max_num_seqs=1 if current_platform.is_rocm() else 2,
|
||||
enable_prompt_embeds=use_prompt_embeds,
|
||||
compilation_config={"cudagraph_capture_sizes": [1, 2]},
|
||||
**vllm_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
if prompt_embeds is not None:
|
||||
vllm_outputs_from_embeds = vllm_model.generate_greedy_logprobs(
|
||||
prompt_embeds, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
if prompt_embeds is not None:
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=vllm_outputs,
|
||||
outputs_1_lst=vllm_outputs_from_embeds,
|
||||
name_0="vllm",
|
||||
name_1="vllm_from_embeds",
|
||||
)
|
||||
|
||||
if use_rocm_aiter:
|
||||
# this is to ensure that vllm engine
|
||||
# has deallocated the memory before running the next
|
||||
# unit tests. On ROCm, when using AITER
|
||||
# the memory might not be deallocated completely
|
||||
# before running the next test case
|
||||
torch.accelerator.synchronize()
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Regression test for hybrid GDN/Mamba models under sleep -> wake.
|
||||
|
||||
Hybrid Mamba / gated-delta-net (GDN) models (e.g. Qwen3-Next) keep a
|
||||
persisted conv + recurrent state cache. With sleep mode (the RLHF reuse
|
||||
pattern: ``sleep()`` -> weight update -> ``wake_up()``) the state-cache tag is
|
||||
discarded on sleep and its device memory is re-created on wake. If a *new*
|
||||
sequence's state slot is consumed before being reset, the gated-delta-rule
|
||||
kernel faithfully propagates whatever is in that (now non-zeroed) memory; when
|
||||
it contains NaN/inf the output becomes NaN and ``argmax`` collapses every token
|
||||
to id 0 (which decodes to ``"!"``), giving ``reward=0`` / NaN log-probs in RL
|
||||
training.
|
||||
|
||||
This test sleeps and wakes a small hybrid GDN model and asserts that
|
||||
post-wake generation is neither degenerate (single repeated token) nor NaN.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
# Small Qwen3-Next (GDN) model already used by the hybrid model test-suite.
|
||||
MODEL = "tiny-random/qwen3-next-moe"
|
||||
|
||||
PROMPTS = [
|
||||
"The capital of France is",
|
||||
"Once upon a time,",
|
||||
"1, 2, 3, 4,",
|
||||
"Water is made of",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.hybrid_model
|
||||
def test_gdn_sleep_wake_no_stale_state():
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=32, logprobs=1)
|
||||
|
||||
# Keep the reserved fraction low. On some (notably ROCm/amdgpu) drivers the
|
||||
# VRAM discarded by ``sleep()`` is not returned to the free pool before
|
||||
# ``wake_up()`` re-creates it, so the woken allocation must coexist with the
|
||||
# not-yet-reclaimed one (~2x peak). A high ``gpu_memory_utilization`` then
|
||||
# OOMs in ``cuMemCreate`` on wake. The model is tiny, so a small fraction
|
||||
# still leaves ample KV/state cache while keeping the sleep/wake cycle well
|
||||
# within device memory.
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
max_model_len=1024,
|
||||
gpu_memory_utilization=0.4,
|
||||
trust_remote_code=True,
|
||||
)
|
||||
|
||||
# Warm generation before sleeping.
|
||||
llm.generate(PROMPTS, sampling_params)
|
||||
|
||||
# Default sleep offloads weights and DISCARDS the kv / GDN state cache;
|
||||
# wake_up re-creates that memory (fresh, not guaranteed zeroed).
|
||||
llm.sleep()
|
||||
llm.wake_up()
|
||||
|
||||
after = llm.generate(PROMPTS, sampling_params)
|
||||
|
||||
for output in after:
|
||||
completion = output.outputs[0]
|
||||
token_ids = list(completion.token_ids)
|
||||
assert token_ids, "empty generation after wake_up"
|
||||
# The bug collapses every token to a single id (e.g. 0 -> "!").
|
||||
assert len(set(token_ids)) > 1, (
|
||||
f"degenerate single-token output after wake_up: {token_ids[:16]}"
|
||||
)
|
||||
# NaN logits surface as NaN log-probs.
|
||||
for step_logprobs in completion.logprobs or []:
|
||||
for logprob in step_logprobs.values():
|
||||
assert logprob.logprob == logprob.logprob, (
|
||||
"NaN log-prob after wake_up (stale GDN state)"
|
||||
)
|
||||
@@ -1,928 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager, nullcontext
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
|
||||
|
||||
from ...utils import check_logprobs_close, check_outputs_equal
|
||||
|
||||
# Mark all tests as hybrid
|
||||
pytestmark = pytest.mark.hybrid_model
|
||||
|
||||
# NOTE: The first model in each list is taken as the primary model,
|
||||
# meaning that it will be used in all tests in this file
|
||||
# The rest of the models will only be tested by test_models
|
||||
|
||||
APC_MULTIPLY_BY = 300
|
||||
|
||||
SSM_MODELS = [
|
||||
"state-spaces/mamba-130m-hf",
|
||||
"tiiuae/falcon-mamba-tiny-dev",
|
||||
# mamba2-codestral in transformers is broken pending:
|
||||
# https://github.com/huggingface/transformers/pull/40861
|
||||
# "yujiepan/mamba2-codestral-v0.1-tiny-random",
|
||||
]
|
||||
|
||||
HYBRID_MODELS = [
|
||||
"ai21labs/Jamba-tiny-dev",
|
||||
"Zyphra/Zamba2-1.2B-instruct",
|
||||
"ibm-granite/granite-4.0-tiny-preview",
|
||||
"tiiuae/Falcon-H1-0.5B-Base",
|
||||
"LiquidAI/LFM2-1.2B",
|
||||
"tiny-random/qwen3-next-moe",
|
||||
]
|
||||
|
||||
FULL_CUDA_GRAPH_MODELS = [
|
||||
"ai21labs/Jamba-tiny-dev",
|
||||
"Zyphra/Zamba2-1.2B-instruct",
|
||||
]
|
||||
|
||||
FP32_STATE_MODELS = [
|
||||
"state-spaces/mamba-130m-hf",
|
||||
"Zyphra/Zamba2-1.2B-instruct",
|
||||
]
|
||||
|
||||
# Avoid OOM
|
||||
MAX_NUM_SEQS = 4
|
||||
|
||||
ATTN_BACKEND = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
|
||||
|
||||
|
||||
def _set_conv_state_layout(monkeypatch, layout: str) -> None:
|
||||
"""Set conv state layout env var and clear cache to pick up new value."""
|
||||
from vllm.model_executor.layers.mamba import mamba_utils
|
||||
|
||||
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", layout)
|
||||
mamba_utils.get_conv_state_layout.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", SSM_MODELS + HYBRID_MODELS)
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
gpu_memory_cleared,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
with hf_runner(model) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
|
||||
def test_batching(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
conv_state_layout: str,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
_set_conv_state_layout(monkeypatch, conv_state_layout)
|
||||
|
||||
for_loop_outputs = []
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
|
||||
) as vllm_model:
|
||||
for prompt in example_prompts:
|
||||
(single_output,) = vllm_model.generate_greedy_logprobs(
|
||||
[prompt], max_tokens, num_logprobs
|
||||
)
|
||||
for_loop_outputs.append(single_output)
|
||||
|
||||
batched_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=for_loop_outputs,
|
||||
outputs_1_lst=batched_outputs,
|
||||
name_0="for_loop_vllm",
|
||||
name_1="batched_vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [10])
|
||||
@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
|
||||
def test_chunked_prefill_with_parallel_sampling(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
conv_state_layout: str,
|
||||
) -> None:
|
||||
"""Tests chunked prefill in conjunction with n > 1.
|
||||
|
||||
In this case, prefill is populated with decoding tokens and
|
||||
we test that it doesn't fail.
|
||||
|
||||
This test might fail if cache is not allocated correctly for n > 1
|
||||
decoding steps inside a chunked prefill forward pass
|
||||
(where we have both prefill and decode together)
|
||||
"""
|
||||
_set_conv_state_layout(monkeypatch, conv_state_layout)
|
||||
|
||||
sampling_params = SamplingParams(n=3, temperature=1, seed=0, max_tokens=max_tokens)
|
||||
with vllm_runner(
|
||||
model,
|
||||
enable_chunked_prefill=True,
|
||||
# forces prefill chunks with decoding
|
||||
max_num_batched_tokens=MAX_NUM_SEQS * 3,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
) as vllm_model:
|
||||
vllm_model.generate(example_prompts, sampling_params)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [20])
|
||||
@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
|
||||
def test_mamba_cache_cg_padding(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
conv_state_layout: str,
|
||||
) -> None:
|
||||
"""This test is for verifying that mamba cache is padded to CG captured
|
||||
batch size. If it's not, a torch RuntimeError will be raised because
|
||||
tensor dimensions aren't compatible.
|
||||
"""
|
||||
_set_conv_state_layout(monkeypatch, conv_state_layout)
|
||||
|
||||
vllm_config = EngineArgs(model=model, trust_remote_code=True).create_engine_config()
|
||||
cudagraph_dispatcher = CudagraphDispatcher(vllm_config)
|
||||
cudagraph_dispatcher.initialize_cudagraph_keys(
|
||||
vllm_config.compilation_config.cudagraph_mode
|
||||
)
|
||||
if cudagraph_dispatcher.cudagraph_mode == CUDAGraphMode.NONE:
|
||||
pytest.skip("CUDA/XPU graph is disabled.Please enable it to run this test. ")
|
||||
while (
|
||||
len(example_prompts)
|
||||
== cudagraph_dispatcher.dispatch(len(example_prompts))[1].num_tokens
|
||||
):
|
||||
example_prompts.append(example_prompts[0])
|
||||
|
||||
try:
|
||||
with vllm_runner(model, enable_chunked_prefill=True) as vllm_model:
|
||||
vllm_model.generate_greedy(example_prompts, max_tokens)
|
||||
except RuntimeError:
|
||||
pytest.fail(
|
||||
"Couldn't run batch size which is not equal to a Cuda Graph "
|
||||
"captured batch size. "
|
||||
"Could be related to mamba cache not padded correctly"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
|
||||
def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""This test is for verifying that the hybrid inner state management doesn't
|
||||
collapse in case where the number of incoming requests and
|
||||
finished_requests_ids is larger than the maximum mamba block capacity.
|
||||
|
||||
This could generally happen due to the fact that hybrid does support
|
||||
statelessness mechanism where it can clean up new incoming requests in
|
||||
a single step.
|
||||
"""
|
||||
try:
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
|
||||
) as vllm_model:
|
||||
vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
|
||||
except ValueError:
|
||||
pytest.fail(
|
||||
"Hybrid inner state wasn't cleaned up properly between"
|
||||
"steps finished requests registered unnecessarily "
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
|
||||
def test_state_cleanup(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""This test is for verifying that the Hybrid state is cleaned up between
|
||||
steps.
|
||||
|
||||
If it's not cleaned, an error would be expected.
|
||||
"""
|
||||
try:
|
||||
with vllm_runner(
|
||||
model, max_num_seqs=MAX_NUM_SEQS, enable_chunked_prefill=True
|
||||
) as vllm_model:
|
||||
for _ in range(10):
|
||||
vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
|
||||
except ValueError:
|
||||
pytest.fail(
|
||||
"Hybrid inner state wasn't cleaned up between states, "
|
||||
"could be related to finished_requests_ids"
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_distributed_correctness(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
tensor_parallel_size=1,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_tp_1 = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
tensor_parallel_size=2,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_tp_2 = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=vllm_outputs_tp_1,
|
||||
outputs_1_lst=vllm_outputs_tp_2,
|
||||
name_0="vllm_tp_1",
|
||||
name_1="vllm_tp_2",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", FULL_CUDA_GRAPH_MODELS)
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_full_cuda_graph(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
with hf_runner(model) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Leave headroom for the HF model that ran earlier in this process on a
|
||||
# 32.5 GiB MIG.
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
gpu_memory_utilization=0.9,
|
||||
enable_chunked_prefill=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", FP32_STATE_MODELS)
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize(
|
||||
"cache_dtype_param", ["mamba_ssm_cache_dtype", "mamba_cache_dtype"]
|
||||
)
|
||||
def test_fp32_cache_state(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
cache_dtype_param: str,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
with hf_runner(model) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Leave enough headroom for repeated engine initialization on a
|
||||
# 32.5 GiB MIG.
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_num_seqs=MAX_NUM_SEQS,
|
||||
gpu_memory_utilization=0.9,
|
||||
enable_chunked_prefill=True,
|
||||
**{cache_dtype_param: "float32"},
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
# Helper functions for the APC tests
|
||||
def _get_vllm_runner_params(
|
||||
model: str,
|
||||
max_model_len: int,
|
||||
tensor_parallel_size: int = 1,
|
||||
):
|
||||
return {
|
||||
"model_name": model,
|
||||
"enable_chunked_prefill": True,
|
||||
"enable_prefix_caching": False,
|
||||
"max_model_len": max_model_len,
|
||||
"tensor_parallel_size": tensor_parallel_size,
|
||||
"gpu_memory_utilization": 0.4,
|
||||
"attention_backend": ATTN_BACKEND,
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _owned_vllm_runner(vllm_runner, kwargs):
|
||||
with vllm_runner(**kwargs) as runner:
|
||||
yield runner
|
||||
|
||||
|
||||
def _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs,
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
num_repetitions=1,
|
||||
vllm_model=None,
|
||||
):
|
||||
runner_context = (
|
||||
_owned_vllm_runner(vllm_runner, kwargs)
|
||||
if vllm_model is None
|
||||
else nullcontext(vllm_model)
|
||||
)
|
||||
with runner_context as runner:
|
||||
outs = []
|
||||
for _ in range(num_repetitions):
|
||||
if num_logprobs < 0:
|
||||
vllm_output = runner.generate_greedy(prompts, max_tokens)
|
||||
else:
|
||||
vllm_output = runner.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs
|
||||
)
|
||||
outs.append(vllm_output)
|
||||
|
||||
return outs, vllm_model
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("n_repetitions", [2])
|
||||
# If num_logprobs is set to -1, then the stringent version
|
||||
# of the test is executed using `check_outputs_equal`
|
||||
# instead of `check_logprobs_close`
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_apc_single_prompt(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
n_repetitions: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
compare_operator: Callable = (
|
||||
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
|
||||
)
|
||||
|
||||
# Sample prompts.
|
||||
generated_prompts = [APC_MULTIPLY_BY * example_prompts[0]]
|
||||
|
||||
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
|
||||
vllm_runner_kwargs = _get_vllm_runner_params(
|
||||
model, max_model_len, tensor_parallel_size=tensor_parallel_size
|
||||
)
|
||||
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
|
||||
vllm_outputs_no_cache, _ = _get_vLLM_output(
|
||||
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
vllm_outputs_cache_rep, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
generated_prompts,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
n_repetitions,
|
||||
)
|
||||
|
||||
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
|
||||
# In the first repetition, the caches are filled
|
||||
# In the second repetition, these caches are reused
|
||||
|
||||
compare_operator(
|
||||
outputs_0_lst=vllm_outputs_no_cache[0],
|
||||
outputs_1_lst=vllm_outputs_cache_itn,
|
||||
name_0="vllm_no_cache",
|
||||
name_1=f"vllm_cache_it_{r_idx + 1}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("n_repetitions", [2])
|
||||
# If num_logprobs is set to -1, then the stringent version
|
||||
# of the test is executed using `check_outputs_equal`
|
||||
# instead of `check_logprobs_close`
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_apc_single_prompt_block_align_alignment(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
n_repetitions: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
compare_operator: Callable = (
|
||||
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
|
||||
)
|
||||
|
||||
# Sample prompts. This custom prompt is used, as it causes the most issues
|
||||
generated_prompts = ["The president of the United States is " * APC_MULTIPLY_BY]
|
||||
|
||||
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
|
||||
vllm_runner_kwargs = _get_vllm_runner_params(
|
||||
model, max_model_len, tensor_parallel_size=tensor_parallel_size
|
||||
)
|
||||
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
|
||||
|
||||
vllm_outputs_no_cache, _ = _get_vLLM_output(
|
||||
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
with vllm_runner(**vllm_runner_kwargs) as vllm_model:
|
||||
# Retrieve the default mamba state block size
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
mamba_block_size = vllm_config.cache_config.mamba_block_size
|
||||
|
||||
# In case the hybrid model does not have the
|
||||
# "mamba_block_size" assume a fixed constant
|
||||
if mamba_block_size is None:
|
||||
mamba_block_size = 512
|
||||
|
||||
mamba_block_size_multiplier = 10
|
||||
for offsets in [-3, 3, mamba_block_size // 4 + 3, mamba_block_size // 2 - 3]:
|
||||
vllm_runner_kwargs["max_num_batched_tokens"] = (
|
||||
mamba_block_size_multiplier * mamba_block_size - offsets
|
||||
)
|
||||
vllm_outputs_cache_rep, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
generated_prompts,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
n_repetitions,
|
||||
)
|
||||
|
||||
# Check alignment of the output logits when using APC
|
||||
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
|
||||
# In the first repetition, the caches are filled
|
||||
# In the second repetition, these caches are reused
|
||||
|
||||
compare_operator(
|
||||
outputs_0_lst=vllm_outputs_no_cache[0],
|
||||
outputs_1_lst=vllm_outputs_cache_itn,
|
||||
name_0="vllm_no_cache",
|
||||
name_1=f"vllm_cache_it_{r_idx + 1}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("n_repetitions", [2])
|
||||
# If num_logprobs is set to -1, then the stringent version
|
||||
# of the test is executed using `check_outputs_equal`
|
||||
# instead of `check_logprobs_close`
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_apc_multiple_prompts_all_cached_outputs(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
n_repetitions: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
compare_operator: Callable = (
|
||||
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
|
||||
)
|
||||
|
||||
# Sample prompts.
|
||||
generated_prompts = [APC_MULTIPLY_BY * prompt for prompt in example_prompts]
|
||||
|
||||
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
|
||||
vllm_runner_kwargs = _get_vllm_runner_params(
|
||||
model, max_model_len, tensor_parallel_size=tensor_parallel_size
|
||||
)
|
||||
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
|
||||
# Reduce the effects of batch variance on ROCm since batch invariance is not
|
||||
# yet supported. See: https://github.com/vllm-project/vllm/issues/27433
|
||||
if current_platform.is_rocm():
|
||||
vllm_runner_kwargs["max_num_seqs"] = 4
|
||||
|
||||
vllm_outputs_no_cache, _ = _get_vLLM_output(
|
||||
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
vllm_outputs_cache_rep, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
generated_prompts,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
n_repetitions,
|
||||
)
|
||||
|
||||
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
|
||||
# In the first repetition, the caches are filled
|
||||
# In the second repetition, these caches are reused
|
||||
|
||||
compare_operator(
|
||||
outputs_0_lst=vllm_outputs_no_cache[0],
|
||||
outputs_1_lst=vllm_outputs_cache_itn,
|
||||
name_0="vllm_no_cache",
|
||||
name_1=f"vllm_cache_it_{r_idx + 1}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("n_repetitions", [2])
|
||||
# If num_logprobs is set to -1, then the stringent version
|
||||
# of the test is executed using `check_outputs_equal`
|
||||
# instead of `check_logprobs_close`
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_apc_multiple_prompts_block_align_alignment(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
n_repetitions: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
compare_operator: Callable = (
|
||||
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
|
||||
)
|
||||
|
||||
# Sample prompts. This custom prompt is used, as it causes the most issues
|
||||
prompt_text = "The president of the United States is "
|
||||
prompt_offsets = [0, 3, 7, 13, 17, 22, 25, 31]
|
||||
generated_prompts = [
|
||||
prompt_text[offset:] * APC_MULTIPLY_BY for offset in prompt_offsets
|
||||
]
|
||||
|
||||
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
|
||||
vllm_runner_kwargs = _get_vllm_runner_params(
|
||||
model, max_model_len, tensor_parallel_size
|
||||
)
|
||||
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
|
||||
|
||||
vllm_outputs_no_cache, _ = _get_vLLM_output(
|
||||
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
with vllm_runner(**vllm_runner_kwargs) as vllm_model:
|
||||
# Retrieve the default mamba state block size
|
||||
vllm_config = vllm_model.llm.llm_engine.vllm_config
|
||||
mamba_block_size = vllm_config.cache_config.mamba_block_size
|
||||
|
||||
# In case the hybrid model does not have the
|
||||
# "mamba_block_size" assume a fixed constant
|
||||
if mamba_block_size is None:
|
||||
mamba_block_size = 512
|
||||
|
||||
mamba_block_size_multiplier = 10
|
||||
for offsets in [-3, 3, mamba_block_size // 4 + 3, mamba_block_size // 2 - 3]:
|
||||
vllm_runner_kwargs["max_num_batched_tokens"] = (
|
||||
mamba_block_size_multiplier * mamba_block_size - offsets
|
||||
)
|
||||
vllm_outputs_cache_rep, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
generated_prompts,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
n_repetitions,
|
||||
)
|
||||
|
||||
# Check alignment of the output logits when using APC
|
||||
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
|
||||
# In the first repetition, the caches are filled
|
||||
# In the second repetition, these caches are reused
|
||||
|
||||
compare_operator(
|
||||
outputs_0_lst=vllm_outputs_no_cache[0],
|
||||
outputs_1_lst=vllm_outputs_cache_itn,
|
||||
name_0="vllm_no_cache",
|
||||
name_1=f"vllm_cache_it_{r_idx + 1}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("n_repetitions", [2])
|
||||
# If num_logprobs is set to -1, then the stringent version
|
||||
# of the test is executed using `check_outputs_equal`
|
||||
# instead of `check_logprobs_close`
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_apc_multiple_prompts_partial_cached_outputs(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
max_tokens: int,
|
||||
n_repetitions: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
) -> None:
|
||||
try:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
compare_operator: Callable = (
|
||||
check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
|
||||
)
|
||||
|
||||
# Sample prompts.
|
||||
generated_prompts = [APC_MULTIPLY_BY * prompt for prompt in example_prompts]
|
||||
|
||||
max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
|
||||
vllm_runner_kwargs = _get_vllm_runner_params(
|
||||
model, max_model_len, tensor_parallel_size=tensor_parallel_size
|
||||
)
|
||||
vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
|
||||
|
||||
vllm_outputs_no_cache, _ = _get_vLLM_output(
|
||||
vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
# Cache only part of all the prompts
|
||||
vllm_runner_kwargs["enable_prefix_caching"] = True
|
||||
with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
|
||||
vllm_outputs_partial_cache, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
generated_prompts[:3],
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
vllm_model=vllm_model,
|
||||
)
|
||||
|
||||
compare_operator(
|
||||
outputs_0_lst=vllm_outputs_no_cache[0][:3],
|
||||
outputs_1_lst=vllm_outputs_partial_cache[0],
|
||||
name_0="vllm_no_cache",
|
||||
name_1="vllm_partial_cache",
|
||||
)
|
||||
|
||||
vllm_outputs_cache_rep, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
vllm_runner_kwargs,
|
||||
generated_prompts,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
n_repetitions,
|
||||
vllm_model=vllm_model,
|
||||
)
|
||||
|
||||
for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
|
||||
# In the first repetition, the caches are filled
|
||||
# In the second repetition, these caches are reused
|
||||
|
||||
compare_operator(
|
||||
outputs_0_lst=vllm_outputs_no_cache[0],
|
||||
outputs_1_lst=vllm_outputs_cache_itn,
|
||||
name_0="vllm_no_cache",
|
||||
name_1=f"vllm_cache_it_{r_idx + 1}",
|
||||
)
|
||||
|
||||
|
||||
# Test that outputs match whether prefix caching is enabled or not for mamba.
|
||||
@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
|
||||
def test_same_mamba_output_apc_on_vs_off(
|
||||
vllm_runner,
|
||||
model: str,
|
||||
) -> None:
|
||||
num_logprobs = 5
|
||||
prompts = [
|
||||
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
|
||||
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
|
||||
]
|
||||
max_tokens = 20
|
||||
max_model_len = max(len(p) for p in prompts) + max_tokens + 64
|
||||
|
||||
base_kwargs = _get_vllm_runner_params(model, max_model_len)
|
||||
base_kwargs.update(
|
||||
enforce_eager=True, block_size=16, seed=42, gpu_memory_utilization=0.8
|
||||
)
|
||||
|
||||
# No prefix caching
|
||||
kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
|
||||
with vllm_runner(**kwargs_no_apc) as vllm_model:
|
||||
outputs_no_apc, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs_no_apc,
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
vllm_model=vllm_model,
|
||||
)
|
||||
# With prefix caching
|
||||
kwargs_with_apc = {
|
||||
**base_kwargs,
|
||||
"enable_prefix_caching": True,
|
||||
"mamba_block_size": 16,
|
||||
}
|
||||
with vllm_runner(**kwargs_with_apc) as vllm_model:
|
||||
outputs_with_apc, _ = _get_vLLM_output(
|
||||
vllm_runner,
|
||||
kwargs_with_apc,
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
vllm_model=vllm_model,
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=outputs_no_apc[0],
|
||||
outputs_1_lst=outputs_with_apc[0],
|
||||
name_0="vllm_no_apc",
|
||||
name_1="vllm_with_apc",
|
||||
)
|
||||
|
||||
|
||||
# we have to use a real large model to get reasonable results
|
||||
# the model can't be a hybrid model as we need block_size 16
|
||||
@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
|
||||
def test_apc_common_prefix_same_batch(
|
||||
model: str,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
# Required to put the two requests in the same batch
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
llm = LLM(
|
||||
model=model,
|
||||
enforce_eager=True,
|
||||
block_size=16,
|
||||
mamba_block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
seed=42,
|
||||
attention_backend=ATTN_BACKEND,
|
||||
)
|
||||
prompts = [
|
||||
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
|
||||
"hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=20)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
for output in outputs:
|
||||
assert "two" in output.outputs[0].text
|
||||
@@ -1,350 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.parser.mistral import MistralToolCall
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
||||
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
MODELS = [
|
||||
"mistralai/Mistral-7B-Instruct-v0.3",
|
||||
]
|
||||
|
||||
MISTRAL_FORMAT_MODELS = [
|
||||
"mistralai/Mistral-7B-Instruct-v0.3",
|
||||
# uses the v3-Tekken tokenizer
|
||||
"mistralai/Ministral-8B-Instruct-2410",
|
||||
# Mistral-Nemo is too big for CI, but passes locally
|
||||
# "mistralai/Mistral-Nemo-Instruct-2407"
|
||||
]
|
||||
|
||||
SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
|
||||
SYMBOLIC_LANG_PROMPTS = [
|
||||
"勇敢な船乗りについての詩を書く", # japanese
|
||||
"寫一首關於勇敢的水手的詩", # chinese
|
||||
"ပုံပြင်လေးပြောပြပါ်:\n", # burmese
|
||||
"Repeat the phrase 'URGENCY🌶️':\nURGENCY🌶️\nURGENCY🌶️\n", # see https://github.com/vllm-project/vllm/pull/9625
|
||||
]
|
||||
|
||||
# for function calling
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_current_weather",
|
||||
"description": "Get the current weather in a given location",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"city": {
|
||||
"type": "string",
|
||||
"description": "The city to find the weather for, e.g. "
|
||||
"'San Francisco'",
|
||||
},
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": "the two-letter abbreviation for the state that "
|
||||
"the city is in, e.g. 'CA' which would mean 'California'",
|
||||
},
|
||||
"unit": {
|
||||
"type": "string",
|
||||
"description": "The unit to fetch the temperature in",
|
||||
"enum": ["celsius", "fahrenheit"],
|
||||
},
|
||||
},
|
||||
"required": ["city", "state", "unit"],
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "rewrite",
|
||||
"description": "Rewrites text",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"required": [],
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string",
|
||||
"description": "The input text to rewrite.",
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
]
|
||||
MSGS = [
|
||||
{"role": "system", "content": "You are an assistant."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Could you please rewrite the below article? \n\n My English needs "
|
||||
"improving, maybe I make errors.",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "bbc5b7ede",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "rewrite",
|
||||
"arguments": '{"text":"My English needs improving, maybe '
|
||||
'I make errors."}',
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"content": '{"action":"rewrite","outcome":"My English needs improving, maybe '
|
||||
'I make errors."}',
|
||||
"tool_call_id": "bbc5b7ede",
|
||||
"name": "rewrite",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "---\n\nMy English needs improving, maybe I make errors",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": (
|
||||
"Can you tell me what the temperate will be in Dallas, in fahrenheit?"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
SAMPLE_JSON_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"age": {"type": "integer"},
|
||||
"skills": {
|
||||
"type": "array",
|
||||
"items": {"type": "string", "maxLength": 10},
|
||||
"minItems": 3,
|
||||
},
|
||||
"work_history": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"company": {"type": "string"},
|
||||
"duration": {"type": "number"},
|
||||
"position": {"type": "string"},
|
||||
},
|
||||
"required": ["company", "position"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["name", "age", "skills", "work_history"],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(model, dtype=dtype, tokenizer_mode="mistral") as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [64])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_mistral_format(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
tokenizer_mode="mistral",
|
||||
load_format="mistral",
|
||||
config_format="mistral",
|
||||
) as mistral_format_model:
|
||||
mistral_format_outputs = mistral_format_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
tokenizer_mode="hf",
|
||||
load_format="safetensors",
|
||||
config_format="hf",
|
||||
) as hf_format_model:
|
||||
hf_format_outputs = hf_format_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_format_outputs,
|
||||
outputs_1_lst=mistral_format_outputs,
|
||||
name_0="hf",
|
||||
name_1="mistral",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
def test_mistral_symbolic_languages(vllm_runner, model: str, dtype: str) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=8192,
|
||||
tokenizer_mode="mistral",
|
||||
config_format="mistral",
|
||||
load_format="mistral",
|
||||
) as vllm_model:
|
||||
for prompt in SYMBOLIC_LANG_PROMPTS:
|
||||
msg = {"role": "user", "content": prompt}
|
||||
outputs = vllm_model.llm.chat([msg], sampling_params=SAMPLING_PARAMS)
|
||||
assert "�" not in outputs[0].outputs[0].text.strip()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
tokenizer_mode="mistral",
|
||||
config_format="mistral",
|
||||
load_format="mistral",
|
||||
) as vllm_model:
|
||||
msgs = copy.deepcopy(MSGS)
|
||||
outputs = vllm_model.llm.chat(
|
||||
msgs, tools=TOOLS, sampling_params=SAMPLING_PARAMS
|
||||
)
|
||||
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
tool_parser = MistralToolParser(tokenizer)
|
||||
|
||||
model_output = outputs[0].outputs[0].text.strip()
|
||||
assert model_output.startswith(tool_parser.bot_token), model_output
|
||||
parsed_message = tool_parser.extract_tool_calls(model_output, None)
|
||||
|
||||
assert parsed_message.tools_called
|
||||
|
||||
assert MistralToolCall.is_valid_id(parsed_message.tool_calls[0].id)
|
||||
assert parsed_message.tool_calls[0].function.name == "get_current_weather"
|
||||
assert (
|
||||
parsed_message.tool_calls[0].function.arguments
|
||||
== '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}'
|
||||
) # noqa
|
||||
assert parsed_message.content is None
|
||||
|
||||
|
||||
def test_mistral_function_call_nested_json():
|
||||
"""Ensure that the function-name regex captures the entire outermost
|
||||
JSON block, including nested braces."""
|
||||
|
||||
# Create a minimal stub tokenizer that provides the few attributes the
|
||||
# parser accesses (`version` and `get_vocab`).
|
||||
class _StubMistralTokenizer(MistralTokenizer):
|
||||
version = 11 # Satisfy the version check
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
def get_vocab():
|
||||
# Provide the special TOOL_CALLS token expected by the parser.
|
||||
return {"[TOOL_CALLS]": 0}
|
||||
|
||||
tokenizer = _StubMistralTokenizer()
|
||||
parser = MistralToolParser(tokenizer)
|
||||
|
||||
# Craft a model output featuring nested JSON inside the arguments.
|
||||
args_dict = {
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
"sub_dict": {"foo": "bar", "inner": {"x": 1, "y": 2}},
|
||||
}
|
||||
|
||||
model_output = f"{parser.bot_token}get_current_weather{json.dumps(args_dict)}"
|
||||
|
||||
parsed = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
# Assertions: the tool call is detected and the full nested JSON is parsed
|
||||
# without truncation.
|
||||
assert parsed.tools_called
|
||||
|
||||
assert MistralToolCall.is_valid_id(parsed.tool_calls[0].id)
|
||||
assert parsed.tool_calls[0].function.name == "get_current_weather"
|
||||
assert json.loads(parsed.tool_calls[0].function.arguments) == args_dict
|
||||
# No additional content outside the tool call should be returned.
|
||||
assert parsed.content is None
|
||||
|
||||
# multiple calls
|
||||
multiple_args_dict = [
|
||||
{
|
||||
"city": "Dallas",
|
||||
"state": "TX",
|
||||
"unit": "fahrenheit",
|
||||
"sub_dict": {"foo": "bar", "inner": {"x": 1, "y": 2}},
|
||||
},
|
||||
{},
|
||||
{"a": 0},
|
||||
{"a": 1, "b": "c"},
|
||||
]
|
||||
names = ["get_current_weather", "get_current_weather_2", "random", "random_2"]
|
||||
|
||||
model_output = "".join(
|
||||
[
|
||||
f"{parser.bot_token}{name}{json.dumps(args)}"
|
||||
for name, args in zip(names, multiple_args_dict)
|
||||
]
|
||||
)
|
||||
|
||||
parsed = parser.extract_tool_calls(model_output, None)
|
||||
|
||||
# Assertions: the tool call is detected and the full nested JSON is parsed
|
||||
# without truncation.
|
||||
assert parsed.tools_called
|
||||
assert len(parsed.tool_calls) == len(multiple_args_dict)
|
||||
|
||||
for i, tool_call in enumerate(parsed.tool_calls):
|
||||
assert MistralToolCall.is_valid_id(tool_call.id)
|
||||
assert tool_call.function.name == names[i]
|
||||
assert json.loads(tool_call.function.arguments) == multiple_args_dict[i]
|
||||
# No additional content outside the tool call should be returned.
|
||||
assert parsed.content is None
|
||||
@@ -1,160 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import correctness_test_embed_models
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
RerankModelInfo,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
from .mteb_score_utils import mteb_test_rerank_models
|
||||
|
||||
MODELS = [
|
||||
########## BertModel
|
||||
EmbedModelInfo(
|
||||
"thenlper/gte-large",
|
||||
mteb_score=0.76807651,
|
||||
mteb_tol=2e-3,
|
||||
architecture="BertModel",
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo("thenlper/gte-base", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo("thenlper/gte-small", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo(
|
||||
"thenlper/gte-large-zh", architecture="BertModel", enable_test=False
|
||||
),
|
||||
EmbedModelInfo("thenlper/gte-base-zh", architecture="BertModel", enable_test=False),
|
||||
EmbedModelInfo(
|
||||
"thenlper/gte-small-zh", architecture="BertModel", enable_test=False
|
||||
),
|
||||
########### NewModel
|
||||
# These three architectures are almost the same, but not exactly the same.
|
||||
# For example,
|
||||
# - whether to use token_type_embeddings
|
||||
# - whether to use context expansion
|
||||
# So only test one (the most widely used) model
|
||||
EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-multilingual-base",
|
||||
architecture="GteNewModel",
|
||||
mteb_score=0.775074696,
|
||||
mteb_tol=2e-3,
|
||||
hf_overrides={"architectures": ["GteNewModel"]},
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-base-en-v1.5",
|
||||
architecture="GteNewModel",
|
||||
hf_overrides={"architectures": ["GteNewModel"]},
|
||||
enable_test=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-large-en-v1.5",
|
||||
architecture="GteNewModel",
|
||||
hf_overrides={"architectures": ["GteNewModel"]},
|
||||
enable_test=False,
|
||||
),
|
||||
########### Qwen2ForCausalLM
|
||||
EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-Qwen2-1.5B-instruct",
|
||||
mteb_score=0.758473459018872,
|
||||
architecture="Qwen2ForCausalLM",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
# Skip: numerical regression with transformers v5.
|
||||
enable_test=False,
|
||||
),
|
||||
########## ModernBertModel
|
||||
EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-modernbert-base",
|
||||
mteb_score=0.748193353,
|
||||
mteb_tol=2e-3,
|
||||
architecture="ModernBertModel",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
########## Qwen3ForCausalLM
|
||||
EmbedModelInfo(
|
||||
"Qwen/Qwen3-Embedding-0.6B",
|
||||
mteb_score=0.771163695,
|
||||
mteb_tol=2e-3,
|
||||
architecture="Qwen3ForCausalLM",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
enable_test=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"Qwen/Qwen3-Embedding-4B",
|
||||
architecture="Qwen3ForCausalLM",
|
||||
enable_test=False,
|
||||
),
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
# classifier_pooling: mean
|
||||
"Alibaba-NLP/gte-reranker-modernbert-base",
|
||||
mteb_score=0.33386,
|
||||
mteb_tol=1e-2,
|
||||
architecture="ModernBertForSequenceClassification",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
RerankModelInfo(
|
||||
"Alibaba-NLP/gte-multilingual-reranker-base",
|
||||
mteb_score=0.33062,
|
||||
mteb_tol=1e-2,
|
||||
architecture="GteNewForSequenceClassification",
|
||||
hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
vllm_extra_kwargs = {}
|
||||
if current_platform.is_rocm():
|
||||
vllm_extra_kwargs["attention_backend"] = "TRITON_ATTN"
|
||||
mteb_test_rerank_models(
|
||||
vllm_runner, model_info, vllm_extra_kwargs=vllm_extra_kwargs
|
||||
)
|
||||
@@ -1,170 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.language.pooling.embed_utils import (
|
||||
check_embeddings_close,
|
||||
correctness_test_embed_models,
|
||||
matryoshka_fy,
|
||||
)
|
||||
from tests.models.utils import (
|
||||
EmbedModelInfo,
|
||||
RerankModelInfo,
|
||||
)
|
||||
from vllm import PoolingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
from .mteb_score_utils import mteb_test_rerank_models
|
||||
|
||||
EMBEDDING_MODELS = [
|
||||
EmbedModelInfo(
|
||||
"jinaai/jina-embeddings-v3",
|
||||
mteb_score=0.824413164,
|
||||
mteb_tol=2e-3,
|
||||
architecture="XLMRobertaModel",
|
||||
is_matryoshka=True,
|
||||
seq_pooling_type="MEAN",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"jinaai/jina-embeddings-v5-text-small",
|
||||
mteb_score=0.794535707854956,
|
||||
mteb_tol=2e-3,
|
||||
architecture="JinaEmbeddingsV5Model",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="decoder",
|
||||
is_prefix_caching_supported=True,
|
||||
is_chunked_prefill_supported=True,
|
||||
),
|
||||
EmbedModelInfo(
|
||||
"jinaai/jina-embeddings-v5-text-nano",
|
||||
architecture="JinaEmbeddingsV5Model",
|
||||
mteb_tol=2e-3,
|
||||
dtype="bfloat16" if current_platform.is_rocm() else "auto",
|
||||
seq_pooling_type="LAST",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
),
|
||||
]
|
||||
|
||||
RERANK_MODELS = [
|
||||
RerankModelInfo(
|
||||
"jinaai/jina-reranker-v2-base-multilingual",
|
||||
mteb_score=0.33643,
|
||||
mteb_tol=1e-2,
|
||||
architecture="XLMRobertaForSequenceClassification",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
|
||||
def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
|
||||
task = "retrieval" if "v5" in model_info.name else "text-matching"
|
||||
prompt_prefix: str | None = "Document: " if "v5" in model_info.name else None
|
||||
|
||||
def hf_model_callback(model):
|
||||
model.encode = partial(model.encode, task=task)
|
||||
|
||||
mteb_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
hf_model_callback=hf_model_callback,
|
||||
prompt_prefix=prompt_prefix,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
|
||||
def test_embed_models_correctness(
|
||||
hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
|
||||
) -> None:
|
||||
task = "retrieval" if "v5" in model_info.name else "text-matching"
|
||||
|
||||
def hf_model_callback(model):
|
||||
model.encode = partial(model.encode, task=task)
|
||||
|
||||
correctness_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
example_prompts,
|
||||
hf_model_callback=hf_model_callback,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.parametrize("model_info", RERANK_MODELS)
|
||||
def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
|
||||
mteb_test_rerank_models(vllm_runner, model_info)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="jinaai/jina-embeddings-v3 custom XLMRobertaLoRA model on HF hub "
|
||||
"is incompatible with transformers v5 (missing all_tied_weights_keys)"
|
||||
)
|
||||
@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("dimensions", [16, 32])
|
||||
def test_matryoshka(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model_info,
|
||||
dtype: str,
|
||||
dimensions: int,
|
||||
example_prompts,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
if not model_info.is_matryoshka:
|
||||
pytest.skip("Model is not matryoshka")
|
||||
|
||||
# ST will strip the input texts, see test_embedding.py
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
task = "retrieval" if "v5" in model_info.name else "text-matching"
|
||||
|
||||
with hf_runner(
|
||||
model_info.name,
|
||||
dtype=dtype,
|
||||
is_sentence_transformer=True,
|
||||
) as hf_model:
|
||||
hf_outputs = hf_model.encode(example_prompts, task=task)
|
||||
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
|
||||
|
||||
with vllm_runner(
|
||||
model_info.name, runner="pooling", dtype=dtype, max_model_len=None
|
||||
) as vllm_model:
|
||||
assert vllm_model.llm.llm_engine.model_config.is_matryoshka
|
||||
|
||||
matryoshka_dimensions = (
|
||||
vllm_model.llm.llm_engine.model_config.matryoshka_dimensions
|
||||
)
|
||||
assert matryoshka_dimensions is not None
|
||||
|
||||
if dimensions not in matryoshka_dimensions:
|
||||
with pytest.raises(ValueError):
|
||||
vllm_model.embed(
|
||||
example_prompts, pooling_params=PoolingParams(dimensions=dimensions)
|
||||
)
|
||||
else:
|
||||
vllm_outputs = vllm_model.embed(
|
||||
example_prompts, pooling_params=PoolingParams(dimensions=dimensions)
|
||||
)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import EmbedModelInfo
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .mteb_embed_utils import mteb_test_embed_models
|
||||
|
||||
|
||||
def _fp8_scaled_mm_unsupported() -> bool:
|
||||
"""Whether the ModernBERT online FP8 test should be skipped.
|
||||
|
||||
``is_quant_method_supported("fp8")`` returns True on MI250 (gfx90a), but
|
||||
serving a per-tensor FP8 linear layer on ROCm requires CDNA3+ or RDNA4,
|
||||
so treat it as unsupported here.
|
||||
"""
|
||||
if not is_quant_method_supported("fp8"):
|
||||
return True
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import get_cdna_version, on_gfx12x
|
||||
|
||||
return get_cdna_version() <= 2 and not on_gfx12x()
|
||||
return False
|
||||
|
||||
|
||||
MODEL_INFO = EmbedModelInfo(
|
||||
"Alibaba-NLP/gte-modernbert-base",
|
||||
mteb_score=0.748193353,
|
||||
architecture="ModernBertModel",
|
||||
seq_pooling_type="CLS",
|
||||
attn_type="encoder_only",
|
||||
is_prefix_caching_supported=False,
|
||||
is_chunked_prefill_supported=False,
|
||||
enable_test=True,
|
||||
)
|
||||
|
||||
|
||||
def _assert_modernbert_online_fp8(model) -> None:
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
|
||||
layer = model.encoder_layer.layers[0]
|
||||
linears = {
|
||||
"attn.Wqkv": layer.attn.Wqkv,
|
||||
"attn.Wo": layer.attn.Wo,
|
||||
"mlp.Wi": layer.mlp.Wi,
|
||||
"mlp.Wo": layer.mlp.Wo,
|
||||
}
|
||||
for name, linear in linears.items():
|
||||
assert isinstance(linear.quant_method, Fp8PerTensorOnlineLinearMethod), (
|
||||
f"{name} was not quantized with online FP8"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=2)
|
||||
@pytest.mark.skipif(
|
||||
_fp8_scaled_mm_unsupported(),
|
||||
reason="No FP8 ScaledMM kernel is available on this GPU type.",
|
||||
)
|
||||
def test_modernbert_online_fp8_mteb(hf_runner, vllm_runner, monkeypatch) -> None:
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
def check_online_fp8(vllm_model) -> None:
|
||||
vllm_model.apply_model(_assert_modernbert_online_fp8)
|
||||
|
||||
mteb_test_embed_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
MODEL_INFO,
|
||||
vllm_extra_kwargs={"quantization": "fp8_per_tensor"},
|
||||
# Account for Online FP8 kernel variance across GPU architectures.
|
||||
atol=2e-3,
|
||||
vllm_model_callback=check_online_fp8,
|
||||
)
|
||||
@@ -1,171 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2025 The vLLM team.
|
||||
# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
|
||||
# reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
MODEL_NAME = "nvidia/audio-flamingo-3-hf"
|
||||
SINGLE_CONVERSATION = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "What is surprising about the relationship between "
|
||||
"the barking and the music?",
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "https://huggingface.co/datasets/nvidia/AudioSkills/"
|
||||
"resolve/main/assets/"
|
||||
"dogs_barking_in_sync_with_the_music.wav",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
BATCHED_CONVERSATIONS = [
|
||||
SINGLE_CONVERSATION,
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Why is the philosopher's name mentioned in the "
|
||||
"lyrics? (A) To express a sense of nostalgia "
|
||||
"(B) To indicate that language cannot express clearly, "
|
||||
"satirizing the inversion of black and white in the world "
|
||||
"(C) To add depth and complexity to the lyrics "
|
||||
"(D) To showcase the wisdom and influence of the "
|
||||
"philosopher",
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "https://huggingface.co/datasets/nvidia/"
|
||||
"AudioSkills/resolve/main/assets/"
|
||||
"Ch6Ae9DT6Ko_00-04-03_00-04-31.wav",
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
]
|
||||
|
||||
|
||||
def get_fixture_path(filename):
|
||||
return os.path.join(
|
||||
os.path.dirname(__file__), "../../fixtures/audioflamingo3", filename
|
||||
)
|
||||
|
||||
|
||||
def assert_output_matches(output, expected_text, expected_token_ids):
|
||||
generated = output.outputs[0]
|
||||
assert generated.text.strip() == expected_text
|
||||
actual_token_ids = list(generated.token_ids)
|
||||
assert (
|
||||
actual_token_ids == expected_token_ids
|
||||
or actual_token_ids == expected_token_ids[:-1]
|
||||
or actual_token_ids[:-1] == expected_token_ids
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
try:
|
||||
return LLM(
|
||||
model=MODEL_NAME,
|
||||
dtype="bfloat16",
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
)
|
||||
except Exception as e:
|
||||
pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
|
||||
|
||||
|
||||
def test_single_generation(llm):
|
||||
fixture_path = get_fixture_path("expected_results_single.json")
|
||||
if not os.path.exists(fixture_path):
|
||||
pytest.skip(f"Fixture not found: {fixture_path}")
|
||||
|
||||
with open(fixture_path) as f:
|
||||
expected = json.load(f)
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
|
||||
|
||||
outputs = llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
assert_output_matches(
|
||||
outputs[0],
|
||||
expected["transcriptions"][0],
|
||||
expected["token_ids"][0],
|
||||
)
|
||||
|
||||
|
||||
def test_batched_generation(llm):
|
||||
fixture_path = get_fixture_path("expected_results_batched.json")
|
||||
if not os.path.exists(fixture_path):
|
||||
pytest.skip(f"Fixture not found: {fixture_path}")
|
||||
|
||||
with open(fixture_path) as f:
|
||||
expected = json.load(f)
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
|
||||
|
||||
outputs = llm.chat(
|
||||
messages=BATCHED_CONVERSATIONS,
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
|
||||
for i, output in enumerate(outputs):
|
||||
assert_output_matches(
|
||||
output,
|
||||
expected["transcriptions"][i],
|
||||
expected["token_ids"][i],
|
||||
)
|
||||
|
||||
|
||||
def test_single_and_batched_generation_match(llm):
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
|
||||
|
||||
single_output = llm.chat(
|
||||
messages=SINGLE_CONVERSATION,
|
||||
sampling_params=sampling_params,
|
||||
)[0]
|
||||
batched_output = llm.chat(
|
||||
messages=BATCHED_CONVERSATIONS,
|
||||
sampling_params=sampling_params,
|
||||
)[0]
|
||||
|
||||
assert single_output.outputs[0].text == batched_output.outputs[0].text
|
||||
assert list(single_output.outputs[0].token_ids) == list(
|
||||
batched_output.outputs[0].token_ids
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,182 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import pytest
|
||||
from transformers import AutoModelForSpeechSeq2Seq
|
||||
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import AudioTestAssets, HfRunner, PromptAudioInput, VllmRunner
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
HF_AUDIO_PROMPT = "<|start_of_role|>system<|end_of_role|>Knowledge Cutoff Date: April 2024.\nToday's Date: December 19, 2024.\nYou are Granite, developed by IBM. You are a helpful AI assistant<|end_of_text|>\n<|start_of_role|>user<|end_of_role|><|audio|>can you transcribe the speech into a written format?<|end_of_text|>\n<|start_of_role|>assistant<|end_of_role|>" # noqa: E501
|
||||
|
||||
|
||||
def vllm_to_hf_output(
|
||||
vllm_output: tuple[list[int], str, SampleLogprobs | None],
|
||||
) -> tuple[list[int], str, SampleLogprobs | None]:
|
||||
"""Sanitize hf output to be comparable with vllm output."""
|
||||
output_ids, output_str, out_logprobs = vllm_output
|
||||
|
||||
hf_output_str = output_str + "<|end_of_text|>"
|
||||
|
||||
return output_ids, hf_output_str, out_logprobs
|
||||
|
||||
|
||||
MODEL_NAME = "ibm-granite/granite-speech-3.3-2b"
|
||||
MODEL_NAME_4_0 = "ibm-granite/granite-4.0-1b-speech"
|
||||
# "plus" variant of granite speech (uses GraniteSpeechPlusForConditionalGeneration).
|
||||
MODEL_NAME_4_1_PLUS = "ibm-granite/granite-speech-4.1-2b-plus"
|
||||
# Audio lora co-exists directly in the 3.3 model directory,
|
||||
# the 4.0 and 4.1-plus models have adapters merged into the weights.
|
||||
models: dict[str, str | None] = {
|
||||
MODEL_NAME: MODEL_NAME,
|
||||
MODEL_NAME_4_0: None,
|
||||
MODEL_NAME_4_1_PLUS: None,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def granite_speech_attention_config():
|
||||
"""Return attention config for Granite Speech tests on ROCm."""
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
# -1 (unknown arch) is truthy; gate on CDNA3+ like other call sites.
|
||||
if get_cdna_version() > 2:
|
||||
return {"backend": "ROCM_AITER_FA"}
|
||||
return {"backend": "TRITON_ATTN"}
|
||||
return None
|
||||
|
||||
|
||||
def run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: Sequence[tuple[list[str], PromptAudioInput]],
|
||||
model: str,
|
||||
*,
|
||||
max_model_len: int,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
attention_config: dict | None = None,
|
||||
audio_lora_path: str | None = None,
|
||||
):
|
||||
"""Inference result should be the same between hf and vllm.
|
||||
|
||||
All the audio fixtures for the test are from AUDIO_ASSETS.
|
||||
For huggingface runner, we provide the audio as input.
|
||||
For vllm runner, we provide MultiModalDataDict objects
|
||||
and corresponding MultiModalConfig as input.
|
||||
Note, the text input is also adjusted to abide by vllm contract.
|
||||
The text output is sanitized to be able to compare with hf.
|
||||
"""
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
# max_model_len should be greater than image_feature_size
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=1,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enable_lora=audio_lora_path is not None,
|
||||
max_lora_rank=64,
|
||||
enforce_eager=True,
|
||||
attention_config=attention_config,
|
||||
) as vllm_model:
|
||||
lora_request = (
|
||||
LoRARequest("audio", 1, audio_lora_path) if audio_lora_path else None
|
||||
)
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=audios,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
for prompts, audios in inputs
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModelForSpeechSeq2Seq) as hf_model:
|
||||
hf_processor = hf_model.processor
|
||||
eos_token_id = hf_processor.tokenizer.eos_token_id
|
||||
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
audios=[audios],
|
||||
eos_token_id=eos_token_id,
|
||||
)
|
||||
for prompts, audios in inputs
|
||||
]
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=[vllm_to_hf_output(output) for output in vllm_outputs],
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,audio_lora_path", models.items())
|
||||
@pytest.mark.parametrize(
|
||||
"dtype", ["float16"] if current_platform.is_rocm() else ["bfloat16"]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"max_model_len", [512] if current_platform.is_rocm() else [2048]
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [10])
|
||||
def test_models(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
model: str,
|
||||
audio_lora_path: str | None,
|
||||
audio_assets: AudioTestAssets,
|
||||
granite_speech_attention_config,
|
||||
dtype: str,
|
||||
max_model_len: int,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
audio, sr = audio_assets[0].audio_and_sample_rate
|
||||
# This model expects 16k sample rate, which our test audio
|
||||
# already is; if this changes, it may break this test,
|
||||
# so we check it directly
|
||||
assert sr == 16000
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
[
|
||||
([HF_AUDIO_PROMPT], [audio]),
|
||||
],
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_model_len=max_model_len,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tensor_parallel_size=1,
|
||||
attention_config=granite_speech_attention_config,
|
||||
audio_lora_path=audio_lora_path,
|
||||
)
|
||||
@@ -1,76 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
from PIL.Image import Image
|
||||
from transformers import AutoProcessor
|
||||
|
||||
from vllm import LLM, EngineArgs, SamplingParams
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
|
||||
MODEL_NAME = "Kwai-Keye/Keye-VL-8B-Preview"
|
||||
|
||||
QUESTION = "What is the content of each image?"
|
||||
|
||||
|
||||
class ModelRequestData(NamedTuple):
|
||||
engine_args: EngineArgs
|
||||
prompt: str
|
||||
image_data: list[Image]
|
||||
stop_token_ids: list[int] | None = None
|
||||
chat_template: str | None = None
|
||||
sampling_params: SamplingParams | None = None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("question", [QUESTION])
|
||||
def test_keye_vl(image_assets, question: str):
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
image_urls = [encode_image_url(image) for image in images]
|
||||
|
||||
placeholders = [{"type": "image", "image": url} for url in image_urls]
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*placeholders,
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
processor = AutoProcessor.from_pretrained(MODEL_NAME, trust_remote_code=True)
|
||||
|
||||
prompt = processor.apply_chat_template(
|
||||
messages, tokenize=False, add_generation_prompt=True
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
trust_remote_code=True,
|
||||
max_model_len=8192,
|
||||
max_num_seqs=5,
|
||||
limit_mm_per_prompt={"image": len(image_urls)},
|
||||
seed=42,
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0, max_tokens=256, stop_token_ids=None
|
||||
)
|
||||
|
||||
outputs = llm.generate(
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"image": images},
|
||||
},
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
|
||||
print("-" * 50)
|
||||
for o in outputs:
|
||||
generated_text = o.outputs[0].text
|
||||
print(generated_text)
|
||||
assert len(generated_text) > 10, (
|
||||
f"Generated text is too short: {generated_text}"
|
||||
)
|
||||
print("-" * 50)
|
||||
@@ -1,713 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Create a reduced-layer version of the Maverick model for testing purposes.
|
||||
|
||||
This script creates a new model with fewer layers by:
|
||||
1. Loading the original Maverick model configuration
|
||||
2. Creating a reduced configuration
|
||||
3. Generating compatible safetensors files with appropriate weights
|
||||
4. Creating the necessary index files for vLLM compatibility
|
||||
"""
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
from transformers import AutoConfig, AutoProcessor, AutoTokenizer, GenerationConfig
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.kv_cache_interface import ChunkedLocalAttentionSpec, FullAttentionSpec
|
||||
|
||||
from ....utils import multi_gpu_test
|
||||
|
||||
# Sample prompts for testing
|
||||
PROMPTS: list[str] = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
|
||||
def run_maverick_serving(model: str):
|
||||
"""Test Llama-4-Maverick model with vLLM LLM class using CLI equivalent
|
||||
options with reduced layers.
|
||||
"""
|
||||
try:
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
|
||||
|
||||
llm = LLM(
|
||||
model=model,
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
tensor_parallel_size=8,
|
||||
enable_expert_parallel=True,
|
||||
trust_remote_code=True,
|
||||
gpu_memory_utilization=0.4,
|
||||
kv_cache_dtype="fp8",
|
||||
)
|
||||
|
||||
outputs = llm.generate(PROMPTS, sampling_params)
|
||||
|
||||
# Print the outputs
|
||||
print("\nGenerated Outputs:\n" + "-" * 60)
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}")
|
||||
print(f"Output: {generated_text!r}")
|
||||
print("-" * 60)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error initializing or running model: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def get_rope_layers_config(model_path: str) -> list[int]:
|
||||
"""Get the interleaved RoPE configuration from HuggingFace config.
|
||||
|
||||
Args:
|
||||
model_path: Path to the local directory containing the reduced
|
||||
Maverick model checkpoint
|
||||
|
||||
Returns:
|
||||
List of 0 or 1 indicating whether each layer uses RoPE and local attn
|
||||
0 indicates that RoPE is not used while 1 indicates that RoPE is used.
|
||||
|
||||
"""
|
||||
config_path = Path(model_path) / "config.json"
|
||||
model_config = json.loads(config_path.read_text())
|
||||
text_config = model_config["text_config"]
|
||||
no_rope_layers = text_config["no_rope_layers"]
|
||||
print(f"Found no_rope_layers: {no_rope_layers}")
|
||||
return no_rope_layers
|
||||
|
||||
|
||||
def create_reduced_maverick_model(
|
||||
original_model_name: str = "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
|
||||
output_dir: str = "/tmp/reduced_maverick",
|
||||
text_layers: int = 4,
|
||||
num_experts: int = 4,
|
||||
vision_layers: int = 2,
|
||||
force_recreate: bool = False,
|
||||
) -> str:
|
||||
"""Create a reduced-layer version of the Maverick model.
|
||||
|
||||
Args:
|
||||
original_model_name: Name of the original Maverick model
|
||||
output_dir: Directory to save the reduced model
|
||||
text_layers: Number of text transformer layers
|
||||
num_experts: Number of experts per layer
|
||||
vision_layers: Number of vision transformer layers
|
||||
force_recreate: Whether to recreate if output_dir already exists
|
||||
|
||||
Returns:
|
||||
Path to the created reduced model directory
|
||||
|
||||
"""
|
||||
print(
|
||||
f"Creating reduced Maverick model with {text_layers} text layers and "
|
||||
f"{vision_layers} vision layers..."
|
||||
)
|
||||
|
||||
# Create output directory
|
||||
output_path = Path(output_dir)
|
||||
if output_path.exists():
|
||||
if force_recreate:
|
||||
shutil.rmtree(output_path)
|
||||
else:
|
||||
print(
|
||||
f"Output directory {output_dir} already exists. "
|
||||
"Use --force-recreate to overwrite."
|
||||
)
|
||||
return str(output_path)
|
||||
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
print("Loading original model configuration...")
|
||||
original_config = AutoConfig.from_pretrained(
|
||||
original_model_name, trust_remote_code=True
|
||||
)
|
||||
print("Creating reduced configuration...")
|
||||
reduced_config = create_reduced_config(
|
||||
original_config, text_layers, num_experts, vision_layers
|
||||
)
|
||||
|
||||
config_path = output_path / "config.json"
|
||||
with open(config_path, "w") as f:
|
||||
json.dump(reduced_config, f, indent=2)
|
||||
print(f"Saved reduced config to {config_path}")
|
||||
|
||||
print("Copying tokenizer files...")
|
||||
copy_tokenizer_files(original_model_name, output_path)
|
||||
|
||||
print("Creating reduced safetensors files...")
|
||||
create_reduced_safetensors(original_config, reduced_config, output_path)
|
||||
|
||||
print("Creating preprocessor config...")
|
||||
create_preprocessor_config(original_config, output_path)
|
||||
|
||||
try:
|
||||
gen_config = GenerationConfig.from_pretrained(original_model_name)
|
||||
gen_config.save_pretrained(output_path)
|
||||
print("Copied generation config")
|
||||
except Exception as e:
|
||||
print(f"Could not copy generation config: {e}")
|
||||
|
||||
print(f"Successfully created reduced Maverick model at {output_path}")
|
||||
return str(output_path)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating reduced model: {e}")
|
||||
# Clean up on failure
|
||||
if output_path.exists():
|
||||
shutil.rmtree(output_path)
|
||||
raise
|
||||
|
||||
|
||||
def create_reduced_config(
|
||||
original_config: Any, text_layers: int, num_experts: int, vision_layers: int
|
||||
) -> dict[str, Any]:
|
||||
"""Create a reduced configuration based on the original."""
|
||||
# Convert config to dictionary
|
||||
config_dict = original_config.to_dict()
|
||||
|
||||
# Reduce text layers
|
||||
if "text_config" in config_dict:
|
||||
original_text_layers = config_dict["text_config"]["num_hidden_layers"]
|
||||
config_dict["text_config"]["num_hidden_layers"] = text_layers
|
||||
original_layer_types = config_dict["text_config"]["layer_types"]
|
||||
config_dict["text_config"]["layer_types"] = original_layer_types[:text_layers]
|
||||
print(f"Reduced text layers from {original_text_layers} to {text_layers}")
|
||||
|
||||
original_num_experts = config_dict["text_config"]["num_local_experts"]
|
||||
config_dict["text_config"]["num_local_experts"] = num_experts
|
||||
print(f"Reduced num experts from {original_num_experts} to {num_experts}")
|
||||
|
||||
hidden_dim_divisor = 4
|
||||
|
||||
original_hidden_size = config_dict["text_config"]["hidden_size"]
|
||||
new_hidden_size = original_hidden_size // hidden_dim_divisor
|
||||
config_dict["text_config"]["hidden_size"] = new_hidden_size
|
||||
print(f"Reduced hidden size from {original_hidden_size} to {new_hidden_size}")
|
||||
|
||||
original_head_dim = config_dict["text_config"]["head_dim"]
|
||||
new_head_dim = original_head_dim // hidden_dim_divisor
|
||||
config_dict["text_config"]["head_dim"] = new_head_dim
|
||||
print(f"Reduced head dim from {original_head_dim} to {new_head_dim}")
|
||||
|
||||
# Reduce vision layers
|
||||
if "vision_config" in config_dict:
|
||||
original_vision_layers = config_dict["vision_config"]["num_hidden_layers"]
|
||||
config_dict["vision_config"]["num_hidden_layers"] = vision_layers
|
||||
print(f"Reduced vision layers from {original_vision_layers} to {vision_layers}")
|
||||
|
||||
# Update model name to indicate it's a reduced version
|
||||
config_dict["_name_or_path"] = f"reduced_maverick_{text_layers}t_{vision_layers}v"
|
||||
|
||||
return config_dict
|
||||
|
||||
|
||||
def copy_tokenizer_files(original_model_name: str, output_path: Path) -> None:
|
||||
"""Copy tokenizer files from the original model."""
|
||||
try:
|
||||
tokenizer = AutoTokenizer.from_pretrained(
|
||||
original_model_name, trust_remote_code=True
|
||||
)
|
||||
tokenizer.save_pretrained(output_path)
|
||||
print("Tokenizer files copied successfully")
|
||||
except Exception as e:
|
||||
print(f"Warning: Could not copy tokenizer files: {e}")
|
||||
|
||||
|
||||
def create_preprocessor_config(original_config: Any, output_path: Path) -> None:
|
||||
"""Create preprocessor_config.json for multimodal model."""
|
||||
# Try to load the original preprocessor config
|
||||
try:
|
||||
processor = AutoProcessor.from_pretrained(
|
||||
original_config._name_or_path
|
||||
or "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
|
||||
trust_remote_code=True,
|
||||
)
|
||||
processor.save_pretrained(output_path)
|
||||
print("Copied original preprocessor config")
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"Could not copy original preprocessor config: {e}")
|
||||
raise
|
||||
|
||||
|
||||
def create_reduced_safetensors(
|
||||
original_config: Any, reduced_config: dict[str, Any], output_path: Path
|
||||
) -> None:
|
||||
"""Create safetensors files with weights for the reduced model."""
|
||||
print("Generating synthetic weights for reduced model...")
|
||||
|
||||
text_config = reduced_config["text_config"]
|
||||
vision_config = reduced_config["vision_config"]
|
||||
|
||||
weights = {}
|
||||
|
||||
print("Creating text model weights...")
|
||||
weights.update(create_text_model_weights(text_config))
|
||||
|
||||
print("Creating vision model weights...")
|
||||
weights.update(create_vision_model_weights(vision_config))
|
||||
|
||||
print("Creating shared model weights...")
|
||||
weights.update(create_shared_weights(text_config, vision_config))
|
||||
|
||||
print("Saving weights to safetensors files...")
|
||||
save_weights_to_safetensors(weights, output_path)
|
||||
|
||||
|
||||
def create_text_model_weights(text_config: dict[str, Any]) -> dict[str, torch.Tensor]:
|
||||
"""Create synthetic weights for the text model with MoE structure."""
|
||||
weights = {}
|
||||
|
||||
vocab_size = text_config["vocab_size"]
|
||||
hidden_size = text_config["hidden_size"]
|
||||
intermediate_size = text_config["intermediate_size"]
|
||||
intermediate_size_mlp = text_config["intermediate_size_mlp"]
|
||||
num_layers = text_config["num_hidden_layers"]
|
||||
num_attention_heads = text_config["num_attention_heads"]
|
||||
num_key_value_heads = text_config.get("num_key_value_heads", num_attention_heads)
|
||||
|
||||
# MoE specific parameters
|
||||
num_experts = text_config.get("num_local_experts")
|
||||
assert num_experts is not None, "num_local_experts must be specified for MoE"
|
||||
|
||||
head_dim = hidden_size // num_attention_heads
|
||||
|
||||
# Embedding layers
|
||||
weights["language_model.model.embed_tokens.weight"] = torch.randn(
|
||||
vocab_size, hidden_size, dtype=torch.float16
|
||||
)
|
||||
|
||||
# Transformer layers
|
||||
for layer_idx in range(num_layers):
|
||||
layer_prefix = f"language_model.model.layers.{layer_idx}"
|
||||
print(f"Creating weights for layer {layer_prefix}...")
|
||||
|
||||
# Self-attention weights (separate q, k, v projections)
|
||||
weights[f"{layer_prefix}.self_attn.q_proj.weight"] = torch.randn(
|
||||
num_attention_heads * head_dim, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.k_proj.weight"] = torch.randn(
|
||||
num_key_value_heads * head_dim, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.v_proj.weight"] = torch.randn(
|
||||
num_key_value_heads * head_dim, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.o_proj.weight"] = torch.randn(
|
||||
hidden_size, num_attention_heads * head_dim, dtype=torch.bfloat16
|
||||
)
|
||||
print("Self-attention weights created.")
|
||||
|
||||
# Feed-forward weights - MoE pattern based on interleave_moe_layer_step
|
||||
# For interleave_moe_layer_step=2: layers 1,3,5,... are MoE, layers
|
||||
# 0,2,4,... are dense
|
||||
interleave_step = text_config.get("interleave_moe_layer_step", 1)
|
||||
is_moe_layer = interleave_step > 0 and (layer_idx + 1) % interleave_step == 0
|
||||
|
||||
if is_moe_layer:
|
||||
# MoE layer structure
|
||||
# 1. Router weights
|
||||
weights[f"{layer_prefix}.feed_forward.router.weight"] = torch.randn(
|
||||
num_experts, hidden_size, dtype=torch.float16
|
||||
)
|
||||
|
||||
# 2. Individual expert weights (not fused)
|
||||
for expert_idx in range(num_experts):
|
||||
expert_prefix = f"{layer_prefix}.feed_forward.experts.{expert_idx}"
|
||||
|
||||
weights[f"{expert_prefix}.gate_proj.weight"] = torch.randn(
|
||||
intermediate_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{expert_prefix}.up_proj.weight"] = torch.randn(
|
||||
intermediate_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{expert_prefix}.down_proj.weight"] = torch.randn(
|
||||
hidden_size, intermediate_size, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Expert weight scales (FP8 quantization)
|
||||
weights[f"{expert_prefix}.gate_proj.weight_scale"] = torch.ones(
|
||||
intermediate_size, 1, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{expert_prefix}.up_proj.weight_scale"] = torch.ones(
|
||||
intermediate_size, 1, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{expert_prefix}.down_proj.weight_scale"] = torch.ones(
|
||||
hidden_size, 1, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# 3. Shared expert weights
|
||||
shared_expert_prefix = f"{layer_prefix}.feed_forward.shared_expert"
|
||||
weights[f"{shared_expert_prefix}.gate_proj.weight"] = torch.randn(
|
||||
intermediate_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{shared_expert_prefix}.up_proj.weight"] = torch.randn(
|
||||
intermediate_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{shared_expert_prefix}.down_proj.weight"] = torch.randn(
|
||||
hidden_size, intermediate_size, dtype=torch.bfloat16
|
||||
)
|
||||
print(f"MoE feed-forward weights created for layer {layer_idx}.")
|
||||
else:
|
||||
# Dense layer structure
|
||||
weights[f"{layer_prefix}.feed_forward.gate_proj.weight"] = torch.randn(
|
||||
intermediate_size_mlp, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.feed_forward.up_proj.weight"] = torch.randn(
|
||||
intermediate_size_mlp, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.feed_forward.down_proj.weight"] = torch.randn(
|
||||
hidden_size, intermediate_size_mlp, dtype=torch.bfloat16
|
||||
)
|
||||
print(f"Dense feed-forward weights created for layer {layer_idx}.")
|
||||
|
||||
# Layer norms
|
||||
weights[f"{layer_prefix}.input_layernorm.weight"] = torch.ones(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.post_attention_layernorm.weight"] = torch.ones(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
print("Layer norms created.")
|
||||
|
||||
# Final layer norm and output projection
|
||||
weights["language_model.model.norm.weight"] = torch.ones(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights["language_model.lm_head.weight"] = torch.randn(
|
||||
vocab_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
return weights
|
||||
|
||||
|
||||
def create_vision_model_weights(
|
||||
vision_config: dict[str, Any],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Create synthetic weights for the vision model."""
|
||||
weights = {}
|
||||
|
||||
hidden_size = vision_config["hidden_size"]
|
||||
intermediate_size = vision_config["intermediate_size"]
|
||||
num_layers = vision_config["num_hidden_layers"]
|
||||
|
||||
# Vision transformer layers
|
||||
for layer_idx in range(num_layers):
|
||||
layer_prefix = f"vision_model.model.layers.{layer_idx}"
|
||||
|
||||
weights[f"{layer_prefix}.self_attn.q_proj.weight"] = torch.randn(
|
||||
hidden_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.q_proj.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.k_proj.weight"] = torch.randn(
|
||||
hidden_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.k_proj.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.v_proj.weight"] = torch.randn(
|
||||
hidden_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.v_proj.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.o_proj.weight"] = torch.randn(
|
||||
hidden_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.self_attn.o_proj.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
weights[f"{layer_prefix}.mlp.fc1.weight"] = torch.randn(
|
||||
intermediate_size, hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.mlp.fc1.bias"] = torch.zeros(
|
||||
intermediate_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.mlp.fc2.weight"] = torch.randn(
|
||||
hidden_size, intermediate_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.mlp.fc2.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
weights[f"{layer_prefix}.input_layernorm.weight"] = torch.ones(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.input_layernorm.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.post_attention_layernorm.weight"] = torch.ones(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
weights[f"{layer_prefix}.post_attention_layernorm.bias"] = torch.zeros(
|
||||
hidden_size, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
return weights
|
||||
|
||||
|
||||
def create_shared_weights(
|
||||
text_config: dict[str, Any], vision_config: dict[str, Any]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Create weights for shared components (vision-language connector)."""
|
||||
weights = {}
|
||||
|
||||
text_hidden_size = text_config["hidden_size"]
|
||||
projector_input_dim = vision_config["projector_input_dim"]
|
||||
|
||||
# Vision-language connector (projects vision features to text space)
|
||||
weights["multi_modal_projector.linear_1.weight"] = torch.randn(
|
||||
text_hidden_size, projector_input_dim, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
return weights
|
||||
|
||||
|
||||
def save_weights_to_safetensors(
|
||||
weights: dict[str, torch.Tensor], output_path: Path
|
||||
) -> None:
|
||||
"""Save weights to safetensors files and create index."""
|
||||
# Determine how to shard the weights
|
||||
max_shard_size = 5 * 1024 * 1024 * 1024 # 5GB per shard
|
||||
|
||||
# Calculate sizes and create shards
|
||||
shards = []
|
||||
current_shard: dict[str, torch.Tensor] = {}
|
||||
current_size = 0
|
||||
|
||||
for name, tensor in weights.items():
|
||||
tensor_size = tensor.numel() * tensor.element_size()
|
||||
|
||||
if current_size + tensor_size > max_shard_size and current_shard:
|
||||
shards.append(current_shard)
|
||||
current_shard = {}
|
||||
current_size = 0
|
||||
|
||||
current_shard[name] = tensor
|
||||
current_size += tensor_size
|
||||
|
||||
if current_shard:
|
||||
shards.append(current_shard)
|
||||
|
||||
# Save shards and create index
|
||||
weight_map = {}
|
||||
|
||||
if len(shards) == 1:
|
||||
# Single file
|
||||
filename = "model.safetensors"
|
||||
save_file(shards[0], output_path / filename)
|
||||
weight_map = {name: filename for name in shards[0]}
|
||||
print(f"Saved weights to single file: {filename}")
|
||||
else:
|
||||
# Multiple shards
|
||||
for i, shard in enumerate(shards):
|
||||
filename = f"model-{i + 1:05d}-of-{len(shards):05d}.safetensors"
|
||||
save_file(shard, output_path / filename)
|
||||
for name in shard:
|
||||
weight_map[name] = filename
|
||||
print(f"Saved shard {i + 1}/{len(shards)}: {filename}")
|
||||
|
||||
# Create index file
|
||||
index_data = {
|
||||
"metadata": {
|
||||
"total_size": sum(
|
||||
tensor.numel() * tensor.element_size() for tensor in weights.values()
|
||||
)
|
||||
},
|
||||
"weight_map": weight_map,
|
||||
}
|
||||
|
||||
index_path = output_path / "model.safetensors.index.json"
|
||||
with open(index_path, "w") as f:
|
||||
json.dump(index_data, f, indent=2)
|
||||
|
||||
print(f"Created index file: {index_path}")
|
||||
print(
|
||||
f"Total model size: {index_data['metadata']['total_size'] / (1024**3):.2f} GB"
|
||||
)
|
||||
|
||||
|
||||
def check_attention_spec_interleaved_rope(
|
||||
llm: LLM,
|
||||
num_attention_layers: int,
|
||||
num_ranks: int,
|
||||
rope_layers: list[int],
|
||||
):
|
||||
"""Check that the attention spec is correct."""
|
||||
assert isinstance(llm.llm_engine.model_executor, Executor)
|
||||
kv_cache_specs_per_rank = llm.llm_engine.model_executor.get_kv_cache_specs()
|
||||
for rank in range(num_ranks):
|
||||
kv_cache_specs = kv_cache_specs_per_rank[rank]
|
||||
assert len(kv_cache_specs.keys()) == num_attention_layers
|
||||
for i in range(num_attention_layers):
|
||||
if rope_layers[i] == 0:
|
||||
expected_spec = FullAttentionSpec
|
||||
else:
|
||||
expected_spec = ChunkedLocalAttentionSpec
|
||||
assert isinstance(
|
||||
kv_cache_specs[f"language_model.model.layers.{i}.self_attn.attn"],
|
||||
expected_spec,
|
||||
)
|
||||
|
||||
|
||||
def run_reduced_model(llm: LLM, should_profile: bool = False) -> None:
|
||||
"""Test the created reduced model with vLLM."""
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=50)
|
||||
|
||||
if should_profile:
|
||||
llm.start_profile()
|
||||
outputs = llm.generate(PROMPTS, sampling_params)
|
||||
if should_profile:
|
||||
llm.stop_profile()
|
||||
|
||||
print("Test generation successful!")
|
||||
for output in outputs:
|
||||
print(f"Prompt: {output.prompt}")
|
||||
print(f"Output: {output.outputs[0].text}")
|
||||
print("-" * 40)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"original_model_name,text_layers,num_experts,vision_layers,",
|
||||
[("meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", 4, 4, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("enforce_eager", [True, False])
|
||||
@pytest.mark.parametrize("tp,ep", [(2, True)])
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
def test_dummy_maverick(
|
||||
monkeypatch,
|
||||
original_model_name: str,
|
||||
text_layers: int,
|
||||
num_experts: int,
|
||||
vision_layers: int,
|
||||
enforce_eager: bool,
|
||||
tp: int,
|
||||
ep: bool,
|
||||
output_dir: str = "/tmp/reduced_maverick",
|
||||
force_recreate: bool = True,
|
||||
profile: bool = False,
|
||||
) -> None:
|
||||
# Disable multiprocessing allows us to access model executor from LLM engine
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
# The forked test process may inherit CUDA state from test collection.
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
model_path = create_reduced_maverick_model(
|
||||
original_model_name=original_model_name,
|
||||
output_dir=output_dir,
|
||||
text_layers=text_layers,
|
||||
num_experts=num_experts,
|
||||
vision_layers=vision_layers,
|
||||
force_recreate=force_recreate,
|
||||
)
|
||||
|
||||
print(f"\nReduced model created successfully at: {model_path}")
|
||||
|
||||
rope_layers = get_rope_layers_config(model_path)
|
||||
|
||||
llm = LLM(
|
||||
model=model_path,
|
||||
trust_remote_code=True,
|
||||
max_model_len=512, # Small context for testing
|
||||
gpu_memory_utilization=0.3, # Conservative memory usage
|
||||
enforce_eager=enforce_eager,
|
||||
tensor_parallel_size=tp,
|
||||
enable_expert_parallel=ep,
|
||||
)
|
||||
|
||||
check_attention_spec_interleaved_rope(
|
||||
llm,
|
||||
text_layers,
|
||||
tp,
|
||||
rope_layers,
|
||||
)
|
||||
|
||||
print(f"\nTesting reduced model at {model_path}...")
|
||||
run_reduced_model(llm=llm, should_profile=profile)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to create and test the reduced model."""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Create a reduced-layer Maverick model"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="/tmp/reduced_maverick",
|
||||
help="Output directory for the reduced model",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text-layers",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Number of text transformer layers",
|
||||
)
|
||||
parser.add_argument("--num-experts", type=int, default=4, help="Number of experts")
|
||||
parser.add_argument(
|
||||
"--vision-layers",
|
||||
type=int,
|
||||
default=2,
|
||||
help="Number of vision transformer layers",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force-recreate",
|
||||
action="store_true",
|
||||
help="Force recreation if output directory exists",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test", action="store_true", help="Test the created model with vLLM"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profile", action="store_true", help="Profile the created model with vLLM"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test-original",
|
||||
action="store_true",
|
||||
help="Test the original model with vLLM",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--original-model",
|
||||
default="meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
|
||||
help="Original model name to base the reduction on",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.test:
|
||||
test_dummy_maverick(
|
||||
original_model_name=args.original_model,
|
||||
output_dir=args.output_dir,
|
||||
text_layers=args.text_layers,
|
||||
num_experts=args.num_experts,
|
||||
vision_layers=args.vision_layers,
|
||||
force_recreate=args.force_recreate,
|
||||
tp=2,
|
||||
ep=True,
|
||||
enforce_eager=True,
|
||||
profile=args.profile,
|
||||
)
|
||||
|
||||
if args.test_original:
|
||||
run_maverick_serving(args.original_model)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(main())
|
||||
@@ -1,183 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gc
|
||||
import random
|
||||
import string
|
||||
import sys
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.utils.mem_constants import KiB_bytes, MiB_bytes
|
||||
from vllm.utils.mem_utils import format_mib
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-VL-4B-Instruct"
|
||||
RANDOM_PREFIX_LEN = 100
|
||||
TEST_IMAGE_NAMES = [
|
||||
"2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
|
||||
"Grayscale_8bits_palette_sample_image.png",
|
||||
]
|
||||
MAX_MODEL_LEN = 8192
|
||||
REQUESTS_PER_ROUND = 4
|
||||
WARMUP_ROUNDS = 2
|
||||
MEASURED_ROUNDS = 16
|
||||
GPU_GROWTH_THRESHOLD_MIB = 0
|
||||
CPU_PEAK_GROWTH_THRESHOLD_MIB = 0
|
||||
|
||||
SAMPLING_PARAMS = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=16,
|
||||
)
|
||||
|
||||
|
||||
def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
|
||||
# Avoid obscuring memory leaks because of prefix caching
|
||||
random_text = "".join(random.choices(string.ascii_uppercase, k=RANDOM_PREFIX_LEN))
|
||||
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Ignore this random string: {random_text}",
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image in one short sentence.",
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _build_request_batch(
|
||||
image_urls: list[str],
|
||||
) -> list[list[ChatCompletionMessageParam]]:
|
||||
return [
|
||||
_make_messages(image_urls[i % len(image_urls)])
|
||||
for i in range(REQUESTS_PER_ROUND)
|
||||
]
|
||||
|
||||
|
||||
def _ru_maxrss_bytes() -> int | None:
|
||||
try:
|
||||
import resource
|
||||
except ImportError:
|
||||
return None
|
||||
|
||||
rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
if rss <= 0:
|
||||
return 0
|
||||
|
||||
# Linux reports kilobytes, macOS reports bytes.
|
||||
return rss if sys.platform == "darwin" else rss * KiB_bytes
|
||||
|
||||
|
||||
def _gpu_used_bytes() -> int:
|
||||
torch.accelerator.synchronize()
|
||||
free_bytes, total_bytes = torch.accelerator.get_memory_info()
|
||||
return int(total_bytes - free_bytes)
|
||||
|
||||
|
||||
def _format_mib(num_bytes: int | None) -> str:
|
||||
if num_bytes is None:
|
||||
return "n/a"
|
||||
|
||||
return f"{format_mib(num_bytes)} MiB"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def llm(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
# pytest caches the fixture so we use weakref.proxy to
|
||||
# enable garbage collection
|
||||
llm_kwargs = dict(
|
||||
model=MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_num_seqs=REQUESTS_PER_ROUND,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
seed=0,
|
||||
disable_log_stats=True,
|
||||
gpu_memory_utilization=0.8,
|
||||
)
|
||||
if current_platform.is_rocm():
|
||||
llm_kwargs["attention_backend"] = "TRITON_ATTN"
|
||||
|
||||
llm = LLM(**llm_kwargs)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_NAMES], indirect=True)
|
||||
def test_no_memory_leak(llm, image_urls: list[str]) -> None:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(MODEL_NAME)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
request_batch = _build_request_batch(image_urls)
|
||||
|
||||
# Establish a warmup baseline after model load and the first multimodal
|
||||
# requests complete. Later rounds should remain near this steady state.
|
||||
for _ in range(WARMUP_ROUNDS):
|
||||
outputs = llm.chat(request_batch, sampling_params=SAMPLING_PARAMS)
|
||||
assert len(outputs) == len(request_batch)
|
||||
assert llm.llm_engine.get_num_unfinished_requests() == 0
|
||||
del outputs
|
||||
|
||||
gc.collect()
|
||||
warmup_gpu = _gpu_used_bytes()
|
||||
warmup_cpu_peak = _ru_maxrss_bytes()
|
||||
|
||||
post_warmup_gpu_samples: list[int] = []
|
||||
post_warmup_cpu_peak_samples: list[int] = []
|
||||
|
||||
for _ in range(MEASURED_ROUNDS):
|
||||
outputs = llm.chat(request_batch, sampling_params=SAMPLING_PARAMS)
|
||||
assert len(outputs) == len(request_batch)
|
||||
assert llm.llm_engine.get_num_unfinished_requests() == 0
|
||||
del outputs
|
||||
|
||||
gc.collect()
|
||||
post_warmup_gpu_samples.append(_gpu_used_bytes())
|
||||
cpu_peak = _ru_maxrss_bytes()
|
||||
if cpu_peak is not None:
|
||||
post_warmup_cpu_peak_samples.append(cpu_peak)
|
||||
|
||||
gpu_growth = max(post_warmup_gpu_samples) - warmup_gpu
|
||||
gpu_threshold = GPU_GROWTH_THRESHOLD_MIB * MiB_bytes
|
||||
|
||||
assert gpu_growth <= gpu_threshold, (
|
||||
"Qwen3-VL GPU memory kept growing after warmup. "
|
||||
f"warmup_baseline={_format_mib(warmup_gpu)}, "
|
||||
f"post_warmup_samples={[_format_mib(x) for x in post_warmup_gpu_samples]}, "
|
||||
f"gpu_growth={_format_mib(gpu_growth)}, "
|
||||
f"gpu_threshold={GPU_GROWTH_THRESHOLD_MIB} MiB"
|
||||
)
|
||||
|
||||
if warmup_cpu_peak is not None and post_warmup_cpu_peak_samples:
|
||||
cpu_peak_growth = max(post_warmup_cpu_peak_samples) - warmup_cpu_peak
|
||||
cpu_threshold = CPU_PEAK_GROWTH_THRESHOLD_MIB * MiB_bytes
|
||||
|
||||
assert cpu_peak_growth <= cpu_threshold, (
|
||||
"Qwen3-VL CPU peak RSS kept growing after warmup. "
|
||||
f"warmup_ru_maxrss={_format_mib(warmup_cpu_peak)}, "
|
||||
f"post_warmup_ru_maxrss={[_format_mib(x) for x in post_warmup_cpu_peak_samples]}, " # noqa: E501
|
||||
f"cpu_peak_growth={_format_mib(cpu_peak_growth)}, "
|
||||
f"cpu_peak_threshold={CPU_PEAK_GROWTH_THRESHOLD_MIB} MiB"
|
||||
)
|
||||
@@ -1,122 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForImageTextToText
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
|
||||
from ....conftest import HfRunner, ImageTestAssets, VllmRunner
|
||||
from .vlm_utils import model_utils
|
||||
|
||||
MODEL = "google/gemma-3-4b-it"
|
||||
PROMPT = (
|
||||
"<bos><start_of_turn>user\n"
|
||||
"<start_of_image>What is the content in the center of the image?"
|
||||
"<end_of_turn>\n<start_of_turn>model\n"
|
||||
)
|
||||
|
||||
|
||||
def _install_prefill_hidden_capture(model):
|
||||
model = getattr(model, "module", model)
|
||||
model._prefill_hidden = None
|
||||
|
||||
language_model = model.language_model.model
|
||||
original_forward = language_model.forward
|
||||
|
||||
def forward(*args, **kwargs):
|
||||
hidden_states = original_forward(*args, **kwargs)
|
||||
if model._prefill_hidden is None and torch.is_tensor(hidden_states):
|
||||
# Capturing hidden states for comparison is a deliberate D2H.
|
||||
with gpu_sync_allowed():
|
||||
model._prefill_hidden = hidden_states.detach().float().cpu()
|
||||
return hidden_states
|
||||
|
||||
language_model.forward = forward
|
||||
|
||||
|
||||
def _get_prefill_hidden(model):
|
||||
model = getattr(model, "module", model)
|
||||
hidden = getattr(model, "_prefill_hidden", None)
|
||||
assert hidden is not None
|
||||
return hidden
|
||||
|
||||
|
||||
def _get_hf_prefill_hidden(hf_model: HfRunner, image: Any):
|
||||
inputs = hf_model.get_inputs([PROMPT], images=[image])[0]
|
||||
with torch.no_grad():
|
||||
outputs = hf_model.model.model(
|
||||
**hf_model.wrap_device(inputs),
|
||||
use_cache=False,
|
||||
)
|
||||
return outputs.last_hidden_state[0].detach().float().cpu()
|
||||
|
||||
|
||||
def _get_vllm_prefill_hidden(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image: Any,
|
||||
vllm_runner_kwargs: dict[str, Any],
|
||||
):
|
||||
with vllm_runner(
|
||||
MODEL,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
**vllm_runner_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_model.apply_model(_install_prefill_hidden_capture)
|
||||
vllm_model.generate_greedy([PROMPT], max_tokens=1, images=[image])
|
||||
return vllm_model.apply_model(_get_prefill_hidden)[0]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm(), reason="ROCm attention has accuracy issue for this test"
|
||||
)
|
||||
def test_mm_prefix_lm_e2e(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Regression: Gemma3 native prefill must apply image prefix-LM mask."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
image = image_assets[0].pil_image
|
||||
|
||||
vllm_runner_kwargs: dict[str, Any] = {
|
||||
"mm_processor_cache_gb": 0,
|
||||
"mm_processor_kwargs": {"do_pan_and_scan": True},
|
||||
}
|
||||
vllm_hidden = _get_vllm_prefill_hidden(vllm_runner, image, vllm_runner_kwargs)
|
||||
|
||||
hf_model = hf_runner(
|
||||
MODEL,
|
||||
auto_cls=AutoModelForImageTextToText,
|
||||
)
|
||||
hf_model = model_utils.gemma3_patch_hf_runner(hf_model)
|
||||
|
||||
with hf_model:
|
||||
hf_hidden = _get_hf_prefill_hidden(hf_model, image)
|
||||
|
||||
assert vllm_hidden.shape == hf_hidden.shape
|
||||
|
||||
full_cos = torch.nn.functional.cosine_similarity(
|
||||
vllm_hidden.flatten(), hf_hidden.flatten(), dim=0
|
||||
)
|
||||
image_cos = torch.nn.functional.cosine_similarity(
|
||||
vllm_hidden[1:769].flatten(), hf_hidden[1:769].flatten(), dim=0
|
||||
)
|
||||
|
||||
assert full_cos > 0.9, (
|
||||
"Gemma3 mm-prefix-LM full prefill hidden states should be close to HF; "
|
||||
f"got {full_cos=}"
|
||||
)
|
||||
assert image_cos > 0.9, (
|
||||
"Gemma3 mm-prefix-LM image prefill hidden states should be close to HF; "
|
||||
f"got {image_cos=}"
|
||||
)
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Generation tests for Moondream3 query and caption support."""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.registry import HF_EXAMPLE_MODELS
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, ImageTestAssets
|
||||
from ....utils import large_gpu_mark, multi_gpu_test
|
||||
|
||||
MOONDREAM3_MODEL_ID = "moondream/moondream3-preview"
|
||||
MOONDREAM3_TOKENIZER = "moondream/starmie-v1"
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color is the stop sign?<|md_reserved_2|>", # noqa: E501
|
||||
"cherry_blossom": "<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>What color are the flowers?<|md_reserved_2|>", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def make_query_prompt(question: str) -> str:
|
||||
"""Create a direct-answer query prompt for Moondream3."""
|
||||
return (
|
||||
"<|endoftext|><image><|md_reserved_0|>query<|md_reserved_1|>"
|
||||
f"{question}<|md_reserved_2|>"
|
||||
)
|
||||
|
||||
|
||||
def make_caption_prompt(length: str = "normal") -> str:
|
||||
"""Create a caption prompt for Moondream3."""
|
||||
return (
|
||||
"<|endoftext|><image><|md_reserved_0|>"
|
||||
f"describe<|md_reserved_1|>{length}<|md_reserved_2|>"
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@large_gpu_mark(min_gb=80)
|
||||
def test_tensor_parallel(image_assets: ImageTestAssets):
|
||||
import gc
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed.parallel_state import destroy_model_parallel
|
||||
|
||||
destroy_model_parallel()
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
llm = LLM(
|
||||
model=MOONDREAM3_MODEL_ID,
|
||||
tokenizer=MOONDREAM3_TOKENIZER,
|
||||
trust_remote_code=True,
|
||||
dtype="bfloat16",
|
||||
tensor_parallel_size=2,
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.45,
|
||||
)
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
prompt = make_query_prompt("What color is the stop sign?")
|
||||
|
||||
try:
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=20, temperature=0),
|
||||
)
|
||||
|
||||
assert len(outputs) > 0
|
||||
assert outputs[0].outputs[0].text is not None
|
||||
finally:
|
||||
del llm
|
||||
gc.collect()
|
||||
current_platform.empty_cache()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info("Moondream3ForCausalLM")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
from vllm import LLM
|
||||
|
||||
try:
|
||||
return LLM(
|
||||
model=MOONDREAM3_MODEL_ID,
|
||||
tokenizer=MOONDREAM3_TOKENIZER,
|
||||
trust_remote_code=True,
|
||||
dtype="bfloat16",
|
||||
max_model_len=2048,
|
||||
enforce_eager=True,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.45,
|
||||
)
|
||||
except Exception as exc:
|
||||
pytest.skip(f"Failed to load {MOONDREAM3_MODEL_ID}: {exc}")
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_model_loading(llm):
|
||||
assert llm is not None
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_query_skill(llm, image_assets: ImageTestAssets):
|
||||
from vllm import SamplingParams
|
||||
|
||||
image = image_assets[0].pil_image
|
||||
prompt = make_query_prompt("What color is the stop sign?")
|
||||
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=50, temperature=0),
|
||||
)
|
||||
|
||||
output_text = outputs[0].outputs[0].text
|
||||
assert output_text is not None
|
||||
assert len(output_text) > 0
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_caption_skill(llm, image_assets: ImageTestAssets):
|
||||
from vllm import SamplingParams
|
||||
|
||||
image = image_assets[1].pil_image
|
||||
prompt = make_caption_prompt()
|
||||
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=100, temperature=0),
|
||||
)
|
||||
|
||||
output_text = outputs[0].outputs[0].text
|
||||
assert output_text is not None
|
||||
assert len(output_text) > 0
|
||||
|
||||
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_batched_inference(llm, image_assets: ImageTestAssets):
|
||||
from vllm import SamplingParams
|
||||
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
prompts = [
|
||||
{"prompt": prompt, "multi_modal_data": {"image": img}}
|
||||
for img, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
outputs = llm.generate(prompts, SamplingParams(max_tokens=50, temperature=0))
|
||||
|
||||
assert len(outputs) == len(images)
|
||||
for output in outputs:
|
||||
assert output.outputs[0].text is not None
|
||||
assert len(output.outputs[0].text) > 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asset_name", ["stop_sign", "cherry_blossom"])
|
||||
@large_gpu_mark(min_gb=48)
|
||||
def test_image_assets(llm, image_assets: ImageTestAssets, asset_name: str):
|
||||
from vllm import SamplingParams
|
||||
|
||||
asset_idx = 0 if asset_name == "stop_sign" else 1
|
||||
image = image_assets[asset_idx].pil_image
|
||||
prompt = HF_IMAGE_PROMPTS[asset_idx]
|
||||
|
||||
outputs = llm.generate(
|
||||
{"prompt": prompt, "multi_modal_data": {"image": image}},
|
||||
SamplingParams(max_tokens=50, temperature=0),
|
||||
)
|
||||
|
||||
output_text = outputs[0].outputs[0].text
|
||||
assert output_text is not None
|
||||
assert len(output_text) > 0
|
||||
@@ -1,132 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
from transformers import AutoModel
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.logprobs import Logprob, SampleLogprobs
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from ....conftest import HfRunner, PromptImageInput, VllmRunner
|
||||
|
||||
IMAGE = ImageAsset("paper-11").pil_image_ext(ext="png").convert("RGB")
|
||||
PROMPT = (
|
||||
"</s><s><predict_bbox><predict_classes><output_markdown><predict_no_text_in_pic>"
|
||||
)
|
||||
|
||||
|
||||
class DummyLogprobs(dict[int, Logprob]):
|
||||
def __init__(self, vocab_ids: Iterable[int]):
|
||||
super().__init__(dict.fromkeys(vocab_ids, Logprob(0.0)))
|
||||
|
||||
def __repr__(self):
|
||||
return "DummyLogprobs()"
|
||||
|
||||
|
||||
def mask_bbox_tokens(
|
||||
output: tuple[list[int], str, SampleLogprobs],
|
||||
tokenizer: TokenizerLike,
|
||||
) -> tuple[list[int], str, SampleLogprobs]:
|
||||
"""Always pass check_logprobs_close check for bounding box tokens
|
||||
because it is reasonable for them to differ slightly.
|
||||
"""
|
||||
ignore_pattern = r"<[xy]_[\d.]+>"
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
output_ids, output_str, out_logprobs = output
|
||||
|
||||
masked_logprobs = list[dict[int, Logprob]]()
|
||||
for token, logprobs in zip(output_ids, out_logprobs):
|
||||
if re.match(ignore_pattern, tokenizer.decode(token)):
|
||||
masked_logprobs.append(DummyLogprobs(vocab.values()))
|
||||
else:
|
||||
masked_logprobs.append(logprobs)
|
||||
|
||||
return output_ids, output_str, masked_logprobs
|
||||
|
||||
|
||||
def run_test(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: Sequence[tuple[list[str], PromptImageInput]],
|
||||
model: str,
|
||||
*,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
"""Verify that the inference result is the same between hf and vllm."""
|
||||
with vllm_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
max_num_seqs=64,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
trust_remote_code=True,
|
||||
) as vllm_model:
|
||||
vllm_outputs_per_case = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
)
|
||||
for prompts, images in inputs
|
||||
]
|
||||
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
|
||||
with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs_per_case = [
|
||||
hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
images=images,
|
||||
tokenization_kwargs={"add_special_tokens": False},
|
||||
)
|
||||
for prompts, images in inputs
|
||||
]
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_case, vllm_outputs_per_case):
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[
|
||||
mask_bbox_tokens(output, tokenizer) for output in hf_outputs
|
||||
],
|
||||
outputs_1_lst=[
|
||||
mask_bbox_tokens(output, tokenizer) for output in vllm_outputs
|
||||
],
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
|
||||
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
|
||||
# instead of shipping a separate lm_head.weight tensor; this
|
||||
# exercises that path (see tie_weights() in nemotron_parse.py).
|
||||
"nvidia/NVIDIA-Nemotron-Parse-2.0",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
hf_runner, vllm_runner, model: str, dtype: str, num_logprobs: int
|
||||
) -> None:
|
||||
run_test(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
inputs=[
|
||||
([PROMPT] * 10, [IMAGE] * 10),
|
||||
],
|
||||
model=model,
|
||||
dtype=dtype,
|
||||
max_tokens=100,
|
||||
num_logprobs=num_logprobs,
|
||||
)
|
||||
@@ -1,261 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import VIDEO_ASSETS
|
||||
from .vlm_utils.builders import sample_frames_with_video_metadata
|
||||
|
||||
models = ["Qwen/Qwen2.5-VL-3B-Instruct"]
|
||||
target_dtype = "bfloat16"
|
||||
|
||||
VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
|
||||
|
||||
|
||||
def qwen2_5_vl_chat_template(*query):
|
||||
return f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{''.join(query)}<|im_end|><|im_start|>assistant\n" # noqa: E501
|
||||
|
||||
|
||||
VIDEO_PROMPTS = VIDEO_ASSETS.prompts(
|
||||
{
|
||||
"baby_reading": qwen2_5_vl_chat_template(
|
||||
VIDEO_PLACEHOLDER,
|
||||
"Describe this video with a short sentence ",
|
||||
"(no more than 20 words)",
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
WINDOW_ATTN_IMAGE_PROMPT = qwen2_5_vl_chat_template(
|
||||
IMAGE_PLACEHOLDER,
|
||||
"Describe the image.",
|
||||
)
|
||||
IMAGE_ONLY_LIMIT_MM_PER_PROMPT = {"image": 1, "video": 0}
|
||||
|
||||
|
||||
def _window_attention_regression_image():
|
||||
# image from regression issue: https://github.com/vllm-project/vllm/issues/15122
|
||||
image = ImageAsset("hato").pil_image
|
||||
return image.resize((image.width // 2, image.height // 2))
|
||||
|
||||
|
||||
def _encoder_cudagraph_config(*, max_vision_items: int) -> dict:
|
||||
return {
|
||||
"cudagraph_mm_encoder": True,
|
||||
"encoder_cudagraph_max_vision_items_per_batch": max_vision_items,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize(
|
||||
"video_pruning_rate", [0.0] if current_platform.is_cpu() else [0.0, 0.75]
|
||||
)
|
||||
@pytest.mark.parametrize("num_frames", [16])
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize(
|
||||
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
|
||||
)
|
||||
def test_qwen2_5_vl_evs_functionality(
|
||||
vllm_runner,
|
||||
video_assets,
|
||||
model,
|
||||
video_pruning_rate: float,
|
||||
num_frames: int,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
use_bytecode_hook: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Test EVS (Efficient Video Sampling) functionality with different
|
||||
pruning rates.
|
||||
"""
|
||||
# Set the environment variable for this test
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
|
||||
|
||||
# Sample frames from video assets
|
||||
sampled_vids = [
|
||||
sample_frames_with_video_metadata(
|
||||
(asset.np_ndarrays, asset.metadata), num_frames
|
||||
)
|
||||
for asset in video_assets
|
||||
]
|
||||
|
||||
prompts = [VIDEO_PROMPTS[0]]
|
||||
videos = [sampled_vids[0]]
|
||||
|
||||
# Initialize model with EVS configuration
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=4000,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
video_pruning_rate=video_pruning_rate,
|
||||
) as vllm_model:
|
||||
# Generate output - this should not crash
|
||||
outputs = vllm_model.generate_greedy(prompts, max_tokens, videos=videos)
|
||||
|
||||
# Basic validation that we got a response
|
||||
assert len(outputs) == 1
|
||||
output_ids, output_text = outputs[0]
|
||||
|
||||
# Ensure we got some output
|
||||
assert len(output_ids) > 0
|
||||
assert len(output_text) > 0
|
||||
|
||||
# Ensure the output is a string
|
||||
assert isinstance(output_text, str)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize(
|
||||
"video_pruning_rate", [0.0] if current_platform.is_cpu() else [0.0, 0.75]
|
||||
)
|
||||
@pytest.mark.parametrize("num_frames", [16])
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize(
|
||||
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
|
||||
)
|
||||
def test_qwen2_5_vl_evs_batched_videos(
|
||||
vllm_runner,
|
||||
video_assets,
|
||||
model,
|
||||
video_pruning_rate: float,
|
||||
num_frames: int,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
use_bytecode_hook: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Test EVS functionality with batched videos.
|
||||
|
||||
This test validates that:
|
||||
1. The model handles batched video inputs correctly with EVS
|
||||
2. Both pruning configurations work with multiple videos
|
||||
3. The model doesn't crash when processing multiple videos simultaneously
|
||||
"""
|
||||
# Set the environment variable for this test
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
|
||||
# Sample frames from video assets
|
||||
sampled_vids = [
|
||||
sample_frames_with_video_metadata(
|
||||
(asset.np_ndarrays, asset.metadata), num_frames
|
||||
)
|
||||
for asset in video_assets
|
||||
]
|
||||
|
||||
# Test batched videos
|
||||
prompts = [VIDEO_PROMPTS[0], VIDEO_PROMPTS[0]]
|
||||
videos = [sampled_vids[0], sampled_vids[0]] # Use same video twice for testing
|
||||
|
||||
# Initialize model with EVS configuration
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=4000,
|
||||
max_num_seqs=2,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt={"video": 2},
|
||||
tensor_parallel_size=1,
|
||||
video_pruning_rate=video_pruning_rate,
|
||||
) as vllm_model:
|
||||
# Generate output - this should not crash
|
||||
outputs = vllm_model.generate_greedy(prompts, max_tokens, videos=videos)
|
||||
|
||||
# Basic validation that we got responses for both videos
|
||||
assert len(outputs) == 2
|
||||
|
||||
for output_ids, output_text in outputs:
|
||||
# Ensure we got some output for each video
|
||||
assert len(output_ids) > 0
|
||||
assert len(output_text) > 0
|
||||
|
||||
# Ensure the output is a string
|
||||
assert isinstance(output_text, str)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize(
|
||||
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
|
||||
)
|
||||
def test_qwen2_5_vl_window_attention_image(
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
use_bytecode_hook: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Regression test for Qwen2.5 window-attention image path."""
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
|
||||
|
||||
prompt = [WINDOW_ATTN_IMAGE_PROMPT]
|
||||
images = [[_window_attention_regression_image()]]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt=IMAGE_ONLY_LIMIT_MM_PER_PROMPT,
|
||||
compilation_config=_encoder_cudagraph_config(max_vision_items=1),
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(prompt, max_tokens, images=images)
|
||||
|
||||
assert len(outputs) == 1
|
||||
output_ids, output_text = outputs[0]
|
||||
assert len(output_ids) > 0
|
||||
assert len(output_text) > 0
|
||||
assert isinstance(output_text, str)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("model", models)
|
||||
@pytest.mark.parametrize("dtype", [target_dtype])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize(
|
||||
"use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
|
||||
)
|
||||
def test_qwen2_5_vl_window_attention_image_batch(
|
||||
vllm_runner,
|
||||
model,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
use_bytecode_hook: bool,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Regression test window-attention with a small image batch."""
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
|
||||
|
||||
image = _window_attention_regression_image()
|
||||
prompts = [WINDOW_ATTN_IMAGE_PROMPT, WINDOW_ATTN_IMAGE_PROMPT]
|
||||
images = [[image], [image]]
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="generate",
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt=IMAGE_ONLY_LIMIT_MM_PER_PROMPT,
|
||||
compilation_config=_encoder_cudagraph_config(max_vision_items=2),
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(prompts, max_tokens, images=images)
|
||||
|
||||
assert len(outputs) == 2
|
||||
for output_ids, output_text in outputs:
|
||||
assert len(output_ids) > 0
|
||||
assert len(output_text) > 0
|
||||
assert isinstance(output_text, str)
|
||||
@@ -1,674 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal.video import sample_frames_from_video
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....conftest import IMAGE_ASSETS, VIDEO_ASSETS
|
||||
from ...utils import dummy_hf_overrides
|
||||
from .vlm_utils.builders import sample_frames_with_video_metadata
|
||||
|
||||
|
||||
@dataclass
|
||||
class VitCudagraphTestConfig:
|
||||
model: str
|
||||
modalities: list[str] = field(default_factory=lambda: ["image", "video"])
|
||||
image_prompt: str | None = None
|
||||
video_prompt: str | None = None
|
||||
dtype: str = "bfloat16"
|
||||
max_model_len: int = 4096
|
||||
max_tokens: int = 64
|
||||
max_num_seqs: int = 2
|
||||
num_video_frames: int = 16
|
||||
needs_video_metadata: bool = False
|
||||
vllm_runner_kwargs: dict = field(default_factory=dict)
|
||||
compilation_config_overrides: dict = field(default_factory=dict)
|
||||
marks: list = field(default_factory=list)
|
||||
skip: bool = False
|
||||
|
||||
|
||||
def params_with_marks(
|
||||
configs: dict[str, VitCudagraphTestConfig],
|
||||
) -> list[pytest.param]:
|
||||
return [
|
||||
pytest.param(model_id, marks=cfg.marks) for model_id, cfg in configs.items()
|
||||
]
|
||||
|
||||
|
||||
def qwen_vl_chat_template(content: str) -> str:
|
||||
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
|
||||
def internvl_chat_template(content: str) -> str:
|
||||
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
|
||||
def kimi_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
f"<|im_user|>user<|im_middle|>{content}<|im_end|>"
|
||||
"<|im_assistant|>assistant<|im_middle|>"
|
||||
)
|
||||
|
||||
|
||||
def deepseek_v41_chat_template(content: str) -> str:
|
||||
return f"<|begin▁of▁sentence|><|User|>{content}<|Assistant|></think>"
|
||||
|
||||
|
||||
def step3_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
"<|begin▁of▁sentence|> You are a helpful assistant.<|BOT|>user\n "
|
||||
f"<im_patch>{content} <|EOT|><|BOT|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
def gemma3_chat_template(content: str) -> str:
|
||||
return f"<bos><start_of_turn>user\n{content}<end_of_turn>\n<start_of_turn>model\n"
|
||||
|
||||
|
||||
def deepseek_vl2_chat_template(content: str) -> str:
|
||||
return f"<|User|>: {content}\n\n<|Assistant|>:"
|
||||
|
||||
|
||||
def ernie45_vl_chat_template(content: str) -> str:
|
||||
return (
|
||||
f"<|begin_of_sentence|>User: {content}"
|
||||
"Picture 1:<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>\n"
|
||||
"Assistant: <think></think>"
|
||||
)
|
||||
|
||||
|
||||
def minicpmv_25_chat_template(content: str) -> str:
|
||||
"""Llama3-style chat template used by MiniCPM-V 2.5."""
|
||||
return (
|
||||
f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n"
|
||||
f"{content}"
|
||||
f"<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"
|
||||
)
|
||||
|
||||
|
||||
def minicpmv_chat_template(content: str) -> str:
|
||||
"""ChatML template used by MiniCPM-V 2.6 / 4.0 / 4.5."""
|
||||
return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
|
||||
MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = {
|
||||
"gemma3": VitCudagraphTestConfig(
|
||||
model="google/gemma-3-4b-it",
|
||||
modalities=["image"],
|
||||
image_prompt=gemma3_chat_template("<start_of_image>What is in this image?"),
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [512],
|
||||
},
|
||||
dtype="bfloat16",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"llama4": VitCudagraphTestConfig(
|
||||
model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
modalities=["image"],
|
||||
image_prompt=(
|
||||
"<|begin_of_text|><|header_start|>user<|header_end|>\n\n"
|
||||
"<|image|>What is in this image?<|eot|>"
|
||||
"<|header_start|>assistant<|header_end|>\n\n"
|
||||
),
|
||||
max_model_len=4096,
|
||||
max_tokens=32,
|
||||
max_num_seqs=2,
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="Llama4ForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen2_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen2-VL-2B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||
),
|
||||
video_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen2_5_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen2.5-VL-3B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||
),
|
||||
video_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"kimi_vl": VitCudagraphTestConfig(
|
||||
model="moonshotai/Kimi-VL-A3B-Instruct",
|
||||
modalities=["image"],
|
||||
image_prompt=kimi_vl_chat_template(
|
||||
"<|media_start|>image<|media_content|><|media_pad|><|media_end|>"
|
||||
"What is in this image?"
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
# Single bucket sized to cover the test images' output tokens.
|
||||
# The default auto-inferred range fans out into multiple power-of-2
|
||||
# buckets, each holding a full ViT capture pool.
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [1024],
|
||||
},
|
||||
# Shrink to 1 text + 1 vision layer with random weights so the
|
||||
# test runs on any CI GPU (incl. L4) and skips the multi-GiB
|
||||
# weight download. The test only validates that encoder CG
|
||||
# capture/replay functions correctly, not output quality.
|
||||
vllm_runner_kwargs={
|
||||
"trust_remote_code": True,
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="KimiVLForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minimax_m3": VitCudagraphTestConfig(
|
||||
model="MiniMaxAI/MiniMax-M3",
|
||||
modalities=["image"],
|
||||
image_prompt=("]~b]user\n]<]image[>[What is in this image?[e~[\n]~b]ai\n"),
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [512],
|
||||
},
|
||||
vllm_runner_kwargs={
|
||||
"trust_remote_code": True,
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="MiniMaxM3SparseForConditionalGeneration",
|
||||
),
|
||||
"mm_processor_kwargs": {"max_long_side_pixel": 336},
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen3_vl": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen3-VL-2B-Instruct",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||
),
|
||||
video_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"qwen3_5": VitCudagraphTestConfig(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
image_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>What is in this image?"
|
||||
),
|
||||
video_prompt=qwen_vl_chat_template(
|
||||
"<|vision_start|><|video_pad|><|vision_end|>"
|
||||
"Describe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=True,
|
||||
vllm_runner_kwargs={"enable_chunked_prefill": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"internvl": VitCudagraphTestConfig(
|
||||
model="OpenGVLab/InternVL3-1B",
|
||||
num_video_frames=8,
|
||||
image_prompt=internvl_chat_template("<image>\nWhat is in this image?"),
|
||||
video_prompt=internvl_chat_template(
|
||||
"<video>\nDescribe this video in one sentence."
|
||||
),
|
||||
needs_video_metadata=False,
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"idefics3": VitCudagraphTestConfig(
|
||||
model="HuggingFaceTB/SmolVLM-256M-Instruct",
|
||||
modalities=["image"],
|
||||
image_prompt=(
|
||||
"<|begin_of_text|>User:<image>What is in this image?"
|
||||
"<end_of_utterance>\nAssistant:"
|
||||
),
|
||||
max_model_len=4096,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [4096],
|
||||
},
|
||||
vllm_runner_kwargs={"gpu_memory_utilization": 0.80},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"deepseek_v41": VitCudagraphTestConfig(
|
||||
model="deepseek-ai/DeepSeek-V4.1-Flash",
|
||||
modalities=["image"],
|
||||
image_prompt=deepseek_v41_chat_template(
|
||||
"<|deepseek_image|>\n\nWhat is in this image?"
|
||||
),
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [1024],
|
||||
},
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"attention_backend": (
|
||||
"ROCM_FLASHMLA_SPARSE_DSV41"
|
||||
if current_platform.is_rocm()
|
||||
else "FLASHMLA_SPARSE_DSV41"
|
||||
),
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="DeepseekV41ForCausalLM",
|
||||
exist_overrides={"vision_n_layers": 1},
|
||||
),
|
||||
},
|
||||
),
|
||||
"step3_vl": VitCudagraphTestConfig(
|
||||
model="stepfun-ai/Step3-VL-10B",
|
||||
modalities=["image"],
|
||||
image_prompt=step3_vl_chat_template("What is in this image?"),
|
||||
# Single bucket sized to cover the largest test image's output
|
||||
# tokens (1152 > 1141 for cherry_blossom). The default auto-
|
||||
# inferred range fans out into multiple power-of-2 buckets, each
|
||||
# holding a full ViT capture pool.
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [1152],
|
||||
},
|
||||
# Shrink to 1 text + 1 vision layer with random weights so the
|
||||
# test runs on any CI GPU (incl. L4) and skips the 20 GiB weight
|
||||
# download. The test only validates that encoder CG capture/
|
||||
# replay functions correctly, not output quality.
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="StepVLForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
),
|
||||
"ernie45_vl": VitCudagraphTestConfig(
|
||||
model="baidu/ERNIE-4.5-VL-28B-A3B-PT",
|
||||
# Image only: Ernie's resampler applies a temporal conv for video
|
||||
# that changes the output token count, so video uses the eager path.
|
||||
modalities=["image"],
|
||||
image_prompt=ernie45_vl_chat_template("What is in this image?"),
|
||||
# Ernie4_5_VLMoeModel is deliberately not torch-compiled, since its
|
||||
# split of text and vision experts breaks compilation, so piecewise
|
||||
# cudagraphs have nothing to partition. Only the encoder graphs this
|
||||
# test covers are captured.
|
||||
compilation_config_overrides={
|
||||
"cudagraph_mode": 2,
|
||||
},
|
||||
# Shrink to 1 text + 1 vision layer with random weights so the test
|
||||
# runs on any CI GPU and skips the ~56 GiB weight download. The test
|
||||
# only validates encoder CG capture/replay, not output quality.
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"trust_remote_code": True,
|
||||
"revision": "refs/pr/17",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="Ernie4_5_VLMoeForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
),
|
||||
"glm4_1v": VitCudagraphTestConfig(
|
||||
model="zai-org/GLM-4.1V-9B-Thinking",
|
||||
image_prompt=(
|
||||
"[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
|
||||
"<|begin_of_image|><|image|><|end_of_image|>"
|
||||
"What is in this image?<|assistant|>assistant\n"
|
||||
),
|
||||
video_prompt=(
|
||||
"[gMASK]<sop><|system|>\nYou are a helpful assistant.<|user|>\n"
|
||||
"<|begin_of_video|><|video|><|end_of_video|>"
|
||||
"Describe this video in one sentence<|assistant|>assistant\n"
|
||||
),
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="Glm4vForConditionalGeneration",
|
||||
),
|
||||
},
|
||||
),
|
||||
"deepseek_ocr": VitCudagraphTestConfig(
|
||||
model="deepseek-ai/DeepSeek-OCR",
|
||||
modalities=["image"],
|
||||
image_prompt="<image>\nWhat is in this image?",
|
||||
marks=[pytest.mark.core_model],
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [272],
|
||||
"mode": 0,
|
||||
"cudagraph_mode": 2,
|
||||
},
|
||||
vllm_runner_kwargs={
|
||||
"load_format": "dummy",
|
||||
"hf_overrides": partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch="DeepseekOCRForCausalLM",
|
||||
),
|
||||
},
|
||||
skip=True, # TODO: Re-enable this once OOM issues are resolved on CI.
|
||||
),
|
||||
"deepseek_vl2": VitCudagraphTestConfig(
|
||||
model="deepseek-ai/deepseek-vl2-tiny",
|
||||
modalities=["image"],
|
||||
image_prompt=deepseek_vl2_chat_template("<image>\nWhat is in this image?"),
|
||||
vllm_runner_kwargs={
|
||||
"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]},
|
||||
},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"gemma4": VitCudagraphTestConfig(
|
||||
model="google/gemma-4-E2B-it",
|
||||
image_prompt=(
|
||||
"<bos><start_of_turn>user\n<|image|>\nWhat is in this image?<end_of_turn>\n"
|
||||
"<start_of_turn>model\n"
|
||||
),
|
||||
video_prompt=(
|
||||
"<bos><start_of_turn>user\n<|video|>\nDescribe this video in one sentence."
|
||||
"<end_of_turn>\n<start_of_turn>model\n"
|
||||
),
|
||||
# The 16-frame test video produces 1056 vision tokens. Capture only
|
||||
# the smallest supported bucket that covers it instead of all default
|
||||
# buckets through max_model_len, which adds unrelated memory pressure.
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [1120],
|
||||
},
|
||||
needs_video_metadata=True,
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_25": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-Llama3-V-2_5",
|
||||
modalities=["image"],
|
||||
image_prompt=minicpmv_25_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
# CI runs on 35GB MIG slices: every budget captures one graph per
|
||||
# patch-grid bucket, so the default budgets through max_model_len
|
||||
# OOM there. A small budget set covers the test images (each item
|
||||
# is at most (max_slice_num + 1) * query_num = 640 tokens).
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_26": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-V-2_6",
|
||||
image_prompt=minicpmv_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
video_prompt=minicpmv_chat_template(
|
||||
"(<video>./</video>)\nDescribe this video in one sentence."
|
||||
),
|
||||
max_model_len=2048,
|
||||
# Fewer frames keep the video item within the 1024 budget.
|
||||
num_video_frames=4,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
"encoder_cudagraph_max_frames_per_batch": 4,
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
"minicpmv_40": VitCudagraphTestConfig(
|
||||
model="openbmb/MiniCPM-V-4",
|
||||
image_prompt=minicpmv_chat_template(
|
||||
"(<image>./</image>)\nWhat is in this image?"
|
||||
),
|
||||
video_prompt=minicpmv_chat_template(
|
||||
"(<video>./</video>)\nDescribe this video in one sentence."
|
||||
),
|
||||
max_model_len=2048,
|
||||
num_video_frames=4,
|
||||
compilation_config_overrides={
|
||||
"encoder_cudagraph_token_budgets": [64, 1024],
|
||||
"encoder_cudagraph_max_frames_per_batch": 4,
|
||||
},
|
||||
vllm_runner_kwargs={"trust_remote_code": True},
|
||||
marks=[pytest.mark.core_model],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def get_compilation_config(config: VitCudagraphTestConfig):
|
||||
return {
|
||||
"cudagraph_mm_encoder": True,
|
||||
"encoder_cudagraph_max_vision_items_per_batch": 1,
|
||||
"encoder_cudagraph_max_frames_per_batch": 16,
|
||||
**config.compilation_config_overrides,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
|
||||
)
|
||||
def test_vit_cudagraph_image(model_id, vllm_runner, image_assets):
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
|
||||
if config.skip:
|
||||
pytest.skip(f"{model_id} is marked to be skipped.")
|
||||
|
||||
if "image" not in config.modalities:
|
||||
pytest.skip(f"{model_id} does not support the image modality.")
|
||||
|
||||
image_prompts = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": config.image_prompt, # type: ignore[typeddict-item]
|
||||
"cherry_blossom": config.image_prompt, # type: ignore[typeddict-item]
|
||||
}
|
||||
)
|
||||
images = [[asset.pil_image] for asset in image_assets]
|
||||
|
||||
with vllm_runner(
|
||||
config.model,
|
||||
dtype=config.dtype,
|
||||
max_model_len=config.max_model_len,
|
||||
max_num_seqs=config.max_num_seqs,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
compilation_config=get_compilation_config(config),
|
||||
**config.vllm_runner_kwargs,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(
|
||||
image_prompts, config.max_tokens, images=images
|
||||
)
|
||||
|
||||
# Basic validation that we got a response
|
||||
assert len(outputs) == 2
|
||||
output_ids, output_text = outputs[0]
|
||||
|
||||
# Ensure we got some output
|
||||
assert len(output_ids) > 0
|
||||
assert len(output_text) > 0
|
||||
|
||||
# Ensure the output is a string
|
||||
assert isinstance(output_text, str)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS))
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
|
||||
)
|
||||
def test_vit_cudagraph_video(model_id, vllm_runner, video_assets):
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
|
||||
if config.skip:
|
||||
pytest.skip(f"{model_id} is marked to be skipped.")
|
||||
|
||||
if "video" not in config.modalities:
|
||||
pytest.skip(f"{model_id} does not support the video modality")
|
||||
|
||||
video_prompts = VIDEO_ASSETS.prompts(
|
||||
{
|
||||
"baby_reading": config.video_prompt, # type: ignore[typeddict-item]
|
||||
}
|
||||
)
|
||||
if config.needs_video_metadata:
|
||||
sampled_vids = [
|
||||
sample_frames_with_video_metadata(
|
||||
(asset.np_ndarrays, asset.metadata), config.num_video_frames
|
||||
)
|
||||
for asset in video_assets
|
||||
]
|
||||
else:
|
||||
sampled_vids = [
|
||||
sample_frames_from_video(asset.np_ndarrays, config.num_video_frames)
|
||||
for asset in video_assets
|
||||
]
|
||||
videos = [sampled_vids[0]]
|
||||
|
||||
with vllm_runner(
|
||||
config.model,
|
||||
dtype=config.dtype,
|
||||
max_model_len=config.max_model_len,
|
||||
max_num_seqs=config.max_num_seqs,
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
compilation_config=get_compilation_config(config),
|
||||
**config.vllm_runner_kwargs,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(
|
||||
video_prompts, config.max_tokens, videos=videos
|
||||
)
|
||||
|
||||
# Basic validation that we got a response
|
||||
assert len(outputs) == 1
|
||||
output_ids, output_text = outputs[0]
|
||||
|
||||
# Ensure we got some output
|
||||
assert len(output_ids) > 0
|
||||
assert len(output_text) > 0
|
||||
|
||||
# Ensure the output is a string
|
||||
assert isinstance(output_text, str)
|
||||
|
||||
|
||||
def _check_eonly_encoder_outputs(worker, batches):
|
||||
"""Compare real E-only replay with eager, retaining outputs across replay."""
|
||||
import torch
|
||||
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
|
||||
runner = worker.model_runner
|
||||
assert isinstance(runner, MMEncoderModelRunner)
|
||||
manager = runner.model_state.encoder_runner.cudagraph_manager
|
||||
assert manager is not None and manager.is_captured()
|
||||
retained: list[tuple[torch.Tensor, torch.Tensor]] = []
|
||||
max_error = 0.0
|
||||
stats = []
|
||||
with torch.inference_mode():
|
||||
for batch in batches:
|
||||
kwargs = {key: value.to(runner.device) for key, value in batch.items()}
|
||||
actual = manager.execute(kwargs)
|
||||
expected = runner.model.embed_multimodal(**kwargs)
|
||||
assert len(actual) == len(expected)
|
||||
for output, reference in zip(actual, expected):
|
||||
assert torch.isfinite(output).all()
|
||||
# Two BF16 ulps at unit scale, fixed before model validation.
|
||||
torch.testing.assert_close(output, reference, rtol=0.016, atol=0.016)
|
||||
max_error = max(max_error, (output - reference).abs().max().item())
|
||||
for output, snapshot in retained:
|
||||
torch.testing.assert_close(output, snapshot, rtol=0, atol=0)
|
||||
retained.extend((output, output.clone()) for output in actual)
|
||||
stats.append(manager.get_cumulative_stats())
|
||||
assert stats[0]["graph_hits"] > 0
|
||||
assert stats[1]["graph_hits"] > stats[0]["graph_hits"]
|
||||
assert stats[2]["graph_misses"] > stats[1]["graph_misses"]
|
||||
assert stats[3]["graph_hits"] > stats[2]["graph_hits"]
|
||||
assert stats[4]["graph_hits"] > stats[3]["graph_hits"]
|
||||
assert stats[5]["graph_hits"] > stats[4]["graph_hits"]
|
||||
assert stats[6]["graph_hits"] > stats[5]["graph_hits"]
|
||||
return {"max_abs_error": max_error, "stats": stats}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
params_with_marks({key: MODEL_CONFIGS[key] for key in ("qwen3_vl", "qwen3_5")}),
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA graphs")
|
||||
def test_eonly_vit_cudagraph_outputs(
|
||||
model_id, vllm_runner, image_assets, video_assets, monkeypatch
|
||||
):
|
||||
"""Exercise startup capture, image/video replay, fallback and output ownership."""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
config = MODEL_CONFIGS[model_id]
|
||||
token_budgets = [64, 256, 2048]
|
||||
processor = AutoProcessor.from_pretrained(config.model)
|
||||
first, second = [asset.pil_image for asset in image_assets]
|
||||
batches = [
|
||||
[first.resize((224, 224))],
|
||||
[second.resize((448, 224)), first.resize((224, 224))],
|
||||
[second.resize((1792, 1792))],
|
||||
[second.resize((224, 224))],
|
||||
[first.resize((1280, 720))],
|
||||
[first.resize((224, 224))],
|
||||
]
|
||||
frames = sample_frames_from_video(video_assets[0].np_ndarrays, 2)
|
||||
video = np.stack(
|
||||
[np.asarray(Image.fromarray(frame).resize((224, 224))) for frame in frames]
|
||||
)
|
||||
with vllm_runner(
|
||||
config.model,
|
||||
dtype=config.dtype,
|
||||
mm_encoder_only=True,
|
||||
# FA padding NaNs are tracked separately in #57136.
|
||||
mm_encoder_attn_backend="FLASHINFER",
|
||||
enable_prefix_caching=False,
|
||||
max_model_len=4096,
|
||||
max_num_seqs=2,
|
||||
limit_mm_per_prompt={"image": 2, "video": 1},
|
||||
compilation_config={
|
||||
"cudagraph_mode": "NONE",
|
||||
"cudagraph_mm_encoder": True,
|
||||
"encoder_cudagraph_token_budgets": token_budgets,
|
||||
"encoder_cudagraph_max_vision_items_per_batch": 2,
|
||||
"encoder_cudagraph_max_frames_per_batch": 2,
|
||||
},
|
||||
) as model:
|
||||
mm_config = model.llm.llm_engine.vllm_config.model_config.multimodal_config
|
||||
processor_kwargs = mm_config.merge_mm_processor_kwargs({})
|
||||
inputs = [
|
||||
dict(
|
||||
processor.image_processor(
|
||||
images=images, return_tensors="pt", **processor_kwargs
|
||||
)
|
||||
)
|
||||
for images in batches
|
||||
]
|
||||
inputs.append(
|
||||
dict(
|
||||
processor.video_processor(
|
||||
videos=[video],
|
||||
do_sample_frames=False,
|
||||
return_tensors="pt",
|
||||
**processor_kwargs,
|
||||
)
|
||||
)
|
||||
)
|
||||
results = model.llm.collective_rpc(
|
||||
partial(_check_eonly_encoder_outputs, batches=inputs)
|
||||
)
|
||||
for result in results:
|
||||
assert result["stats"][0]["token_budgets"] == token_budgets
|
||||
print(f"E-only {model_id}: {results}")
|
||||
@@ -1,207 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Core test implementation to be shared across modalities."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .....conftest import HfRunner, VllmRunner
|
||||
from ....registry import HF_EXAMPLE_MODELS
|
||||
from .types import PromptWithMultiModalInput, RunnerOutput
|
||||
|
||||
|
||||
def run_test(
|
||||
*,
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
inputs: list[PromptWithMultiModalInput],
|
||||
model: str,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
enforce_eager: bool,
|
||||
max_model_len: int,
|
||||
max_num_seqs: int,
|
||||
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
|
||||
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None,
|
||||
auto_cls: type[_BaseAutoModelClass],
|
||||
use_tokenizer_eos: bool,
|
||||
comparator: Callable[..., None],
|
||||
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None,
|
||||
stop_str: list[str] | None,
|
||||
limit_mm_per_prompt: dict[str, int],
|
||||
vllm_runner_kwargs: dict[str, Any] | None,
|
||||
hf_model_kwargs: dict[str, Any] | None,
|
||||
hf_processor: Callable[[str], Any] | None,
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None,
|
||||
runner: RunnerOption = "auto",
|
||||
distributed_executor_backend: str | None = None,
|
||||
tensor_parallel_size: int = 1,
|
||||
vllm_embeddings: torch.Tensor | None = None,
|
||||
):
|
||||
"""Modality agnostic test executor for comparing HF/vLLM outputs."""
|
||||
# In the case of embeddings, vLLM takes separate input tensors
|
||||
vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
|
||||
# Disable other modalities to save memory
|
||||
default_limits = {"image": 0, "video": 0, "audio": 0}
|
||||
limit_mm_per_prompt = default_limits | limit_mm_per_prompt
|
||||
|
||||
vllm_outputs_per_mm = []
|
||||
hf_outputs_per_mm = []
|
||||
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
vllm_runner_kwargs_: dict[str, Any] = {"mm_processor_cache_gb": 0}
|
||||
if model_info.tokenizer:
|
||||
vllm_runner_kwargs_["tokenizer_name"] = model_info.tokenizer
|
||||
if model_info.tokenizer_mode:
|
||||
vllm_runner_kwargs_["tokenizer_mode"] = model_info.tokenizer_mode
|
||||
if model_info.hf_overrides:
|
||||
vllm_runner_kwargs_["hf_overrides"] = model_info.hf_overrides
|
||||
if model_info.require_embed_inputs:
|
||||
for k in ("skip_tokenizer_init", "enable_prompt_embeds", "enable_mm_embeds"):
|
||||
vllm_runner_kwargs_[k] = model_info.require_embed_inputs
|
||||
if not model_info.enable_prefix_caching:
|
||||
vllm_runner_kwargs_["enable_prefix_caching"] = False
|
||||
|
||||
if vllm_runner_kwargs:
|
||||
vllm_runner_kwargs_.update(vllm_runner_kwargs)
|
||||
|
||||
# Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
|
||||
# already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
|
||||
if "limit_mm_per_prompt" in vllm_runner_kwargs_:
|
||||
limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=max_model_len,
|
||||
max_num_seqs=max_num_seqs,
|
||||
dtype=dtype,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=enforce_eager,
|
||||
runner=runner,
|
||||
**vllm_runner_kwargs_,
|
||||
) as vllm_model:
|
||||
tokenizer = vllm_model.llm.get_tokenizer()
|
||||
|
||||
vllm_kwargs: dict[str, Any] = {}
|
||||
if get_stop_token_ids is not None:
|
||||
vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
|
||||
if stop_str:
|
||||
vllm_kwargs["stop"] = stop_str
|
||||
|
||||
for prompts, image_data, video_data, audio_data in vllm_inputs:
|
||||
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
|
||||
vllm_kwargs_with_mm_data = vllm_kwargs | mm_data
|
||||
vllm_output = vllm_model.generate_greedy_logprobs(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
**vllm_kwargs_with_mm_data,
|
||||
)
|
||||
vllm_outputs_per_mm.append(vllm_output)
|
||||
|
||||
hf_runner_kwargs: dict[str, Any] = {}
|
||||
if model_info.tokenizer:
|
||||
hf_runner_kwargs["tokenizer_name"] = model_info.tokenizer
|
||||
if hf_processor is not None:
|
||||
hf_runner_kwargs["processor"] = hf_processor(model)
|
||||
|
||||
hf_model = hf_runner(
|
||||
model,
|
||||
dtype=dtype,
|
||||
auto_cls=auto_cls,
|
||||
model_kwargs=hf_model_kwargs,
|
||||
**hf_runner_kwargs,
|
||||
)
|
||||
|
||||
# Some models need to patch things like the model processor, e.g., internvl
|
||||
if patch_hf_runner is not None:
|
||||
hf_model = patch_hf_runner(hf_model)
|
||||
|
||||
with hf_model, torch.no_grad():
|
||||
tokenizer = hf_model.tokenizer
|
||||
|
||||
# Some models need to explicitly pass the eos_token_id off the tokenizer
|
||||
# or processor for a good comparison;
|
||||
# currently assume processor/tokenizer agree on the EOS, and pull it off
|
||||
# the tokenizer if requested.
|
||||
hf_kwargs = {}
|
||||
if use_tokenizer_eos:
|
||||
hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
|
||||
if stop_str:
|
||||
hf_kwargs["stop_strings"] = stop_str
|
||||
|
||||
for prompts, image_data, video_data, audio_data in inputs:
|
||||
mm_data = dict(images=image_data, videos=video_data, audios=audio_data)
|
||||
hf_kwargs_with_mm_data = hf_kwargs | mm_data
|
||||
hf_output = hf_model.generate_greedy_logprobs_limit(
|
||||
prompts,
|
||||
max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
**hf_kwargs_with_mm_data,
|
||||
)
|
||||
hf_outputs_per_mm.append(hf_output)
|
||||
|
||||
# Apply output processing / sanitation to the vLLM and HF runner results
|
||||
hf_outputs_per_mm, vllm_outputs_per_mm = process_runner_outputs(
|
||||
model,
|
||||
first_runner_outputs=hf_outputs_per_mm,
|
||||
second_runner_outputs=vllm_outputs_per_mm,
|
||||
first_runner_processor=hf_output_post_proc,
|
||||
second_runner_processor=vllm_output_post_proc,
|
||||
)
|
||||
|
||||
for hf_outputs, vllm_outputs in zip(hf_outputs_per_mm, vllm_outputs_per_mm):
|
||||
# This is usually check_logprobs_close, but it's passed through to
|
||||
# allow things like check_outputs_equal where needed
|
||||
comparator(
|
||||
outputs_0_lst=hf_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
def process_runner_outputs(
|
||||
model,
|
||||
first_runner_outputs,
|
||||
second_runner_outputs,
|
||||
first_runner_processor=None,
|
||||
second_runner_processor=None,
|
||||
):
|
||||
"""Applies the runner processor(s) to the runner outputs, if any."""
|
||||
if first_runner_processor is not None:
|
||||
first_runner_outputs = process_outputs(
|
||||
first_runner_processor, model, first_runner_outputs
|
||||
)
|
||||
if second_runner_processor is not None:
|
||||
second_runner_outputs = process_outputs(
|
||||
second_runner_processor, model, second_runner_outputs
|
||||
)
|
||||
return first_runner_outputs, second_runner_outputs
|
||||
|
||||
|
||||
def process_outputs(output_processor, model, outputs_per_image):
|
||||
"""Applies a model specific post-processor function to a runner's output."""
|
||||
return [
|
||||
[output_processor(res, model) for res in outputs]
|
||||
for outputs in outputs_per_image
|
||||
]
|
||||
@@ -1,218 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Types for writing multimodal model tests."""
|
||||
|
||||
from collections.abc import Callable, Iterable
|
||||
from enum import Enum
|
||||
from pathlib import PosixPath
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import torch
|
||||
from pytest import MarkDecorator
|
||||
from transformers import AutoModelForCausalLM
|
||||
from transformers.models.auto.auto_factory import _BaseAutoModelClass
|
||||
|
||||
from vllm.config.model import RunnerOption
|
||||
from vllm.logprobs import SampleLogprobs
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .....conftest import (
|
||||
AUDIO_ASSETS,
|
||||
IMAGE_ASSETS,
|
||||
HfRunner,
|
||||
ImageAsset,
|
||||
ImageTestAssets,
|
||||
PromptAudioInput,
|
||||
PromptImageInput,
|
||||
PromptVideoInput,
|
||||
)
|
||||
from ....utils import check_logprobs_close
|
||||
|
||||
# meta image tag; will be replaced by the appropriate tag for the model
|
||||
TEST_IMG_PLACEHOLDER = "<vlm_image>"
|
||||
TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
|
||||
TEST_AUDIO_PLACEHOLDER = "<lmm_audio>"
|
||||
|
||||
SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
|
||||
"cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
|
||||
}
|
||||
)
|
||||
SINGLE_AUDIO_BASE_PROMPT = AUDIO_ASSETS.prompts(
|
||||
{
|
||||
"mary_had_lamb": f"{TEST_AUDIO_PLACEHOLDER}Transcribe this audio into English.", # noqa: E501
|
||||
"winning_call": f"{TEST_AUDIO_PLACEHOLDER}What is happening in this audio clip?", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
|
||||
VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
|
||||
|
||||
|
||||
IMAGE_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
|
||||
EMBEDDING_SIZE_FACTORS = [(1.0,), (1.0, 1.0, 1.0)]
|
||||
RunnerOutput = tuple[list[int], str, SampleLogprobs | None]
|
||||
|
||||
|
||||
class PromptWithMultiModalInput(NamedTuple):
|
||||
"""Holds the multimodal input for a single test case."""
|
||||
|
||||
prompts: list[str]
|
||||
image_data: PromptImageInput | None = None
|
||||
video_data: PromptVideoInput | None = None
|
||||
audio_data: PromptAudioInput | None = None
|
||||
|
||||
|
||||
class VLMTestType(Enum):
|
||||
IMAGE = 1
|
||||
MULTI_IMAGE = 2
|
||||
EMBEDDING = 3
|
||||
VIDEO = 4
|
||||
AUDIO = 5
|
||||
CUSTOM_INPUTS = 6
|
||||
|
||||
|
||||
class SizeType(Enum):
|
||||
SIZE_FACTOR = 1
|
||||
FIXED_SIZE = 2
|
||||
|
||||
|
||||
class CustomTestOptions(NamedTuple):
|
||||
inputs: list[PromptWithMultiModalInput]
|
||||
limit_mm_per_prompt: dict[str, int]
|
||||
|
||||
|
||||
class ImageSizeWrapper(NamedTuple):
|
||||
type: SizeType
|
||||
# A size factor is a wrapper of 0+ floats,
|
||||
# while a fixed size contains an iterable of integer pairs
|
||||
data: Iterable[float] | Iterable[tuple[int, int]]
|
||||
|
||||
|
||||
class VLMTestInfo(NamedTuple):
|
||||
"""Holds the configuration for 1+ tests for one model architecture."""
|
||||
|
||||
models: list[str]
|
||||
test_type: VLMTestType | Iterable[VLMTestType]
|
||||
|
||||
# Should be None only if this is a CUSTOM_INPUTS test
|
||||
prompt_formatter: Callable[[str], str] | None = None
|
||||
img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
|
||||
video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
|
||||
audio_idx_to_prompt: Callable[[int], str] = lambda idx: "<audio>\n"
|
||||
|
||||
# Most models work on the single / multi-image prompts above, but in some
|
||||
# cases the log prob check fails, e.g., for paligemma. We allow passing
|
||||
# an override for the single image prompts / multi-image prompt for this
|
||||
# reason.
|
||||
single_image_prompts: Iterable[str] = SINGLE_IMAGE_BASE_PROMPTS
|
||||
multi_image_prompt: str = MULTI_IMAGE_BASE_PROMPT
|
||||
|
||||
# Function for converting ImageAssets to image embeddings;
|
||||
# We need to define this explicitly for embedding tests
|
||||
convert_assets_to_embeddings: (
|
||||
Callable[[ImageTestAssets], list[torch.Tensor]] | None
|
||||
) = None
|
||||
|
||||
# Exposed options for vLLM runner; we change these in a several tests,
|
||||
# but the defaults are derived from VllmRunner & the engine defaults
|
||||
# These settings are chosen to avoid OOMs when running in the CI
|
||||
enforce_eager: bool = True
|
||||
max_model_len: int = 1024
|
||||
max_num_seqs: int = 256
|
||||
runner: RunnerOption = "auto"
|
||||
tensor_parallel_size: int = 1
|
||||
vllm_runner_kwargs: dict[str, Any] | None = None
|
||||
|
||||
# Optional callable which gets a list of token IDs from the model tokenizer
|
||||
get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None
|
||||
# Optional list of strings to stop generation, useful when stop tokens are
|
||||
# not special tokens in the tokenizer
|
||||
stop_str: list[str] | None = None
|
||||
|
||||
# Exposed options for HF runner
|
||||
hf_model_kwargs: dict[str, Any] | None = None
|
||||
hf_processor: Callable[[str], Any] | None = None
|
||||
# Indicates we should explicitly pass the EOS from the tokenizer
|
||||
use_tokenizer_eos: bool = False
|
||||
auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM
|
||||
patch_hf_runner: Callable[[HfRunner], HfRunner] | None = None
|
||||
|
||||
# Post processors that if defined, will run oun the outputs of the
|
||||
# vLLM and HF runner, respectively (useful for sanitization, etc).
|
||||
vllm_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
|
||||
hf_output_post_proc: Callable[[RunnerOutput, str], Any] | None = None
|
||||
|
||||
# Consumes the output of the callables above and checks if they're equal
|
||||
comparator: Callable[..., None] = check_logprobs_close
|
||||
|
||||
# Default expandable params per test; these defaults can be overridden in
|
||||
# instances of this object; the complete set of test cases for the model
|
||||
# is all combinations of .models + all fields below
|
||||
max_tokens: int = 128
|
||||
num_logprobs: int = 5
|
||||
dtype: str = "auto"
|
||||
distributed_executor_backend: str | None = None
|
||||
# Only expanded in video tests
|
||||
num_video_frames: int | tuple[int] = 16
|
||||
needs_video_metadata: bool = False
|
||||
|
||||
# Fixed image sizes / image size factors; most tests use image_size_factors.
|
||||
# Each inner iterable defines one request batch. All batches are run against
|
||||
# the same model instance.
|
||||
image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
|
||||
image_sizes: Iterable[Iterable[tuple[int, int]]] | None = None
|
||||
|
||||
# Hack for updating a prompt to take into a local path; currently only used
|
||||
# for Qwen-VL, which requires encoding the image path / url into the prompt
|
||||
# for HF runner
|
||||
prompt_path_encoder: (
|
||||
Callable[[PosixPath, str, list[ImageAsset] | ImageTestAssets], str] | None
|
||||
) = None # noqa: E501
|
||||
|
||||
# Allows configuring a test to run with custom inputs
|
||||
custom_test_opts: list[CustomTestOptions] | None = None
|
||||
|
||||
marks: list[MarkDecorator] | None = None
|
||||
|
||||
def get_non_parametrized_runner_kwargs(self):
|
||||
"""Returns a dictionary of expandable kwargs for items that are used
|
||||
in all test types, which are NOT used when creating the parametrized
|
||||
test cases.
|
||||
"""
|
||||
return {
|
||||
"enforce_eager": self.enforce_eager,
|
||||
"max_model_len": self.max_model_len,
|
||||
"max_num_seqs": self.max_num_seqs,
|
||||
"runner": self.runner,
|
||||
"tensor_parallel_size": self.tensor_parallel_size,
|
||||
"vllm_runner_kwargs": self.vllm_runner_kwargs,
|
||||
"hf_output_post_proc": self.hf_output_post_proc,
|
||||
"vllm_output_post_proc": self.vllm_output_post_proc,
|
||||
"auto_cls": self.auto_cls,
|
||||
"use_tokenizer_eos": self.use_tokenizer_eos,
|
||||
"comparator": self.comparator,
|
||||
"get_stop_token_ids": self.get_stop_token_ids,
|
||||
"hf_model_kwargs": self.hf_model_kwargs,
|
||||
"hf_processor": self.hf_processor,
|
||||
"stop_str": self.stop_str,
|
||||
"patch_hf_runner": self.patch_hf_runner,
|
||||
}
|
||||
|
||||
|
||||
class ExpandableVLMTestArgs(NamedTuple):
|
||||
"""The expanded kwargs which correspond to a single test case."""
|
||||
|
||||
model: str
|
||||
max_tokens: int
|
||||
num_logprobs: int
|
||||
dtype: str
|
||||
distributed_executor_backend: str | None
|
||||
# Sizes are used for everything except audio and custom input tests.
|
||||
size_wrappers: tuple[ImageSizeWrapper, ...] = ()
|
||||
# Video only
|
||||
num_video_frames: int | None = None
|
||||
needs_video_metadata: bool = False
|
||||
# Custom inputs only
|
||||
custom_test_opts: CustomTestOptions | None = None
|
||||
@@ -1,558 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from collections.abc import Set as AbstractSet
|
||||
from functools import partial
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from tests.models.utils import build_model_context
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalInput
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.renderers.chat_utils.item_tracker import _get_embeds_data
|
||||
from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ....multimodal.utils import random_audio, random_image, random_video
|
||||
from ...registry import (
|
||||
_MULTIMODAL_EXAMPLE_MODELS,
|
||||
_TRANSFORMERS_BACKEND_MODELS,
|
||||
HF_EXAMPLE_MODELS,
|
||||
)
|
||||
|
||||
|
||||
def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
"""Add metadata to video mm_data."""
|
||||
|
||||
def create_metadata(frames: np.ndarray):
|
||||
num_frames = len(frames)
|
||||
return {
|
||||
"total_num_frames": num_frames,
|
||||
"fps": 2.0,
|
||||
"duration": num_frames / 2.0,
|
||||
"video_backend": "opencv",
|
||||
"frames_indices": list(range(num_frames)),
|
||||
"do_sample_frames": True,
|
||||
}
|
||||
|
||||
# Ensure video metadata is included
|
||||
if "video" in mm_data:
|
||||
video = mm_data["video"]
|
||||
if isinstance(video, list):
|
||||
# multiple videos
|
||||
mm_data["video"] = [(vid, create_metadata(vid)) for vid in video]
|
||||
else:
|
||||
# single video
|
||||
mm_data["video"] = (video, create_metadata(video))
|
||||
return mm_data
|
||||
|
||||
|
||||
def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict:
|
||||
"""Patch the multimodal data for GLM-ASR model.
|
||||
GLM-ASR requires text and audio to match 1:1, so we limit audio to 1.
|
||||
"""
|
||||
if "audio" in mm_data:
|
||||
audio = mm_data["audio"]
|
||||
if isinstance(audio, list) and len(audio) > 1:
|
||||
# Limit to single audio to match text requirement
|
||||
mm_data["audio"] = [audio[0]]
|
||||
return mm_data
|
||||
|
||||
|
||||
_IGNORE_MM_KEYS = {
|
||||
# Dithering causes minor divergence
|
||||
"cohere_asr": {"input_features"},
|
||||
# In Ultravox, the audio_features can be different depending on padding
|
||||
# The slight difference should not be a problem though, since
|
||||
# attention_mask lets us ignore the difference.
|
||||
"ultravox": {"audio_features"},
|
||||
}
|
||||
|
||||
MM_DATA_PATCHES = {
|
||||
"glmasr": glmasr_patch_mm_data,
|
||||
}
|
||||
|
||||
_XPU_EXCLUDED_MODEL_IDS = {
|
||||
"baidu/Unlimited-OCR",
|
||||
"mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
|
||||
"moonshotai/Kimi-K3",
|
||||
"Qwen/Qwen2.5-Omni-7B-AWQ",
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
}
|
||||
|
||||
_CPU_EXCLUDED_MODEL_IDS = {
|
||||
# DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now.
|
||||
"deepseek-ai/DeepSeek-V4-Flash-Vision-Exp",
|
||||
}
|
||||
|
||||
|
||||
def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
|
||||
for model_arch in model_arch_list:
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
|
||||
yield model_info.default
|
||||
|
||||
for extra_type, extra_model_id in model_info.extras.items():
|
||||
if "fp" in extra_type:
|
||||
continue # Redundant to test quantized models
|
||||
|
||||
yield extra_model_id
|
||||
|
||||
|
||||
def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
|
||||
model_ids = list(_iter_model_ids_to_test(model_arch_list))
|
||||
|
||||
if current_platform.is_xpu():
|
||||
for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS:
|
||||
while excluded_model_id in model_ids:
|
||||
model_ids.remove(excluded_model_id)
|
||||
|
||||
if current_platform.is_cpu():
|
||||
for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS:
|
||||
while excluded_model_id in model_ids:
|
||||
model_ids.remove(excluded_model_id)
|
||||
|
||||
return model_ids
|
||||
|
||||
|
||||
def get_model_ids_to_test():
|
||||
transformers_arch_ids = {
|
||||
model_id
|
||||
for info in _TRANSFORMERS_BACKEND_MODELS.values()
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
}
|
||||
vllm_only_archs = {
|
||||
arch
|
||||
for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items()
|
||||
if not any(
|
||||
model_id in transformers_arch_ids
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
)
|
||||
}
|
||||
|
||||
return _get_model_ids_to_test(vllm_only_archs)
|
||||
|
||||
|
||||
def get_transformers_backend_model_ids_to_test():
|
||||
return sorted(
|
||||
{
|
||||
model_id
|
||||
for arch, info in _TRANSFORMERS_BACKEND_MODELS.items()
|
||||
if "MultiModal" in arch
|
||||
for model_id in (info.default, *info.extras.values())
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_token_prompt(
|
||||
processor: BaseMultiModalProcessor,
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> list[int]:
|
||||
tokenizer: TokenizerLike = processor.info.get_tokenizer()
|
||||
model_config = processor.info.ctx.model_config
|
||||
|
||||
if processor.info.data_parser.video_needs_metadata:
|
||||
mm_data = add_video_metadata(mm_data)
|
||||
|
||||
model_type = model_config.hf_config.model_type
|
||||
if model_type in MM_DATA_PATCHES:
|
||||
mm_data = MM_DATA_PATCHES[model_type](mm_data)
|
||||
|
||||
parsed_data = processor.info.parse_mm_data(mm_data)
|
||||
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
|
||||
|
||||
if is_mistral_tokenizer(tokenizer):
|
||||
inputs = processor.get_dummy_inputs(
|
||||
model_config.max_model_len,
|
||||
mm_counts,
|
||||
mm_options=MultiModalDummyOptions(),
|
||||
# Assume all Mistral models define this extra argument
|
||||
mm_data=mm_data, # type: ignore[call-arg]
|
||||
)
|
||||
else:
|
||||
inputs = processor.get_dummy_inputs(
|
||||
model_config.max_model_len,
|
||||
mm_counts,
|
||||
mm_options=MultiModalDummyOptions(),
|
||||
)
|
||||
|
||||
if not isinstance(inputs.prompt, list):
|
||||
raise TypeError(type(inputs.prompt))
|
||||
|
||||
return inputs.prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_id,durations",
|
||||
[
|
||||
("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]),
|
||||
("nvidia/audio-flamingo-3-hf", [31.0]),
|
||||
("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]),
|
||||
("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]),
|
||||
("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]),
|
||||
],
|
||||
)
|
||||
def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch):
|
||||
"""EC metadata preserves the raw-audio prompt without running the HF processor."""
|
||||
ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)})
|
||||
mm_config = ctx.model_config.multimodal_config
|
||||
mm_config.enable_mm_embeds = True
|
||||
mm_config.allow_missing_mm_embeddings = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
sample_rate = processor.info.get_feature_extractor().sampling_rate
|
||||
audios = [
|
||||
np.zeros(int(duration * sample_rate), dtype=np.float32)
|
||||
for duration in durations
|
||||
]
|
||||
prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)})
|
||||
raw = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
features = [
|
||||
MultiModalFeatureSpec(
|
||||
data=item,
|
||||
modality="audio",
|
||||
identifier=str(index),
|
||||
mm_position=position,
|
||||
)
|
||||
for index, (item, position) in enumerate(
|
||||
zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"])
|
||||
)
|
||||
]
|
||||
published = collect_ec_item_metadata(
|
||||
features, PlaceholderMetadataResolver(ctx.model_config)
|
||||
)
|
||||
metadata = json.loads(
|
||||
json.dumps([entry["metadata"] for entry in published.values()])
|
||||
)
|
||||
assert len(metadata) == len(audios)
|
||||
assert all(metadata)
|
||||
audio_data = _get_embeds_data("audio", metadata, processor)
|
||||
|
||||
def unexpected_preprocessing(*args, **kwargs):
|
||||
pytest.fail("The EC consumer must not preprocess audio")
|
||||
|
||||
monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing)
|
||||
remote = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data({"audio": audio_data}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert remote["prompt_token_ids"] == raw["prompt_token_ids"]
|
||||
assert remote["mm_placeholders"] == raw["mm_placeholders"]
|
||||
# Omni also consumes these lengths when constructing M-RoPE positions.
|
||||
if "audio_feature_lengths" in audio_data:
|
||||
assert torch.equal(
|
||||
remote["mm_kwargs"].get_data()["audio_feature_lengths"],
|
||||
raw["mm_kwargs"].get_data()["audio_feature_lengths"],
|
||||
)
|
||||
|
||||
mm_config.allow_missing_mm_embeddings = False
|
||||
ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
with pytest.raises(ValueError, match="should contain the fields"):
|
||||
ordinary_processor.info.parse_mm_data({"audio": audio_data})
|
||||
|
||||
|
||||
def random_vision_chunk(
|
||||
rng: np.random.RandomState,
|
||||
min_wh: int,
|
||||
max_wh: int,
|
||||
min_frames: int,
|
||||
max_frames: int,
|
||||
) -> dict:
|
||||
num_frames = rng.randint(min_frames, max_frames + 1)
|
||||
if num_frames == 1:
|
||||
# Single image chunk
|
||||
wh = rng.randint(min_wh, max_wh + 1)
|
||||
image = random_image(rng, wh, wh + 1)
|
||||
return {"type": "image", "image": image}
|
||||
frames = []
|
||||
for _ in range(num_frames):
|
||||
wh = rng.randint(min_wh, max_wh + 1)
|
||||
frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8)
|
||||
frames.append(frame)
|
||||
video_array = np.stack(frames, axis=0)
|
||||
return {"type": "video_chunk", "video_chunk": video_array}
|
||||
|
||||
|
||||
def _test_processing_correctness(
|
||||
model_id_or_arch: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
model_impl: str = "auto",
|
||||
):
|
||||
if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs():
|
||||
# Use model architecture to get the default model id
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch)
|
||||
model_id = model_info.default
|
||||
else:
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch)
|
||||
model_id = model_id_or_arch
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=model_info.dtype,
|
||||
model_impl=model_impl,
|
||||
)
|
||||
# Ensure that the cache can fit all of the data
|
||||
# (set after because ModelConfig would set it to 0 for encoder-decoder models)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 2048
|
||||
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
factories = model_cls._processor_factory
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
|
||||
processing_info = factories.info(ctx)
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
# Keep integer limits for local data generation
|
||||
limit_mm_per_prompt_ints = {
|
||||
modality: 3 if limit is None else limit
|
||||
for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
|
||||
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
|
||||
if modality == "video":
|
||||
return VideoDummyOptions(count=count)
|
||||
if modality == "image":
|
||||
return ImageDummyOptions(count=count)
|
||||
if modality == "audio":
|
||||
return AudioDummyOptions(count=count)
|
||||
return BaseDummyOptions(count=count)
|
||||
|
||||
# Assign normalized DummyOptions to the model config
|
||||
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
|
||||
{
|
||||
modality: _to_dummy_options(modality, count)
|
||||
for modality, count in limit_mm_per_prompt_ints.items()
|
||||
}
|
||||
)
|
||||
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
rng = np.random.RandomState(0)
|
||||
|
||||
# GLM-ASR requires a minimum audio length of 70ms
|
||||
min_audio_len = 512 if model_config.hf_config.model_type != "glmasr" else 1120
|
||||
input_to_hit = {
|
||||
"image": Image.new("RGB", size=(128, 128)),
|
||||
"video": np.zeros((4, 128, 128, 3), dtype=np.uint8),
|
||||
"audio": (np.zeros((min_audio_len,)), 16000),
|
||||
"vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))},
|
||||
}
|
||||
input_factory = {
|
||||
"image": partial(random_image, rng, min_wh=128, max_wh=256),
|
||||
"video": partial(
|
||||
random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256
|
||||
),
|
||||
"audio": partial(
|
||||
random_audio,
|
||||
rng,
|
||||
min_len=min_audio_len,
|
||||
max_len=min_audio_len + 512,
|
||||
sr=16000,
|
||||
),
|
||||
"vision_chunk": partial(
|
||||
random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1
|
||||
),
|
||||
}
|
||||
|
||||
for batch_idx in range(num_batches):
|
||||
mm_data = {
|
||||
k: [
|
||||
(input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]())
|
||||
for _ in range(rng.randint(limit + 1))
|
||||
]
|
||||
for k, limit in limit_mm_per_prompt_ints.items()
|
||||
}
|
||||
|
||||
# Drop unnecessary keys and test single -> multi conversion
|
||||
if rng.rand() < simplify_rate:
|
||||
for k in list(mm_data.keys()):
|
||||
if not mm_data[k]:
|
||||
del mm_data[k]
|
||||
elif len(mm_data[k]) == 1:
|
||||
mm_data[k] = mm_data[k][0]
|
||||
|
||||
_test_processing_correctness_one(
|
||||
model_config,
|
||||
mm_data,
|
||||
processor,
|
||||
batch_idx,
|
||||
hit_rate,
|
||||
num_batches,
|
||||
simplify_rate,
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
|
||||
def _test_processing_correctness_one(
|
||||
model_config: ModelConfig,
|
||||
mm_data: MultiModalDataDict,
|
||||
processor: BaseMultiModalProcessor,
|
||||
batch_idx: int,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
cache: MultiModalProcessorOnlyCache,
|
||||
):
|
||||
model_type = model_config.hf_config.model_type
|
||||
|
||||
token_prompt = get_token_prompt(processor, mm_data)
|
||||
mm_items = processor.info.parse_mm_data(mm_data)
|
||||
ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]())
|
||||
|
||||
baseline_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
cached_tokenized_result = processor(
|
||||
token_prompt,
|
||||
mm_items=mm_items,
|
||||
hf_processor_mm_kwargs={},
|
||||
cache=cache,
|
||||
)
|
||||
|
||||
_assert_inputs_equal(
|
||||
baseline_tokenized_result,
|
||||
cached_tokenized_result,
|
||||
ignore_mm_keys=ignore_mm_keys,
|
||||
msg=(
|
||||
f"Failed ({batch_idx=}, {hit_rate=}, "
|
||||
f"{num_batches=}, {simplify_rate=}, "
|
||||
f"{token_prompt=}, {mm_data=})"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_processing_correctness(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
if model_id == "openvla/openvla-7b":
|
||||
pytest.skip(
|
||||
"OpenVLA uses a custom vLLM processor because its HF remote "
|
||||
"processor is incompatible with current Transformers."
|
||||
)
|
||||
if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602":
|
||||
pytest.skip(
|
||||
"Voxtral Realtime doesn't make use of any place-holder "
|
||||
"tokens and hence cannot pass the processing "
|
||||
"correctness test as is. Let's revisit adapting this "
|
||||
"test once more realtime models exist."
|
||||
)
|
||||
if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"):
|
||||
pytest.skip(
|
||||
"MOSS-Audio uses a custom processor that dynamically expands "
|
||||
"audio placeholders from processed audio lengths. Its vLLM "
|
||||
"processor paths are covered by test_moss_audio.py."
|
||||
)
|
||||
if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct":
|
||||
pytest.skip(
|
||||
"LLaVA-OneVision-2 video processing routes frames through custom "
|
||||
"video backends (qwen_vl_utils / codec) that require real encoded "
|
||||
"video bytes and metadata. The synthetic numpy-array videos used by "
|
||||
"this test yield empty video features, so the generic correctness "
|
||||
"check cannot exercise the video path. Image processing is covered "
|
||||
"by registration/inference tests."
|
||||
)
|
||||
|
||||
_test_processing_correctness(
|
||||
model_id,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test())
|
||||
@pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0])
|
||||
@pytest.mark.parametrize("num_batches", [32])
|
||||
@pytest.mark.parametrize("simplify_rate", [1.0])
|
||||
def test_processing_correctness_transformers(
|
||||
model_id: str,
|
||||
hit_rate: float,
|
||||
num_batches: int,
|
||||
simplify_rate: float,
|
||||
):
|
||||
_test_processing_correctness(
|
||||
model_id,
|
||||
hit_rate=hit_rate,
|
||||
num_batches=num_batches,
|
||||
simplify_rate=simplify_rate,
|
||||
model_impl="transformers",
|
||||
)
|
||||
|
||||
|
||||
def _assert_inputs_equal(
|
||||
a: MultiModalInput,
|
||||
b: MultiModalInput,
|
||||
*,
|
||||
ignore_mm_keys: set[str] | None = None,
|
||||
msg: str = "",
|
||||
):
|
||||
if ignore_mm_keys is None:
|
||||
ignore_mm_keys = set()
|
||||
|
||||
ignore_prompt_keys = ("prompt", "mm_kwargs")
|
||||
a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys}
|
||||
b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys}
|
||||
|
||||
assert a_rest == b_rest, msg
|
||||
|
||||
a_data = a["mm_kwargs"].get_data()
|
||||
b_data = b["mm_kwargs"].get_data()
|
||||
|
||||
for key in ignore_mm_keys:
|
||||
a_data.pop(key, None)
|
||||
b_data.pop(key, None)
|
||||
|
||||
assert batched_tensors_equal(a_data, b_data), msg
|
||||
@@ -1,241 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Video placeholder accounting for GLM-5.3-Flash.
|
||||
|
||||
``Glm4vProcessingInfo._construct_video_placeholder`` emits one frame of
|
||||
placeholders per timestamp, so the timestamps returned by
|
||||
``_get_video_second_idx_glm46v`` decide how many placeholders the prompt gets
|
||||
while ``video_grid_thw`` decides how many rows the vision tower produces. If
|
||||
the two disagree they collide in ``_merge_multimodal_embeddings``, which raises
|
||||
inside a worker and takes the engine down with it.
|
||||
|
||||
The checks below are arithmetic: no weights, no GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.models.glm5_next.video_processing_glm5_next import (
|
||||
Glm5NextVideoProcessor,
|
||||
smart_resize,
|
||||
)
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def processor():
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
)
|
||||
|
||||
|
||||
def _pixel_path_grid(
|
||||
video_processor: Glm5NextVideoProcessor,
|
||||
num_frames: int,
|
||||
height: int,
|
||||
width: int,
|
||||
) -> tuple[int, int, int]:
|
||||
"""The ``video_grid_thw`` ``Glm5NextVideoProcessor._preprocess`` builds."""
|
||||
resized_height, resized_width = smart_resize(
|
||||
num_frames=num_frames,
|
||||
height=height,
|
||||
width=width,
|
||||
temporal_factor=video_processor.temporal_patch_size,
|
||||
factor=video_processor.patch_size
|
||||
* video_processor.merge_size
|
||||
* video_processor.patch_expand_factor,
|
||||
min_pixels=video_processor.min_image_tokens,
|
||||
max_pixels=video_processor.max_image_tokens,
|
||||
)
|
||||
padded_frames = num_frames + (-num_frames % video_processor.temporal_patch_size)
|
||||
return (
|
||||
padded_frames // video_processor.temporal_patch_size,
|
||||
resized_height // video_processor.patch_size,
|
||||
resized_width // video_processor.patch_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_num_frames", "fps", "duration", "height", "width", "expected_grid"),
|
||||
[
|
||||
# 4 s at 8 fps: the GLM-4.6V sampler asks for 3x as many timestamps.
|
||||
(32, 8.0, 4.0, 480, 640, (4, 36, 46)),
|
||||
# 1080p, 20 s: same factor of 3 at a full-size canvas.
|
||||
(600, 30.0, 20.0, 1080, 1920, (20, 58, 102)),
|
||||
# 1080p, 60 s: the one duration window where the two samplers agree
|
||||
# anyway -- a regression guard, the count must not move.
|
||||
(1800, 30.0, 60.0, 1080, 1920, (60, 34, 58)),
|
||||
# Past 300 s the GLM-4.6V sampler asks for half as many instead.
|
||||
(9030, 30.0, 301.0, 720, 1280, (301, 14, 26)),
|
||||
],
|
||||
)
|
||||
def test_video_placeholders_match_encoder_rows(
|
||||
processor,
|
||||
total_num_frames: int,
|
||||
fps: float,
|
||||
duration: float,
|
||||
height: int,
|
||||
width: int,
|
||||
expected_grid: tuple[int, int, int],
|
||||
):
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
frame_indices = video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=total_num_frames, fps=fps, duration=duration)
|
||||
)
|
||||
grid_t, grid_h, grid_w = _pixel_path_grid(
|
||||
video_processor, len(frame_indices), height, width
|
||||
)
|
||||
assert (grid_t, grid_h, grid_w) == expected_grid
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": total_num_frames,
|
||||
"fps": fps,
|
||||
"duration": duration,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
total_num_frames,
|
||||
)
|
||||
|
||||
merge_length = video_processor.merge_size**2
|
||||
tokens_per_frame = grid_h * grid_w // merge_length
|
||||
encoder_rows = grid_t * grid_h * grid_w // merge_length
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
assert len(timestamps) * tokens_per_frame == encoder_rows
|
||||
assert timestamps == sorted(timestamps)
|
||||
assert timestamps[0] == 0
|
||||
assert timestamps[-1] <= duration
|
||||
|
||||
|
||||
def test_video_placeholders_match_encoder_rows_when_presampled(processor):
|
||||
"""The loader may pre-sample and hand the frames over as they are."""
|
||||
info = processor.info
|
||||
video_processor = info.get_video_processor()
|
||||
|
||||
num_frames = 32
|
||||
grid_t, _, _ = _pixel_path_grid(video_processor, num_frames, 480, 640)
|
||||
|
||||
timestamps = info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 256,
|
||||
"fps": 8.0,
|
||||
"duration": 32.0,
|
||||
"do_sample_frames": False,
|
||||
"frames_indices": list(range(0, 256, 256 // num_frames)),
|
||||
},
|
||||
num_frames,
|
||||
)
|
||||
|
||||
assert len(timestamps) == grid_t
|
||||
|
||||
|
||||
def test_video_shorter_than_one_sampling_interval_is_rejected(processor):
|
||||
"""A clip the sampler cannot pick a single frame from is a bad request."""
|
||||
with pytest.raises(ValueError, match="selected no frames"):
|
||||
processor.info._get_video_second_idx_glm46v(
|
||||
{
|
||||
"total_num_frames": 2,
|
||||
"fps": 8.0,
|
||||
"duration": 0.25,
|
||||
"do_sample_frames": True,
|
||||
},
|
||||
2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
)
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (310, 470), color=(17, 89, 231)),
|
||||
Image.new("RGB", (480, 320), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = "<|begin_of_image|><|image|><|end_of_image|>" * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def _image_info(**kwargs):
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
**kwargs,
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
).info
|
||||
|
||||
|
||||
def test_image_encoder_cache_covers_full_token_budget():
|
||||
"""The most-features probe must reach the processor's token ceiling.
|
||||
|
||||
The inherited square probe refits to 89x89 = 7921 tokens under the
|
||||
max_image_tokens=8000 budget, so the encoder cache came up short and
|
||||
ordinary non-square images in 7922-8000 tokens were refused with
|
||||
HTTP 400 (#59539).
|
||||
"""
|
||||
info = _image_info()
|
||||
assert info.get_max_image_tokens() == 8000
|
||||
|
||||
# The shapes from the issue, with the token counts the processor
|
||||
# actually produces; every one must fit the cache.
|
||||
for width, height, expected_tokens in [
|
||||
(4032, 3024, 7931), # phone photo, 4:3
|
||||
(3840, 2160, 7973), # 4K frame, 16:9
|
||||
(3508, 2480, 7950), # A4 at 300 dpi
|
||||
(2600, 2400, 7998), # 13:12
|
||||
(3000, 3000, 7921), # square worst case before the fix
|
||||
]:
|
||||
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
|
||||
assert num_tokens == expected_tokens
|
||||
assert num_tokens <= info.get_max_image_tokens()
|
||||
|
||||
# The profiling dummy covers the real worst case, not the square one.
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert size.width * size.height == 2240 * 2800
|
||||
|
||||
|
||||
def test_image_encoder_cache_follows_max_pixels_override():
|
||||
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
|
||||
assert info.get_max_image_tokens() == 100
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert (
|
||||
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
|
||||
== 100
|
||||
)
|
||||
@@ -1,81 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoImageProcessor
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processors.kimi_k25_vision_fused import (
|
||||
KimiK25FusedVisionProcessor,
|
||||
)
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_MODEL_ID = "moonshotai/Kimi-K3"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_processors():
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
k25_fused = processor.info.image_processor
|
||||
assert isinstance(k25_fused, KimiK25FusedVisionProcessor)
|
||||
k3_hf = AutoImageProcessor.from_pretrained(_MODEL_ID, trust_remote_code=True)
|
||||
return k25_fused, k3_hf
|
||||
|
||||
|
||||
def _noise_image(width: int, height: int) -> Image.Image:
|
||||
rng = np.random.default_rng(0)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("width", "height"),
|
||||
[(310, 470), (7300, 30), (13, 9)],
|
||||
)
|
||||
def test_fused_image_processor_matches_hf(image_processors, width: int, height: int):
|
||||
fused, hf = image_processors
|
||||
image = _noise_image(width, height)
|
||||
expected = hf.preprocess([{"type": "image", "image": image}], return_tensors="pt")
|
||||
actual = fused.preprocess([{"type": "image", "image": image}], return_tensors="pt")
|
||||
|
||||
torch.testing.assert_close(actual["grid_thws"], expected["grid_thws"])
|
||||
torch.testing.assert_close(actual["pixel_values"], expected["pixel_values"])
|
||||
assert fused.media_tokens_calculator(
|
||||
{"type": "image", "image": image}
|
||||
) == hf.media_tokens_calculator({"type": "image", "image": image})
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_mm_device_do_normalize():
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 2})
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
images = [
|
||||
Image.new("RGB", (310, 470), color=(17, 89, 231)),
|
||||
Image.new("RGB", (480, 320), color=(201, 13, 127)),
|
||||
]
|
||||
prompt = processor.info.get_hf_config().image_placeholder * len(images)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
|
||||
normalized_inputs = processor(prompt, mm_items=mm_items)
|
||||
normalized_values = normalized_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
raw_inputs = processor(prompt, mm_items=mm_items)
|
||||
raw_values = raw_inputs["mm_kwargs"].get_data()["pixel_values"]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
|
||||
)
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for Nemotron-Parse's device-side normalisation path."""
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import build_model_context
|
||||
|
||||
_MODEL_ID = "nvidia/NVIDIA-Nemotron-Parse-2.0"
|
||||
|
||||
|
||||
def _noise_image(width: int, height: int) -> Image.Image:
|
||||
rng = np.random.default_rng(0)
|
||||
return Image.fromarray(rng.integers(0, 256, (height, width, 3), dtype=np.uint8))
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize(("width", "height"), [(1240, 1754), (310, 470)])
|
||||
def test_mm_device_do_normalize(width: int, height: int):
|
||||
device = current_platform.device_type
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
assert ctx.model_config.multimodal_config.mm_device_do_normalize
|
||||
|
||||
images = [_noise_image(width, height)]
|
||||
prompt = [0]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
mm_items = processor.info.parse_mm_data({"image": images})
|
||||
normalized_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
|
||||
"pixel_values"
|
||||
]
|
||||
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = True
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
raw_values = processor(prompt, mm_items=mm_items)["mm_kwargs"].get_data()[
|
||||
"pixel_values"
|
||||
]
|
||||
assert raw_values.dtype == torch.uint8
|
||||
assert raw_values.shape == normalized_values.shape
|
||||
|
||||
input_norm = build_mm_input_norm(ctx.model_config).to(device)
|
||||
output = input_norm(raw_values.flatten(1).to(device), normalized_values.dtype)
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.flatten(1).to(device), rtol=1.6e-2, atol=1e-5
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("mm_device_do_normalize", [False, True])
|
||||
@pytest.mark.parametrize("layout", ["ndarray_hwc", "tensor_hwc", "tensor_chw"])
|
||||
def test_array_image_inputs(mm_device_do_normalize: bool, layout: str):
|
||||
ctx = build_model_context(_MODEL_ID, limit_mm_per_prompt={"image": 1})
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = mm_device_do_normalize
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
image = _noise_image(310, 470)
|
||||
array = np.asarray(image)
|
||||
item = {
|
||||
"ndarray_hwc": array,
|
||||
"tensor_hwc": torch.from_numpy(array),
|
||||
"tensor_chw": torch.from_numpy(array).permute(2, 0, 1),
|
||||
}[layout]
|
||||
|
||||
def pixel_values(img: object) -> torch.Tensor:
|
||||
mm_items = processor.info.parse_mm_data({"image": [img]})
|
||||
return processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
|
||||
|
||||
torch.testing.assert_close(pixel_values(item), pixel_values(image), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
def test_explicit_kwargs_without_device_normalize():
|
||||
ctx = build_model_context(
|
||||
_MODEL_ID,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
mm_processor_kwargs={"do_rescale": False, "do_normalize": False},
|
||||
)
|
||||
ctx.model_config.multimodal_config.mm_device_do_normalize = False
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
|
||||
mm_items = processor.info.parse_mm_data({"image": [_noise_image(310, 470)]})
|
||||
values = processor([0], mm_items=mm_items)["mm_kwargs"].get_data()["pixel_values"]
|
||||
# The remote processor ignores these kwargs and still normalises on the host.
|
||||
assert values.is_floating_point()
|
||||
@@ -1,272 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import tempfile
|
||||
from collections.abc import Iterable
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
ImageDummyOptions,
|
||||
MultiModalDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import supports_multimodal
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, BatchedTensorInputs
|
||||
from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
|
||||
from vllm.multimodal.utils import group_and_batch_mm_kwargs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
from vllm.utils.collection_utils import is_list_of
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
from ....utils import create_new_process_for_each_test
|
||||
from ...registry import HF_EXAMPLE_MODELS
|
||||
from ...utils import dummy_hf_overrides
|
||||
from .test_common import get_model_ids_to_test, get_token_prompt
|
||||
|
||||
ImageInput = list[Image.Image]
|
||||
VideoInput: TypeAlias = (
|
||||
list[Image.Image] | list[np.ndarray] | list[tuple[np.ndarray, dict[str, Any]]]
|
||||
)
|
||||
AudioInput = list[tuple[np.ndarray, int]]
|
||||
|
||||
|
||||
def _resize_data(
|
||||
_data: Image.Image | np.ndarray, size_factor: float
|
||||
) -> Image.Image | np.ndarray:
|
||||
assert size_factor <= 1, "Size factor must be less than 1"
|
||||
# Image input
|
||||
if isinstance(_data, Image.Image):
|
||||
W, H = _data.width, _data.height
|
||||
W, H = map(lambda x: int(x * size_factor), (W, H))
|
||||
return _data.resize((W, H))
|
||||
# Video input with PIL Images
|
||||
elif is_list_of(_data, Image.Image):
|
||||
W, H = next(iter(_data)).width, next(iter(_data)).height
|
||||
T = len(_data)
|
||||
T, W, H = map(lambda x: max(int(x * size_factor), 2), (T, W, H))
|
||||
return [d.resize((W, H)) for d in _data[:T]]
|
||||
# Video input with numpy arrays
|
||||
elif isinstance(_data, np.ndarray) and _data.ndim >= 4:
|
||||
T, H, W, C = _data.shape[-4:]
|
||||
T, H, W = map(lambda x: max(int(x * size_factor), 2), (T, H, W))
|
||||
return _data[..., :T, :H, :W, :C]
|
||||
# Audio input
|
||||
elif isinstance(_data, np.ndarray) and _data.ndim == 1:
|
||||
return _data[: int(len(_data) * size_factor)]
|
||||
raise AssertionError("This line should be unreachable.")
|
||||
|
||||
|
||||
def resize_mm_data(
|
||||
data: ImageInput | VideoInput | AudioInput, size_factors: tuple[float, ...]
|
||||
) -> ImageInput | VideoInput | AudioInput:
|
||||
size_factors = size_factors[: len(data)]
|
||||
if is_list_of(data, (Image.Image, np.ndarray, list)):
|
||||
return [_resize_data(d, s) for d, s in zip(data, size_factors)]
|
||||
elif is_list_of(data, tuple):
|
||||
return [_resize_data(d, s) for (d, _), s in zip(data, size_factors)]
|
||||
raise ValueError("Unsupported multimodal data type.")
|
||||
|
||||
|
||||
def create_batched_mm_kwargs(
|
||||
model_config: ModelConfig,
|
||||
processor: BaseMultiModalProcessor,
|
||||
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
|
||||
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
|
||||
processing_info = processor.info
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
mm_counts = {
|
||||
modality: 3 if limit is None else limit
|
||||
for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
processor_inputs = processor.get_dummy_inputs(
|
||||
seq_len=model_config.max_model_len,
|
||||
mm_counts=mm_counts,
|
||||
mm_options=MultiModalDummyOptions(),
|
||||
)
|
||||
mm_items = processor_inputs.mm_data_items
|
||||
resized_mm_data = {
|
||||
modality: resize_mm_data(items.data, size_factors)
|
||||
for modality, items in mm_items.items()
|
||||
}
|
||||
|
||||
# video metadata will be added back to the resized video data here.
|
||||
token_prompt = get_token_prompt(processor, resized_mm_data)
|
||||
|
||||
mm_kwargs = processor(
|
||||
prompt=token_prompt,
|
||||
mm_items=processor.info.parse_mm_data(resized_mm_data),
|
||||
hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
|
||||
)["mm_kwargs"].require_data()
|
||||
|
||||
return group_and_batch_mm_kwargs(
|
||||
[
|
||||
(modality, item)
|
||||
for modality in supported_mm_limits
|
||||
for item in mm_kwargs[modality]
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# TODO(Isotr0py): Don't initialize model during test
|
||||
@contextmanager
|
||||
def initialize_dummy_model(
|
||||
model_cls: type[nn.Module],
|
||||
model_config: ModelConfig,
|
||||
):
|
||||
temp_file = tempfile.mkstemp()[1]
|
||||
current_device = torch.get_default_device()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config, cache_config=CacheConfig(block_size=16)
|
||||
)
|
||||
with set_current_vllm_config(vllm_config=vllm_config):
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=0,
|
||||
backend="nccl",
|
||||
)
|
||||
initialize_model_parallel(tensor_model_parallel_size=1)
|
||||
|
||||
with set_default_torch_dtype(model_config.dtype):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
model = model_cls(vllm_config=vllm_config)
|
||||
torch.set_default_device(current_device)
|
||||
yield model
|
||||
|
||||
del model
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("model_id", get_model_ids_to_test())
|
||||
def test_model_tensor_schema(model_id: str):
|
||||
if model_id == "moonshotai/Kimi-K2.5":
|
||||
# FIXME(Isotr0py): Fix Kimi-K2.5's offline inference about vision chunks.
|
||||
pytest.skip(
|
||||
"Kimi-K2.5's offline inference has issues about vision chunks. Fix later."
|
||||
)
|
||||
|
||||
if (
|
||||
model_id == "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp"
|
||||
and not current_platform.is_cuda_alike()
|
||||
):
|
||||
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
|
||||
|
||||
if model_id == "zai-org/GLM-5.3-Flash" and (current_platform.is_xpu()):
|
||||
pytest.skip("GLM-5.3-Flash is not supported on XPU")
|
||||
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_arch = next(
|
||||
arch for arch, info in HF_EXAMPLE_MODELS.hf_models.items() if info == model_info
|
||||
)
|
||||
|
||||
hf_overrides_fn = partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch=model_arch,
|
||||
exist_overrides=model_info.hf_overrides,
|
||||
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
|
||||
)
|
||||
|
||||
# ROCm: Detect if model uses AWQ quantization and set appropriate dtype
|
||||
if "awq" in model_id.lower() and current_platform.is_rocm():
|
||||
dtype = "float16"
|
||||
else:
|
||||
dtype = model_info.dtype
|
||||
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
hf_overrides=hf_overrides_fn,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config)
|
||||
assert supports_multimodal(model_cls)
|
||||
|
||||
factories = model_cls._processor_factory
|
||||
|
||||
# Capture helpers return capture-buffer containers (e.g.
|
||||
# EncoderCudaGraphCaptureInputs), not TensorSchema mm inputs.
|
||||
capture_helpers = {"prepare_encoder_cudagraph_capture_inputs"}
|
||||
|
||||
inputs_parse_methods = []
|
||||
for attr_name in dir(model_cls):
|
||||
if attr_name in capture_helpers:
|
||||
continue
|
||||
attr = getattr(model_cls, attr_name)
|
||||
if hasattr(attr, "__annotations__"):
|
||||
return_type = attr.__annotations__.get("return", None)
|
||||
if return_type is not None and "Input" in str(return_type):
|
||||
inputs_parse_methods.append(attr_name)
|
||||
|
||||
if not any(inputs_parse_methods):
|
||||
pytest.skip(f"{model_arch} does not support tensor schema validation.")
|
||||
|
||||
ctx = InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
processing_info = factories.info(ctx)
|
||||
supported_mm_limits = processing_info.get_supported_mm_limits()
|
||||
limit_mm_per_prompt = {
|
||||
modality: 3 if limit is None else limit
|
||||
for modality, limit in supported_mm_limits.items()
|
||||
}
|
||||
|
||||
def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions:
|
||||
if modality == "video":
|
||||
return VideoDummyOptions(count=count)
|
||||
if modality == "image":
|
||||
return ImageDummyOptions(count=count)
|
||||
if modality == "audio":
|
||||
return AudioDummyOptions(count=count)
|
||||
return BaseDummyOptions(count=count)
|
||||
|
||||
model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions(
|
||||
{
|
||||
modality: _to_dummy_options(modality, count)
|
||||
for modality, count in limit_mm_per_prompt.items()
|
||||
}
|
||||
)
|
||||
processor = factories.build_processor(ctx)
|
||||
|
||||
with initialize_dummy_model(model_cls, model_config) as model:
|
||||
for modality, _, mm_kwargs in create_batched_mm_kwargs(model_config, processor):
|
||||
for method_name in inputs_parse_methods:
|
||||
print(
|
||||
f"Testing `{method_name}` with modality={modality} "
|
||||
f"and mm_kwargs{list(mm_kwargs.keys())}"
|
||||
)
|
||||
getattr(model, method_name)(modality=modality, **mm_kwargs)
|
||||
@@ -1,175 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# flake8: noqa
|
||||
"""Tests fp8 models against ground truth generation
|
||||
Note: these tests will only pass on L4 GPU.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_kv_cache_dtype
|
||||
from vllm.platforms import current_platform
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype,base_model,test_model",
|
||||
[
|
||||
# Test FP8 checkpoint w. fp8_e4m3 kv-cache scaling factors.
|
||||
(
|
||||
"fp8_e4m3",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"nm-testing/Llama-3.2-1B-Instruct-FP8-KV",
|
||||
),
|
||||
# Test BF16 checkpoint w. fp8_e5m2 kv-cache.
|
||||
(
|
||||
"fp8_e5m2",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
# Test BF16 checkpoint w. fp8_e4m3 kv-cache scaling factors in json.
|
||||
(
|
||||
"fp8_e4m3",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
],
|
||||
)
|
||||
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
@pytest.mark.parametrize("enforce_eager", [True])
|
||||
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
|
||||
# NOTE: Increasing this in this suite will fail CI because we currently cannot
|
||||
# reset distributed env properly. Use a value > 1 just when you test.
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_models(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
kv_cache_dtype: str,
|
||||
base_model: str,
|
||||
test_model: str,
|
||||
max_tokens: int,
|
||||
enforce_eager: bool,
|
||||
backend: str,
|
||||
tensor_parallel_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
Only checks log probs match to cover the discrepancy in
|
||||
numerical sensitive kernels.
|
||||
"""
|
||||
|
||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_rocm():
|
||||
pytest.skip(f"{kv_cache_dtype} is currently not supported on ROCm/HIP.")
|
||||
if kv_cache_dtype == "fp8_e5m2" and current_platform.is_cuda():
|
||||
pytest.skip(f"{kv_cache_dtype} is not supported by FLASH_ATTN on CUDA.")
|
||||
|
||||
if not flash_attn_supports_kv_cache_dtype(kv_cache_dtype):
|
||||
pytest.skip(
|
||||
f"{kv_cache_dtype} is not supported on this GPU type with {backend} attention."
|
||||
)
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
with vllm_runner(
|
||||
base_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype="auto",
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
test_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="fp16_kv_cache",
|
||||
name_1="fp8_kv_cache",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_model
|
||||
@pytest.mark.skipif(not current_platform.is_cpu(), reason="test for the CPU backend.")
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype,base_model,test_model",
|
||||
[
|
||||
# Test BF16 checkpoint w. fp8_e5m2 kv-cache.
|
||||
(
|
||||
"fp8_e5m2",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
],
|
||||
)
|
||||
# Due to low-precision numerical divergence, we only test logprob of 4 tokens
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
def test_cpu_models(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
kv_cache_dtype: str,
|
||||
base_model: str,
|
||||
test_model: str,
|
||||
max_tokens: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""
|
||||
Only checks log probs match to cover the discrepancy in
|
||||
numerical sensitive kernels.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
with vllm_runner(
|
||||
base_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
dtype="bfloat16",
|
||||
kv_cache_dtype="auto",
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
test_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
dtype="bfloat16",
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16_kv_cache",
|
||||
name_1="fp8_kv_cache",
|
||||
)
|
||||
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end accuracy test for GPT-OSS model quantization.
|
||||
|
||||
Config:
|
||||
Task: gsm8k_platinum
|
||||
Filter: flexible-extract
|
||||
n-shot: 5
|
||||
Metric: exact_match
|
||||
|
||||
Run: pytest tests/models/quantization/test_gpt_oss.py
|
||||
"""
|
||||
|
||||
import importlib.metadata
|
||||
import importlib.util
|
||||
from dataclasses import dataclass
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
import pytest
|
||||
import torch
|
||||
from packaging import version
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
mxfp4_w4a8_moe_quant_config,
|
||||
mxfp4_w4a16_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a8_moe import (
|
||||
aiter_triton_kernel_w4a8_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
aiter_triton_kernel_w4a16_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
else:
|
||||
|
||||
def on_gfx950() -> bool:
|
||||
return False
|
||||
|
||||
def on_gfx1250() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
MODEL_ACCURACIES = {
|
||||
# Full quantization: attention linears and MoE linears
|
||||
"amd/gpt-oss-20b-WFP8-AFP8-KVFP8": 0.89,
|
||||
# MoE linears only quantization
|
||||
"amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-FP8-KV-FP8": 0.89,
|
||||
# MoE linears only quantization
|
||||
# "amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-MXFP4-KV-FP8": 0.90,
|
||||
}
|
||||
|
||||
QUARK_MXFP4_AVAILABLE = importlib.util.find_spec("quark") is not None and version.parse(
|
||||
importlib.metadata.version("amd-quark")
|
||||
) >= version.parse("0.9.0")
|
||||
|
||||
|
||||
def has_huggingface_access(repo):
|
||||
try:
|
||||
hf_api().list_repo_refs(repo)
|
||||
return True
|
||||
except huggingface_hub.errors.RepositoryNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
HF_HUB_AMD_ORG_ACCESS = all(
|
||||
[has_huggingface_access(model_name) for model_name in MODEL_ACCURACIES]
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelCase:
|
||||
model_id: str
|
||||
tp: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvaluationConfig:
|
||||
model_name: str
|
||||
|
||||
def get_model_args(self, tp_size: int):
|
||||
return {
|
||||
"pretrained": self.model_name,
|
||||
"chat_template_args": {"reasoning_effort": "low"},
|
||||
"enable_thinking": True,
|
||||
"think_end_token": "200008",
|
||||
"tensor_parallel_size": tp_size,
|
||||
"dtype": "auto",
|
||||
"gpu_memory_utilization": 0.95,
|
||||
"trust_remote_code": False,
|
||||
"enable_prefix_caching": False,
|
||||
"enforce_eager": False,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(not QUARK_MXFP4_AVAILABLE, reason="amd-quark>=0.9 is not available")
|
||||
@pytest.mark.skipif(
|
||||
not HF_HUB_AMD_ORG_ACCESS,
|
||||
reason="Read access to huggingface.co/amd is required for this test.",
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
|
||||
@pytest.mark.parametrize("model_name, expected_accuracy", MODEL_ACCURACIES.items())
|
||||
def test_gpt_oss_attention_quantization(
|
||||
model_name: str,
|
||||
tp_size: int,
|
||||
expected_accuracy: float,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if tp_size > current_platform.device_count():
|
||||
pytest.skip("Not enough GPUs to run this test case")
|
||||
|
||||
if "amd/gpt-oss-20b-MoE-Quant-W-MXFP4-A-FP8-KV-FP8" in model_name and on_gfx950():
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
model_args = EvaluationConfig(model_name).get_model_args(tp_size)
|
||||
|
||||
extra_run_kwargs = {
|
||||
"gen_kwargs": {"max_gen_toks": 8000},
|
||||
"apply_chat_template": True,
|
||||
"fewshot_as_multiturn": True,
|
||||
"num_fewshot": 5,
|
||||
}
|
||||
|
||||
lm_eval_out = lm_eval.simple_evaluate(
|
||||
model="vllm",
|
||||
model_args=model_args,
|
||||
tasks="gsm8k_platinum",
|
||||
batch_size="auto",
|
||||
**extra_run_kwargs,
|
||||
)
|
||||
measured_accuracy = float(
|
||||
lm_eval_out["results"]["gsm8k_platinum"]["exact_match,flexible-extract"]
|
||||
)
|
||||
|
||||
rtol = 0.02
|
||||
assert measured_accuracy >= expected_accuracy - rtol, (
|
||||
f"Accuracy {measured_accuracy:.4f} is below threshold "
|
||||
f"{expected_accuracy - rtol:.4f} (expected >= {expected_accuracy} - {rtol})"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (on_gfx950() or on_gfx1250()),
|
||||
reason="AITER MXFP4 MoE requires gfx950 or gfx1250",
|
||||
)
|
||||
@pytest.mark.skipif(not is_aiter_found_and_supported(), reason="aiter is not installed")
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() == 0, reason="no gpu available")
|
||||
@pytest.mark.parametrize(
|
||||
"mxfp4_backend",
|
||||
[Mxfp4MoeBackend.AITER_MXFP4_FP8, Mxfp4MoeBackend.AITER_MXFP4_BF16],
|
||||
ids=["w4a8", "w4a16"],
|
||||
)
|
||||
@pytest.mark.parametrize("num_experts", [32, 64, 128])
|
||||
def test_aiter_mxfp4_moe_ignores_padded_rows(
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
num_experts: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
dist_init,
|
||||
) -> None:
|
||||
"""Garbage in cudagraph padding rows must not reach the unpadded outputs.
|
||||
|
||||
A cudagraph replay of a size-`TOKENS_PADDED` graph driven by
|
||||
`TOKENS_UNPADDED` real tokens leaves the trailing padding row containing
|
||||
-inf/inf/nan.
|
||||
"""
|
||||
TOKENS_PADDED = 8
|
||||
TOKENS_UNPADDED = 7
|
||||
|
||||
# `rocm_aiter_ops` snapshots the environment at import time, so setting the
|
||||
# env var alone is not enough.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
set_random_seed(0)
|
||||
device = torch.device("cuda")
|
||||
hidden_size = 2048
|
||||
intermediate_size = 512
|
||||
|
||||
num_experts = 32
|
||||
topk = 4
|
||||
is_w4a8 = mxfp4_backend is Mxfp4MoeBackend.AITER_MXFP4_FP8
|
||||
|
||||
# TODO: clean this in oracle/mxfp4.
|
||||
# `convert_gpt_oss_weight_to_mxfp4_moe_kernel_format` maps
|
||||
# AITER_MXFP4_BF16 onto the CK layout (`shuffle_weight_a16w4`), which the
|
||||
# triton monolithic cannot consume -- it wants the `_swizzle_mxfp4`
|
||||
# PrecisionConfig that the TRITON branch of the same converter produces.
|
||||
weight_backend = (
|
||||
Mxfp4MoeBackend.AITER_MXFP4_FP8 if is_w4a8 else Mxfp4MoeBackend.TRITON
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.w13_weight = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 2),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
layer.w2_weight = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(num_experts, hidden_size, intermediate_size // 2),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# Keep the exponents near 127 (2**0) so the dequantized weights stay in a sane
|
||||
# range and 255 (e8m0 NaN) is never hit.
|
||||
layer.w13_weight_scale = torch.randint(
|
||||
124,
|
||||
131,
|
||||
(num_experts, 2 * intermediate_size, hidden_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
layer.w2_weight_scale = torch.randint(
|
||||
124,
|
||||
131,
|
||||
(num_experts, hidden_size, intermediate_size // 32),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
layer.w13_bias = torch.randn(
|
||||
(num_experts, 2 * intermediate_size), dtype=torch.float32, device=device
|
||||
)
|
||||
layer.w2_bias = torch.randn(
|
||||
(num_experts, hidden_size), dtype=torch.float32, device=device
|
||||
)
|
||||
# Read off `layer`, not off the arguments, by the AITER_MXFP4_FP8 branch.
|
||||
layer.w13_input_scale = torch.full(
|
||||
(num_experts,), 0.1, dtype=torch.float32, device=device
|
||||
)
|
||||
layer.w2_input_scale = torch.full(
|
||||
(num_experts,), 0.1, dtype=torch.float32, device=device
|
||||
)
|
||||
|
||||
(
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
) = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
weight_backend,
|
||||
layer,
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
layer.w13_weight_scale,
|
||||
layer.w2_weight_scale,
|
||||
w13_bias=layer.w13_bias,
|
||||
w2_bias=layer.w2_bias,
|
||||
)
|
||||
|
||||
quant_config_factory = (
|
||||
mxfp4_w4a8_moe_quant_config if is_w4a8 else mxfp4_w4a16_moe_quant_config
|
||||
)
|
||||
quant_config = quant_config_factory(
|
||||
w1_scale=w13_scale,
|
||||
w2_scale=w2_scale,
|
||||
w1_bias=w13_bias,
|
||||
w2_bias=w2_bias,
|
||||
)
|
||||
|
||||
hidden_states = torch.randn(
|
||||
(TOKENS_PADDED, hidden_size), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
gating_output = torch.randn(
|
||||
(TOKENS_PADDED, num_experts), dtype=torch.bfloat16, device=device
|
||||
)
|
||||
|
||||
hidden_states[TOKENS_UNPADDED:, 0] = float("inf")
|
||||
hidden_states[TOKENS_UNPADDED:, 1] = float("-inf")
|
||||
hidden_states[TOKENS_UNPADDED:, 2] = float("nan")
|
||||
|
||||
gating_output[TOKENS_UNPADDED:, :] = float("-inf")
|
||||
gating_output[TOKENS_UNPADDED:, 0] = float("inf")
|
||||
gating_output[TOKENS_UNPADDED:, 1] = float("nan")
|
||||
gating_output[TOKENS_UNPADDED:, 2] = -float("nan")
|
||||
|
||||
is_padding = torch.zeros(TOKENS_PADDED, dtype=torch.bool, device=device)
|
||||
is_padding[TOKENS_UNPADDED:] = True
|
||||
|
||||
# Poison memory: after `del blocks`, further `torch.empty` in routing with wrongful
|
||||
# expert ids may read into these memory sections resulting in
|
||||
# potential memory access fault.
|
||||
blocks = []
|
||||
for numel in (16, 32, 64, 128, 256, 512, 1024, 4096, 1 << 14, 1 << 16, 1 << 20):
|
||||
blocks.append(torch.full((numel,), 100e7, dtype=torch.int32, device=device))
|
||||
# return blocks
|
||||
del blocks
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
with set_forward_context(None, get_current_vllm_config(), is_padding=is_padding):
|
||||
if is_w4a8:
|
||||
# AITER_MXFP4_FP8 routes through the triton monolithic expert, which
|
||||
# does its own top-k inside `aiter.ops.triton.moe.moe_routing`.
|
||||
output = aiter_triton_kernel_w4a8_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
w1=w13_weight,
|
||||
w2=w2_weight,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
global_num_experts=num_experts,
|
||||
unpadded_N_w1=2 * intermediate_size,
|
||||
unpadded_K_w1=hidden_size,
|
||||
unpadded_N_w2=hidden_size,
|
||||
unpadded_K_w2=intermediate_size,
|
||||
)
|
||||
else:
|
||||
output = aiter_triton_kernel_w4a16_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
w1=w13_weight,
|
||||
w2=w2_weight,
|
||||
gating_output=gating_output,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
global_num_experts=num_experts,
|
||||
unpadded_N_w1=2 * intermediate_size,
|
||||
unpadded_K_w1=hidden_size,
|
||||
unpadded_N_w2=hidden_size,
|
||||
unpadded_K_w2=intermediate_size,
|
||||
)
|
||||
|
||||
assert torch.isfinite(output[:TOKENS_UNPADDED]).all()
|
||||
@@ -1,88 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests AutoGPTQ (GPTQ with Marlin kernels) output correctness.
|
||||
|
||||
Note: Marlin internally uses locks to synchronize the threads. This can
|
||||
result in very slight nondeterminism for Marlin. As a result, we re-run the test
|
||||
up to 3 times to see if we pass.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.model_executor.layers.rotary_embedding import _ROPE_DICT
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = [
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "main"),
|
||||
# desc_act=True is a no-op for channelwise quantization.
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ", "gptq-8bit--1g-actorder_True"),
|
||||
("TechxGenus/gemma-2b-it-GPTQ", "main"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("auto_gptq")
|
||||
or current_platform.is_rocm()
|
||||
or not current_platform.is_cuda(),
|
||||
reason="auto_gptq is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@pytest.mark.parametrize("dtype", ["half", "bfloat16"])
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model,
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
model_name, revision = model
|
||||
|
||||
# Run marlin.
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
revision=revision,
|
||||
dtype=dtype,
|
||||
quantization="marlin",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=1,
|
||||
) as gptq_marlin_model:
|
||||
gptq_marlin_outputs = gptq_marlin_model.generate_greedy_logprobs(
|
||||
example_prompts[:-1], max_tokens, num_logprobs
|
||||
)
|
||||
_ROPE_DICT.clear() # clear rope cache to avoid rope dtype error
|
||||
|
||||
# Run gptq.
|
||||
# The naive gptq kernel doesn't support bf16 yet.
|
||||
# Here we always compare fp16/bf16 gpt marlin kernel
|
||||
# to fp16 gptq kernel.
|
||||
with vllm_runner(
|
||||
model_name=model_name,
|
||||
revision=revision,
|
||||
dtype="half",
|
||||
quantization="gptq",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=1,
|
||||
) as gptq_model:
|
||||
gptq_outputs = gptq_model.generate_greedy_logprobs(
|
||||
example_prompts[:-1], max_tokens, num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=gptq_outputs,
|
||||
outputs_1_lst=gptq_marlin_outputs,
|
||||
name_0="gptq",
|
||||
name_1="gptq_marlin",
|
||||
)
|
||||
@@ -1,84 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# flake8: noqa
|
||||
"""Tests Model Optimizer fp8 models against ground truth generation
|
||||
Note: these tests will only pass on H100
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = ["nvidia/Llama-3.1-8B-Instruct-FP8"]
|
||||
|
||||
EXPECTED_STRS_MAP = {
|
||||
"nvidia/Llama-3.1-8B-Instruct-FP8": [
|
||||
"You're referring to VLLM, a high-performance Large Language Model (LLM) inference and",
|
||||
"Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ",
|
||||
"The comparison between artificial intelligence (AI) and human intelligence in terms of processing information is a complex and",
|
||||
'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
|
||||
"**The Spark of Imagination**\n\nZeta-5, a sleek and efficient robot, whir",
|
||||
"The COVID-19 pandemic has had a profound impact on global economic structures and business models, leading to",
|
||||
"The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of",
|
||||
"Here are the translations:\n\n**Japanese:** 「早起きは早く獲物をとる",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# This test compares against golden strings for exact match since
|
||||
# there is no baseline implementation to compare against
|
||||
# and is unstable w.r.t specifics of the fp8 implementation or
|
||||
# the hardware being run on.
|
||||
# Disabled to prevent it from breaking the build
|
||||
@pytest.mark.skip(
|
||||
reason="Prevent unstable test based on golden strings from breaking the build."
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_models(example_prompts, model_name) -> None:
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
quantization="modelopt",
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
formatted_prompts = [
|
||||
tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
for prompt in example_prompts
|
||||
]
|
||||
params = SamplingParams(max_tokens=20, temperature=0)
|
||||
generations: list[str] = []
|
||||
# Note: these need to be run 1 at a time due to numerical precision,
|
||||
# since the expected strs were generated this way.
|
||||
for prompt in formatted_prompts:
|
||||
outputs = llm.generate(prompt, params)
|
||||
generations.append(outputs[0].outputs[0].text)
|
||||
del llm
|
||||
|
||||
print(model_name, generations)
|
||||
expected_strs = EXPECTED_STRS_MAP[model_name]
|
||||
for i in range(len(example_prompts)):
|
||||
generated_str = generations[i]
|
||||
expected_str = expected_strs[i]
|
||||
assert expected_str == generated_str, (
|
||||
f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}"
|
||||
)
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# flake8: noqa
|
||||
"""Tests Quark mxfp4 models against ground truth generation"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
MODELS = ["amd/Llama-2-7b-chat-hf-wmxfp4-amxfp4-kvfp8-scale-uint8"]
|
||||
|
||||
EXPECTED_STRS_MAP = {
|
||||
"amd/Llama-2-7b-chat-hf-wmxfp4-amxfp4-kvfp8-scale-uint8": [
|
||||
"\n### Key Features\n\n* **High-throughput Inference**: vLL",
|
||||
"\nArtificial intelligence (AI) has evolved significantly since its inception in the 1",
|
||||
"Artificial intelligence (AI) and human intelligence (HI) are two distinct concepts that have been",
|
||||
"A neural network is a machine learning model inspired by the structure of the human brain. It consists of",
|
||||
"\nTitle: The Dreaming Robot\n\nAs the sun set on the bustling metropol",
|
||||
"\nThe COVID-19 pandemic has had a profound impact on global economic structures and business",
|
||||
"The Mona Lisa painting, created by Leonardo da Vinci in the early 16th",
|
||||
" everybody knows this proverbial saying, but did you know that it's not entirely accurate?",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="Model to be released in the future")
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_models(example_prompts, model_name) -> None:
|
||||
sampling_params = SamplingParams(max_tokens=20, temperature=0)
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
kv_cache_dtype="fp8",
|
||||
quantization="quark",
|
||||
)
|
||||
outputs = llm.generate(example_prompts, sampling_params)
|
||||
for i, output in enumerate(outputs):
|
||||
output_str = output.outputs[0].text
|
||||
expected_str = EXPECTED_STRS_MAP[model_name][i]
|
||||
assert expected_str == output_str, (
|
||||
f"Expected: {expected_str!r}\nvLLM: {output_str!r}"
|
||||
)
|
||||
@@ -1,255 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for online MXFP8 quantization.
|
||||
|
||||
Loads a BF16 model with ``--quantization mxfp8`` (online quantization) and
|
||||
compares log-probabilities against the same model served in BF16 without
|
||||
quantization. This exercises the full pipeline: config parsing,
|
||||
``Mxfp8OnlineLinearMethod``, ``Mxfp8OnlineMoEMethod``, weight loading,
|
||||
online quantization / shuffling, and inference through ``apply_monolithic``.
|
||||
|
||||
Layer skipping (``modules_to_not_convert``) is configured in the model's
|
||||
``config.json`` under ``quantization_config`` and is not tested here.
|
||||
|
||||
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
|
||||
diverse prompts from ``tests/prompts/example.txt``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
# A small MoE model that fits on a single GPU and has both linear + MoE layers.
|
||||
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
# A small dense model (no MoE) to validate the linear-only path.
|
||||
DENSE_MODEL = "Qwen/Qwen3-0.6B"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_TOKENS = 4
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_mxfp8_logprobs(
|
||||
vllm_runner,
|
||||
quant_baseline_logprobs,
|
||||
example_prompts,
|
||||
model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare BF16 baseline logprobs against online MXFP8-quantized model.
|
||||
|
||||
Runs the same model twice -- once in BF16 (baseline) and once with
|
||||
online MXFP8 quantization -- then checks that the top log-probabilities
|
||||
are close. Only 4 tokens are generated to keep the test fast while
|
||||
still catching numerical divergence.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
baseline_outputs = quant_baseline_logprobs(
|
||||
model,
|
||||
example_prompts,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_tokens=MAX_TOKENS,
|
||||
num_logprobs=NUM_LOG_PROBS,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
quantization="mxfp8",
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16",
|
||||
name_1="mxfp8",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AITER MXFP8 MoE backend is ROCm-only.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
def test_mxfp8_aiter_requires_swigluoai_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_mxfp8_moe.AiterMxfp8Experts,
|
||||
"_supports_current_device",
|
||||
staticmethod(lambda: True),
|
||||
)
|
||||
|
||||
config = FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend="aiter",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="requires activation=swigluoai_uninterleave"):
|
||||
select_mxfp8_moe_backend(config)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AITER MXFP8 MoE backend is ROCm-only.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
def test_mxfp8_aiter_requires_swigluoai_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_mxfp8_moe.AiterMxfp8Experts,
|
||||
"_supports_current_device",
|
||||
staticmethod(lambda: True),
|
||||
)
|
||||
|
||||
config = FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend="aiter",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="hardcodes SwiGLU-OAI"):
|
||||
select_mxfp8_moe_backend(config)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="AITER MXFP8 MoE backend is ROCm-only.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
def test_mxfp8_aiter_accepts_swigluoai_params(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts import aiter_mxfp8_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp8 import (
|
||||
select_mxfp8_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
aiter_mxfp8_moe.AiterMxfp8Experts,
|
||||
"_supports_current_device",
|
||||
staticmethod(lambda: True),
|
||||
)
|
||||
|
||||
config = FusedMoEConfig(
|
||||
num_experts=8,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=256,
|
||||
num_local_experts=8,
|
||||
num_logical_experts=8,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SWIGLUOAI_UNINTERLEAVE,
|
||||
in_dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_backend="aiter",
|
||||
swiglu_alpha=aiter_mxfp8_moe._AITER_SWIGLU_ALPHA,
|
||||
swiglu_beta=aiter_mxfp8_moe._AITER_SWIGLU_BETA,
|
||||
)
|
||||
|
||||
backend, experts_cls = select_mxfp8_moe_backend(config)
|
||||
|
||||
assert backend == Fp8MoeBackend.AITER_MXFP8
|
||||
assert experts_cls is aiter_mxfp8_moe.AiterMxfp8Experts
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("mxfp8"),
|
||||
reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_mxfp8_generation(vllm_runner, model: str) -> None:
|
||||
"""Smoke test: verify online MXFP8 model generates coherent text."""
|
||||
prompt = "1 2 3 4 5"
|
||||
with vllm_runner(
|
||||
model,
|
||||
enforce_eager=True,
|
||||
quantization="mxfp8",
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
) as vllm_model:
|
||||
output = vllm_model.generate_greedy([prompt], max_tokens=5)
|
||||
|
||||
generated = output[0][1]
|
||||
assert len(generated) > len(prompt), (
|
||||
f"MXFP8 model produced no new tokens. Output: {generated!r}"
|
||||
)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# flake8: noqa
|
||||
"""Tests Model Optimizer nvfp4 models against ground truth generation
|
||||
Note: these tests will only pass on B200
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
import pytest
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import LLM, SamplingParams
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "true"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
|
||||
MODELS = ["nvidia/Llama-3.3-70B-Instruct-FP4"]
|
||||
|
||||
EXPECTED_STRS_MAP = {
|
||||
"nvidia/Llama-3.3-70B-Instruct-FP4": [
|
||||
"vLLM (Vectorized Large Language Model) is indeed a high-throughput and memory-efficient inference",
|
||||
"Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ",
|
||||
"Artificial intelligence (AI) and human intelligence (HI) are two distinct forms of intelligence that process",
|
||||
"A neural network is a type of machine learning model inspired by the structure and function of the human brain",
|
||||
"In the heart of a cutting-edge robotics lab, a team of engineers had been working tirelessly to push",
|
||||
"The COVID-19 pandemic has had a profound impact on global economic structures and future business models, leading",
|
||||
"The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of",
|
||||
"Here are the translations:\n\n* Japanese: (Sasuga no tori ga miwa o ts",
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
# This test compares against golden strings for exact match since
|
||||
# there is no baseline implementation to compare against
|
||||
# and is unstable w.r.t specifics of the fp4 implementation or
|
||||
# the hardware being run on.
|
||||
# Disabled to prevent it from breaking the build
|
||||
@pytest.mark.skip(
|
||||
reason="Prevent unstable test based on golden strings from breaking the build "
|
||||
" and test input model being too large and hanging the system."
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("modelopt_fp4"),
|
||||
reason="modelopt_fp4 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.parametrize("model_name", MODELS)
|
||||
def test_models(example_prompts, model_name) -> None:
|
||||
llm = LLM(
|
||||
model=model_name,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
trust_remote_code=True,
|
||||
enforce_eager=True,
|
||||
quantization="modelopt_fp4",
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
formatted_prompts = [
|
||||
tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": prompt}],
|
||||
tokenize=False,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
for prompt in example_prompts
|
||||
]
|
||||
params = SamplingParams(max_tokens=20, temperature=0)
|
||||
generations: List[str] = []
|
||||
# Note: these need to be run 1 at a time due to numerical precision,
|
||||
# since the expected strs were generated this way.
|
||||
for prompt in formatted_prompts:
|
||||
outputs = llm.generate(prompt, params)
|
||||
generations.append(outputs[0].outputs[0].text)
|
||||
del llm
|
||||
|
||||
print(model_name, generations)
|
||||
expected_strs = EXPECTED_STRS_MAP[model_name]
|
||||
for i in range(len(example_prompts)):
|
||||
generated_str = generations[i]
|
||||
expected_str = expected_strs[i]
|
||||
assert expected_str == generated_str, (
|
||||
f"Test{i}:\nExpected: {expected_str!r}\nvLLM: {generated_str!r}"
|
||||
)
|
||||
|
||||
|
||||
EAGER = [True, False]
|
||||
|
||||
SM_100_NVFP4_BACKENDS = [
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_cutlass",
|
||||
"torch",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["nvidia/Llama-3.1-8B-Instruct-NVFP4"])
|
||||
@pytest.mark.parametrize("eager", EAGER)
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
"emulation",
|
||||
"flashinfer_cutedsl",
|
||||
"flashinfer_cudnn",
|
||||
"flashinfer_trtllm", # the small seq_len ensures trtllm_8x4_layout backend is used
|
||||
"flashinfer_cutlass",
|
||||
"torch",
|
||||
],
|
||||
)
|
||||
def test_nvfp4(vllm_runner, model, eager, backend):
|
||||
if backend == "flashinfer_cutedsl" and not (
|
||||
current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("The flashinfer_cutedsl backend is only supported on SM10x")
|
||||
|
||||
if (
|
||||
not current_platform.has_device_capability(100)
|
||||
and backend in SM_100_NVFP4_BACKENDS
|
||||
):
|
||||
pytest.skip(
|
||||
f"The backend {backend} is not supported with current_platform.has_device_capability(100) == False"
|
||||
)
|
||||
|
||||
with vllm_runner(model, enforce_eager=eager, linear_backend=backend) as llm:
|
||||
output = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
|
||||
assert output[0][1] == "1 2 3 4 5 6"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"nvidia/Qwen3-30B-A3B-NVFP4",
|
||||
"RedHatAI/Qwen3-30B-A3B-NVFP4",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("backend", ["emulation"])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
|
||||
)
|
||||
def test_nvfp4_moe(vllm_runner, model, backend):
|
||||
with vllm_runner(
|
||||
model,
|
||||
moe_backend=backend,
|
||||
linear_backend=backend,
|
||||
load_format="dummy",
|
||||
hf_overrides={"num_hidden_layers": 2},
|
||||
) as llm:
|
||||
_ = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2)
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""End-to-end accuracy tests for per-token-head KV cache quantization.
|
||||
|
||||
Compares logprobs between a baseline bf16 model and the same model with
|
||||
per-token-head quantized KV cache (int8 or fp8) using the Triton attention
|
||||
backend.
|
||||
|
||||
Run: pytest tests/models/quantization/test_per_token_kv_cache.py -v -s
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Per-token-head KV cache requires CUDA or ROCm GPU.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,test_model",
|
||||
[
|
||||
(
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_cache_dtype",
|
||||
["int4_per_token_head", "int8_per_token_head", "fp8_per_token_head"],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [4])
|
||||
@pytest.mark.parametrize("enforce_eager", [True])
|
||||
@pytest.mark.parametrize("backend", ["TRITON_ATTN"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1])
|
||||
def test_per_token_head_kv_cache_accuracy(
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
base_model: str,
|
||||
test_model: str,
|
||||
kv_cache_dtype: str,
|
||||
max_tokens: int,
|
||||
enforce_eager: bool,
|
||||
backend: str,
|
||||
tensor_parallel_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare logprobs between bf16 baseline and per-token-head quantized KV
|
||||
cache.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
with vllm_runner(
|
||||
base_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype="auto",
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
baseline_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
test_model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enforce_eager=enforce_eager,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
attention_config={"backend": backend},
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16_kv_cache",
|
||||
name_1=f"{kv_cache_dtype}_kv_cache",
|
||||
)
|
||||
@@ -1,407 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the fused QSA prepare kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.models.qwen4_exp.common.qsa_cache import (
|
||||
canonical_qsa_rope_positions,
|
||||
circular_qsa_slot_mapping,
|
||||
compressed_qsa_slot_mapping,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.indexer_qsa import apply_qsa_rope
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa import (
|
||||
qsa_compress_groups_with_ratio,
|
||||
qsa_store_cache_rows,
|
||||
)
|
||||
from vllm.models.qwen4_exp.nvidia.ops.qsa_prepare import qsa_prepare
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
|
||||
requires_qsa_kernels = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not HAS_TRITON,
|
||||
reason="QSA kernels require CUDA and Triton",
|
||||
)
|
||||
|
||||
HQ, D = 4, 128
|
||||
CR = 4
|
||||
MROPE_SECTION = (11, 11, 10)
|
||||
EPS = 1e-6
|
||||
BLOCK_SIZE = 16
|
||||
COMP_PAGE = BLOCK_SIZE // CR
|
||||
ROPE_POS_OFFSET = D
|
||||
RTOL = 1.6e-2
|
||||
ATOL = 1e-2
|
||||
MIXED_BATCH = ([260, 259, 138], [1, 1, 37], [8, 8, 8])
|
||||
MAIN_HQ, MAIN_HK, MAIN_D, MAIN_PAGE = 6, 2, 256, 16
|
||||
|
||||
|
||||
def _make_block_table(block_counts):
|
||||
num_blocks = sum(block_counts)
|
||||
table = torch.full((len(block_counts), max(block_counts)), -1, dtype=torch.int32)
|
||||
physical_blocks = torch.randperm(num_blocks)
|
||||
offset = 0
|
||||
for request, count in enumerate(block_counts):
|
||||
table[request, :count] = physical_blocks[offset : offset + count]
|
||||
offset += count
|
||||
return table, num_blocks
|
||||
|
||||
|
||||
def assert_fp8_within_one_ulp(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
# e4m3 is sign-magnitude, so within a sign the uint8 code order matches the
|
||||
# value order and one ulp is one code step. The two paths' intermediates
|
||||
# differ in the pooling accumulation order, which at denormal magnitudes
|
||||
# (absolute grid step 2^-9) shows up as up to 2 code steps.
|
||||
code_diff = (
|
||||
actual.view(torch.uint8).int() - expected.view(torch.uint8).int()
|
||||
).abs()
|
||||
abs_diff = (actual.float() - expected.float()).abs()
|
||||
assert bool(((code_diff <= 1) | (abs_diff <= 2**-8)).all())
|
||||
|
||||
|
||||
def _make_main_inputs(num_tokens: int, fp8_cache: bool) -> dict:
|
||||
"""Random main-attention arguments with a paged cache in vLLM's layout."""
|
||||
num_blocks = num_tokens // MAIN_PAGE + 2
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
MAIN_HK,
|
||||
MAIN_PAGE,
|
||||
2 * MAIN_D,
|
||||
dtype=torch.uint8 if fp8_cache else torch.bfloat16,
|
||||
device="cuda",
|
||||
).transpose(1, 2)
|
||||
slots = torch.randperm(num_blocks * MAIN_PAGE, device="cuda")[:num_tokens]
|
||||
slots[-1] = -1
|
||||
norm_weights = torch.randn(2, MAIN_D, dtype=torch.bfloat16, device="cuda") * 0.2
|
||||
qkv_width = 2 * (MAIN_HQ + MAIN_HK) * MAIN_D
|
||||
return dict(
|
||||
main_qkv=torch.randn(
|
||||
num_tokens,
|
||||
qkv_width + (HQ + 1) * D,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)[:, :qkv_width],
|
||||
main_q_norm_weight=norm_weights[0],
|
||||
main_k_norm_weight=norm_weights[1],
|
||||
main_eps=EPS,
|
||||
main_kv_cache=cache.view(torch.float8_e4m3fn) if fp8_cache else cache,
|
||||
main_slot_mapping=slots,
|
||||
main_k_scale=0.5,
|
||||
main_v_scale=2.0,
|
||||
)
|
||||
|
||||
|
||||
def _check_main_outputs(main: dict, q_out, gate_out, rope, positions) -> None:
|
||||
"""Compare with fused_qk_rmsnorm_rope_gate + reshape_and_cache_flash."""
|
||||
from vllm._custom_ops import reshape_and_cache_flash
|
||||
from vllm.model_executor.layers.fused_qk_norm_rope import (
|
||||
fused_qk_rmsnorm_rope_gate,
|
||||
)
|
||||
|
||||
q_gate, k, v = main["main_qkv"].split(
|
||||
[2 * MAIN_HQ * MAIN_D, MAIN_HK * MAIN_D, MAIN_HK * MAIN_D], dim=-1
|
||||
)
|
||||
q, k, gate = fused_qk_rmsnorm_rope_gate(
|
||||
q_gate,
|
||||
k,
|
||||
main["main_q_norm_weight"],
|
||||
main["main_k_norm_weight"],
|
||||
rope.cos_sin_cache,
|
||||
positions,
|
||||
EPS,
|
||||
MAIN_HQ,
|
||||
MAIN_HK,
|
||||
MAIN_D,
|
||||
rope.rotary_dim,
|
||||
mrope_section=MROPE_SECTION if positions.ndim == 2 else None,
|
||||
norm_beta=1.0,
|
||||
)
|
||||
kv_cache = main["main_kv_cache"]
|
||||
fp8_cache = kv_cache.dtype == torch.float8_e4m3fn
|
||||
cache = torch.zeros_like(kv_cache.view(torch.uint8) if fp8_cache else kv_cache)
|
||||
key_cache, value_cache = cache.split(MAIN_D, dim=-1)
|
||||
reshape_and_cache_flash(
|
||||
k.view(-1, MAIN_HK, MAIN_D),
|
||||
v.view(-1, MAIN_HK, MAIN_D),
|
||||
key_cache,
|
||||
value_cache,
|
||||
main["main_slot_mapping"],
|
||||
"fp8" if fp8_cache else "auto",
|
||||
torch.tensor(main["main_k_scale"], device="cuda"),
|
||||
torch.tensor(main["main_v_scale"], device="cuda"),
|
||||
)
|
||||
torch.testing.assert_close(q_out.flatten(1), q, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(gate_out.flatten(1), gate)
|
||||
if fp8_cache:
|
||||
assert_fp8_within_one_ulp(kv_cache, cache.view(torch.float8_e4m3fn))
|
||||
else:
|
||||
torch.testing.assert_close(kv_cache, cache, rtol=RTOL, atol=ATOL)
|
||||
|
||||
|
||||
@requires_qsa_kernels
|
||||
@pytest.mark.usefixtures("default_vllm_config")
|
||||
@pytest.mark.parametrize("indexer_dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize(
|
||||
"mrope,is_2d_positions,cache_rope_positions,state_size,seq_lens,query_lens,history_lens",
|
||||
[
|
||||
pytest.param(True, True, True, 4, *MIXED_BATCH, id="mrope"),
|
||||
pytest.param(False, False, False, 4, *MIXED_BATCH, id="text"),
|
||||
pytest.param(True, False, True, 8, *MIXED_BATCH, id="mrope-model-1d"),
|
||||
pytest.param(True, True, False, 4, *MIXED_BATCH, id="mrope-no-position-cache"),
|
||||
pytest.param(
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
8,
|
||||
*MIXED_BATCH,
|
||||
id="mrope-model-1d-no-position-cache",
|
||||
),
|
||||
pytest.param(
|
||||
False, False, True, 4, *MIXED_BATCH, id="text-with-position-cache"
|
||||
),
|
||||
pytest.param(True, True, True, 4, [37], [37], [0], id="fresh"),
|
||||
pytest.param(True, True, True, 4, [4097], [4097], [0], id="tiled"),
|
||||
],
|
||||
)
|
||||
def test_qsa_fused_prepare_matches_unfused(
|
||||
indexer_dtype,
|
||||
mrope,
|
||||
is_2d_positions,
|
||||
cache_rope_positions,
|
||||
state_size,
|
||||
seq_lens,
|
||||
query_lens,
|
||||
history_lens,
|
||||
) -> None:
|
||||
from flashinfer.norm import gemma_rmsnorm
|
||||
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
|
||||
device = "cuda"
|
||||
rope_params = {
|
||||
"partial_rotary_factor": 0.25,
|
||||
"rope_theta": 10000000,
|
||||
"rope_type": "default",
|
||||
}
|
||||
if mrope:
|
||||
rope_params["mrope_interleaved"] = True
|
||||
rope_params["mrope_section"] = list(MROPE_SECTION)
|
||||
with torch.device(device):
|
||||
rope = get_rope(
|
||||
head_size=256,
|
||||
max_position=32768,
|
||||
rope_parameters=rope_params,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
token_to_req = torch.cat(
|
||||
[
|
||||
torch.full((length,), request, dtype=torch.int32)
|
||||
for request, length in enumerate(query_lens)
|
||||
]
|
||||
).to(device)
|
||||
logical_positions = torch.cat(
|
||||
[
|
||||
torch.arange(seq_len - query_len, seq_len, dtype=torch.int64)
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
]
|
||||
).to(device)
|
||||
num_tokens = logical_positions.numel()
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
).to(device)
|
||||
positions = (
|
||||
torch.stack(
|
||||
[
|
||||
logical_positions,
|
||||
logical_positions // 7 + 3,
|
||||
logical_positions // 13 + 11,
|
||||
]
|
||||
)
|
||||
if is_2d_positions
|
||||
else logical_positions
|
||||
)
|
||||
position_rows = (
|
||||
canonical_qsa_rope_positions(positions)
|
||||
if cache_rope_positions
|
||||
else logical_positions.view(-1, 1, 1).expand(-1, 1, 3)
|
||||
)
|
||||
|
||||
compressed_block_counts = [
|
||||
(seq_len // CR + COMP_PAGE - 1) // COMP_PAGE for seq_len in seq_lens
|
||||
]
|
||||
raw_block_table, num_raw_blocks = _make_block_table([1] * len(seq_lens))
|
||||
compressed_block_table, num_compressed_blocks = _make_block_table(
|
||||
compressed_block_counts
|
||||
)
|
||||
raw_block_table = raw_block_table.to(device)
|
||||
raw_slots = circular_qsa_slot_mapping(
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
state_size,
|
||||
query_start_loc,
|
||||
)
|
||||
compressed_slots = compressed_qsa_slot_mapping(
|
||||
compressed_block_table.to(device),
|
||||
token_to_req,
|
||||
logical_positions,
|
||||
COMP_PAGE,
|
||||
CR,
|
||||
)
|
||||
group_counts = torch.tensor(
|
||||
[
|
||||
seq_len // CR - (seq_len - query_len) // CR
|
||||
for seq_len, query_len in zip(seq_lens, query_lens)
|
||||
],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
k_work_counts = torch.maximum(group_counts, torch.ones_like(group_counts))
|
||||
k_start_loc = torch.cat([k_work_counts.new_zeros(1), k_work_counts.cumsum(0)])
|
||||
work_requests = torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32, device=device),
|
||||
k_work_counts,
|
||||
)
|
||||
local_work = torch.arange(
|
||||
int(k_start_loc[-1]), dtype=torch.int32, device=device
|
||||
) - torch.repeat_interleave(k_start_loc[:-1], k_work_counts)
|
||||
max_k_work = (num_tokens + (CR - 1) * len(query_lens)) // CR
|
||||
k_work_metadata = torch.full((max_k_work, 2), -1, dtype=torch.int32, device=device)
|
||||
k_work_metadata[: work_requests.numel()] = torch.stack(
|
||||
(work_requests, local_work), dim=1
|
||||
)
|
||||
|
||||
raw_width = D + 12 if cache_rope_positions else D
|
||||
# Match vLLM's padded-page cache layout: rows are contiguous, while physical
|
||||
# blocks have a larger stride than their logical contents.
|
||||
raw_page_elements = state_size * raw_width
|
||||
fused_raw_storage = torch.zeros(
|
||||
num_raw_blocks,
|
||||
raw_page_elements + 16,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
fused_raw = torch.as_strided(
|
||||
fused_raw_storage,
|
||||
(num_raw_blocks, state_size, 1, raw_width),
|
||||
(raw_page_elements + 16, raw_width, raw_width, 1),
|
||||
)
|
||||
for request, history_len in enumerate(history_lens):
|
||||
history_end = seq_lens[request] - query_lens[request]
|
||||
for position in range(history_end - history_len, history_end):
|
||||
block = int(raw_block_table[request, 0])
|
||||
row = fused_raw[block, position % state_size, 0]
|
||||
row[:D] = torch.randn(D, dtype=torch.bfloat16, device=device)
|
||||
if cache_rope_positions:
|
||||
row[ROPE_POS_OFFSET:].view(torch.int64).copy_(
|
||||
torch.tensor(
|
||||
[position, position // 7 + 3, position // 13 + 11],
|
||||
dtype=torch.int64,
|
||||
device=device,
|
||||
)
|
||||
)
|
||||
unfused_raw = fused_raw.clone()
|
||||
compressed_page_elements = COMP_PAGE * D
|
||||
fused_compressed_storage = torch.zeros(
|
||||
num_compressed_blocks,
|
||||
compressed_page_elements + 16,
|
||||
dtype=indexer_dtype,
|
||||
device=device,
|
||||
)
|
||||
fused_compressed = torch.as_strided(
|
||||
fused_compressed_storage,
|
||||
(num_compressed_blocks, COMP_PAGE, 1, D),
|
||||
(compressed_page_elements + 16, D, D, 1),
|
||||
)
|
||||
unfused_compressed = fused_compressed.clone()
|
||||
|
||||
# TP=4 QKVG width makes the indexer slice row-strided, as in the merged
|
||||
# projection. The exact width is unimportant; the larger stride is tested.
|
||||
main_qkvg_width = 3584
|
||||
projected_qk = torch.randn(
|
||||
num_tokens, main_qkvg_width + (HQ + 1) * D, dtype=torch.bfloat16, device=device
|
||||
)[:, main_qkvg_width:]
|
||||
q_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
k_weight = torch.randn(D, dtype=torch.bfloat16, device=device) * 0.2
|
||||
|
||||
fused_query = torch.empty(num_tokens, HQ, D, dtype=indexer_dtype, device=device)
|
||||
# The main-attention cache follows the recipe's pairing with the indexer.
|
||||
main = _make_main_inputs(num_tokens, indexer_dtype == torch.float8_e4m3fn)
|
||||
main_q_out, main_gate_out = qsa_prepare(
|
||||
projected_qk[:, : HQ * D],
|
||||
projected_qk[:, HQ * D :],
|
||||
positions,
|
||||
rope.cos_sin_cache,
|
||||
q_weight,
|
||||
k_weight,
|
||||
EPS,
|
||||
fused_query,
|
||||
fused_raw,
|
||||
raw_slots,
|
||||
raw_block_table,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
fused_compressed,
|
||||
compressed_slots,
|
||||
k_work_metadata,
|
||||
compress_ratio=CR,
|
||||
mrope_section=MROPE_SECTION if mrope else None,
|
||||
rope_pos_offset=ROPE_POS_OFFSET if cache_rope_positions else None,
|
||||
**main,
|
||||
)
|
||||
_check_main_outputs(main, main_q_out, main_gate_out, rope, positions)
|
||||
|
||||
unfused_query = projected_qk[:, : HQ * D].reshape(num_tokens, HQ, D)
|
||||
unfused_query = gemma_rmsnorm(
|
||||
unfused_query.reshape(-1, D), q_weight, EPS
|
||||
).reshape_as(unfused_query)
|
||||
unfused_query = apply_qsa_rope(rope, positions, unfused_query)
|
||||
|
||||
raw_keys = unfused_raw[..., :D]
|
||||
rope_positions = (
|
||||
unfused_raw[..., ROPE_POS_OFFSET:].view(torch.int64)
|
||||
if cache_rope_positions
|
||||
else None
|
||||
)
|
||||
pooled, first_positions = qsa_compress_groups_with_ratio(
|
||||
projected_qk[:, HQ * D :].reshape(-1, 1, D),
|
||||
position_rows,
|
||||
raw_keys,
|
||||
raw_block_table,
|
||||
token_to_req,
|
||||
query_start_loc,
|
||||
logical_positions,
|
||||
compressed_slots,
|
||||
CR,
|
||||
rope_positions,
|
||||
)
|
||||
compressed_rows = gemma_rmsnorm(pooled.reshape(-1, D), k_weight, EPS).reshape(
|
||||
-1, 1, D
|
||||
)
|
||||
group_positions = (
|
||||
first_positions.transpose(0, 1) if mrope else first_positions[:, 0]
|
||||
)
|
||||
compressed_rows = apply_qsa_rope(rope, group_positions, compressed_rows)
|
||||
qsa_store_cache_rows(unfused_compressed, compressed_slots, compressed_rows)
|
||||
qsa_store_cache_rows(raw_keys, raw_slots, projected_qk[:, HQ * D :])
|
||||
if rope_positions is not None:
|
||||
qsa_store_cache_rows(rope_positions, raw_slots, position_rows)
|
||||
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
# Both paths round the same ~bf16 intermediates to e4m3. Bitwise
|
||||
# equality (the dsv4 indexer test's bar) does not hold here: the 1D
|
||||
# reference rope (forward_cuda) differs from _norm_rope by up to 1
|
||||
# bf16 ulp, and even the MRoPE pair flips a code occasionally.
|
||||
unfused_query = unfused_query.to(indexer_dtype)
|
||||
assert_fp8_within_one_ulp(fused_query, unfused_query)
|
||||
else:
|
||||
torch.testing.assert_close(fused_query, unfused_query, rtol=RTOL, atol=ATOL)
|
||||
assert torch.equal(fused_raw.view(torch.int16), unfused_raw.view(torch.int16))
|
||||
if indexer_dtype == torch.float8_e4m3fn:
|
||||
assert_fp8_within_one_ulp(fused_compressed, unfused_compressed)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
fused_compressed, unfused_compressed, rtol=RTOL, atol=ATOL
|
||||
)
|
||||
@@ -1,185 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import multi_gpu_marks
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.metrics.reader import Counter
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
def _small_hyv4_config(config):
|
||||
# Apply to both target and MTP draft while retaining MLA/indexer dimensions.
|
||||
config.update(
|
||||
{
|
||||
"hidden_size": 512,
|
||||
"num_hidden_layers": 2,
|
||||
"num_attention_heads": 64,
|
||||
"q_lora_rank": 128,
|
||||
"intermediate_size": 512,
|
||||
"moe_intermediate_size": 256,
|
||||
"n_routed_experts": 4,
|
||||
"num_experts_per_tok": 2,
|
||||
"n_shared_experts": 1,
|
||||
"enable_ihc": True,
|
||||
"index_n_heads": 32,
|
||||
"index_topk": 2048,
|
||||
"layer_types": ["deepseek_sparse_attention"] * 2,
|
||||
"indexer_types": ["full", "shared"],
|
||||
"num_nextn_predict_layers": 1,
|
||||
"vocab_size": 256,
|
||||
"bos_token_id": 1,
|
||||
"eos_token_id": 2,
|
||||
"pad_token_id": 0,
|
||||
}
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _num_captured_graphs(_worker):
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
|
||||
return compilation_counter.num_cudagraph_captured
|
||||
|
||||
|
||||
def _piecewise_compile_state(worker):
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
|
||||
# `cudagraph_mode` is returned by name: the enum does not survive the
|
||||
# collective_rpc round trip, it arrives as its bare value.
|
||||
return (
|
||||
compilation_counter.num_models_seen,
|
||||
compilation_counter.num_cudagraph_captured,
|
||||
worker.vllm_config.compilation_config.cudagraph_mode.name,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp,enforce_eager,mtp,dtype,kv_cache_dtype",
|
||||
[
|
||||
pytest.param(1, True, False, "bfloat16", "auto", id="eager-tp1"),
|
||||
pytest.param(1, True, False, "float16", "auto", id="eager-fp16"),
|
||||
pytest.param(1, True, False, "bfloat16", "fp8", id="eager-fp8-kv"),
|
||||
pytest.param(
|
||||
2,
|
||||
False,
|
||||
False,
|
||||
"bfloat16",
|
||||
"auto",
|
||||
id="graphs-tp2",
|
||||
marks=multi_gpu_marks(num_gpus=2),
|
||||
),
|
||||
pytest.param(1, True, True, "bfloat16", "auto", id="eager-mtp"),
|
||||
],
|
||||
)
|
||||
def test_hyv4_dummy_generation(
|
||||
vllm_runner,
|
||||
enable_pickle,
|
||||
tp: int,
|
||||
enforce_eager: bool,
|
||||
mtp: bool,
|
||||
dtype: str,
|
||||
kv_cache_dtype: str,
|
||||
) -> None:
|
||||
"""Exercise prefill, decode, and batch consistency with dummy HY weights."""
|
||||
with vllm_runner(
|
||||
"tencent/Hy4-preview",
|
||||
trust_remote_code=True,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
disable_log_stats=not mtp,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
hf_overrides=_small_hyv4_config,
|
||||
max_model_len=256,
|
||||
max_num_batched_tokens=128,
|
||||
max_num_seqs=8,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
kv_cache_memory_bytes=128 * 1024 * 1024,
|
||||
enforce_eager=enforce_eager,
|
||||
tensor_parallel_size=tp,
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 2}
|
||||
if mtp
|
||||
else None,
|
||||
seed=17,
|
||||
# Exercise supported decode graphs independently of default graph policy.
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.NONE,
|
||||
cudagraph_mode=CUDAGraphMode.NONE
|
||||
if enforce_eager
|
||||
else CUDAGraphMode.FULL_DECODE_ONLY,
|
||||
cudagraph_capture_sizes=[] if enforce_eager else [1, 2, 4],
|
||||
),
|
||||
) as model:
|
||||
if not enforce_eager:
|
||||
assert model.collective_rpc(_num_captured_graphs) == [3] * tp
|
||||
prompts = [
|
||||
{"prompt_token_ids": [11, 12, 13, 14]},
|
||||
{"prompt_token_ids": list(range(20, 84))},
|
||||
]
|
||||
params = SamplingParams(
|
||||
temperature=0, max_tokens=4, ignore_eos=True, logprobs=1
|
||||
)
|
||||
outputs = model.llm.generate(prompts, params)
|
||||
assert len(outputs) == len(prompts)
|
||||
for prompt, output in zip(prompts, outputs):
|
||||
completion = output.outputs[0]
|
||||
assert len(completion.token_ids) == 4
|
||||
assert completion.cumulative_logprob is not None
|
||||
assert math.isfinite(completion.cumulative_logprob)
|
||||
single = model.llm.generate([prompt], params)[0].outputs[0]
|
||||
assert single.token_ids == completion.token_ids
|
||||
|
||||
if mtp:
|
||||
assert any(
|
||||
isinstance(metric, Counter)
|
||||
and metric.name == "vllm:spec_decode_num_drafts"
|
||||
and metric.value > 0
|
||||
for metric in model.llm.get_metrics()
|
||||
), "MTP drafter did not run"
|
||||
|
||||
|
||||
def test_hyv4_backbone_reaches_the_compiler(vllm_runner, enable_pickle) -> None:
|
||||
"""Guard against the backbone losing `@support_torch_compile`.
|
||||
|
||||
An undecorated backbone is not a hard failure: vLLM downgrades
|
||||
`cudagraph_mode` to NONE and serves the whole model eager, which on TP8
|
||||
gfx942 cost ~4x decode latency.
|
||||
"""
|
||||
with vllm_runner(
|
||||
"tencent/Hy4-preview",
|
||||
trust_remote_code=True,
|
||||
skip_tokenizer_init=True,
|
||||
load_format="dummy",
|
||||
dtype="bfloat16",
|
||||
hf_overrides=_small_hyv4_config,
|
||||
max_model_len=256,
|
||||
max_num_batched_tokens=128,
|
||||
max_num_seqs=8,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=False,
|
||||
kv_cache_memory_bytes=128 * 1024 * 1024,
|
||||
enforce_eager=False,
|
||||
seed=17,
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE,
|
||||
cudagraph_capture_sizes=[1, 2, 4],
|
||||
),
|
||||
) as model:
|
||||
for models_seen, captured, mode in model.collective_rpc(
|
||||
_piecewise_compile_state
|
||||
):
|
||||
assert models_seen > 0, "backbone was never handed to the compiler"
|
||||
assert mode == CUDAGraphMode.PIECEWISE.name
|
||||
assert captured > 0
|
||||
@@ -1,264 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import gc
|
||||
from collections.abc import Generator
|
||||
from functools import partial
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_configs
|
||||
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
|
||||
from vllm.v1.engine.core import EngineCore as V1EngineCore
|
||||
|
||||
from ..utils import (
|
||||
create_new_process_for_each_test,
|
||||
requires_spawn_multiprocessing,
|
||||
wait_for_gpu_memory_to_clear,
|
||||
wait_for_memory_to_settle,
|
||||
)
|
||||
from .registry import (
|
||||
_TRANSFORMERS_BACKEND_MODELS,
|
||||
AUTO_EXAMPLE_MODELS,
|
||||
HF_EXAMPLE_MODELS,
|
||||
HfExampleModels,
|
||||
)
|
||||
from .utils import dummy_hf_overrides
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# This minimal list of model architectures is smaller than the total list of
|
||||
# supported models. The intention is that in the "typical" regression testing
|
||||
# scenario, we only test initializing these models. This subset was chosen
|
||||
# to include representative examples of model varieties/workloads (conditional
|
||||
# generation, sequence classification, causal LM, ranking, chat, reward model,
|
||||
# multimodal, geospatial, voice, embedding, MTP)
|
||||
MINIMAL_MODEL_ARCH_LIST = [
|
||||
"LlavaForConditionalGeneration",
|
||||
"Llama4ForConditionalGeneration",
|
||||
"BertForSequenceClassification",
|
||||
"Gemma3nForCausalLM",
|
||||
"JinaVLForRanking",
|
||||
"InternVLChatModel",
|
||||
"InternLM2ForRewardModel",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
"Terratorch",
|
||||
"UltravoxModel",
|
||||
"DeepSeekMTPModel",
|
||||
"XLMRobertaModel",
|
||||
]
|
||||
|
||||
# This list is the complement of the minimal list above. The intention is that
|
||||
# this list of models is only tested in a "special case" i.e. most PRs should
|
||||
# not test these models
|
||||
OTHER_MODEL_ARCH_LIST = set(HF_EXAMPLE_MODELS.get_supported_archs()) - set(
|
||||
MINIMAL_MODEL_ARCH_LIST
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_gpu_memory_between_tests() -> Generator[None, None, None]:
|
||||
"""Reclaim GPU memory left over by each test's forked subprocess.
|
||||
|
||||
These tests are known to leave behind GPU memory (see the comment at
|
||||
``gpu_memory_utilization=0.80`` below). Each parametrized case builds an
|
||||
LLM in a forked subprocess whose VRAM is only reclaimed asynchronously
|
||||
when the subprocess exits, so the leftovers can OOM a later test in this
|
||||
module (observed with the heavier multimodal archs, e.g.
|
||||
InternVLChatModel). Mirrors ``clean_gpu_memory_between_tests`` in the
|
||||
top-level conftest, but unconditionally for this module.
|
||||
"""
|
||||
yield
|
||||
gc.collect()
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
torch.accelerator.empty_cache()
|
||||
if current_platform.is_rocm() or current_platform.is_xpu():
|
||||
wait_for_memory_to_settle()
|
||||
return
|
||||
try:
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(torch.accelerator.device_count())),
|
||||
threshold_ratio=0.1,
|
||||
)
|
||||
except Exception as e:
|
||||
# NVML is not usable on MIG slices (raises NVMLError_NoPermission);
|
||||
# cleanup is best-effort, so never fail a test over it.
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def can_initialize(
|
||||
model_arch: str, monkeypatch: pytest.MonkeyPatch, EXAMPLE_MODELS: HfExampleModels
|
||||
):
|
||||
"""The reason for using create_new_process_for_each_test is to avoid
|
||||
the WARNING:
|
||||
"We must use the 'spawn' multiprocessing start method. Overriding
|
||||
VLLM_WORKER_MULTIPROC_METHOD to 'spawn'."
|
||||
The spawn process causes the _initialize_kv_caches_v1 function below to
|
||||
become ineffective.
|
||||
"""
|
||||
model_info = EXAMPLE_MODELS.get_hf_info(model_arch)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
hf_overrides_fn = partial(
|
||||
dummy_hf_overrides,
|
||||
model_arch=model_arch,
|
||||
exist_overrides=model_info.hf_overrides,
|
||||
use_original_num_layers=getattr(model_info, "use_original_num_layers", False),
|
||||
)
|
||||
|
||||
# Avoid calling model.forward()
|
||||
def _initialize_kv_caches_v1(self, vllm_config):
|
||||
kv_cache_specs = self.model_executor.get_kv_cache_specs()
|
||||
layout = resolve_kv_cache_layout(
|
||||
vllm_config,
|
||||
self.model_executor.get_supported_kv_cache_layouts(),
|
||||
[spec for worker_specs in kv_cache_specs for spec in worker_specs.values()],
|
||||
)
|
||||
self.model_executor.set_kv_cache_layout(layout.name)
|
||||
kv_cache_configs = get_kv_cache_configs(
|
||||
vllm_config,
|
||||
kv_cache_specs,
|
||||
[10 * GiB_bytes],
|
||||
)
|
||||
scheduler_kv_cache_config = generate_scheduler_kv_cache_config(kv_cache_configs)
|
||||
vllm_config.cache_config.num_gpu_blocks = scheduler_kv_cache_config.num_blocks
|
||||
kv_cache_groups = scheduler_kv_cache_config.kv_cache_groups
|
||||
if kv_cache_groups:
|
||||
vllm_config.cache_config.block_size = min(
|
||||
g.kv_cache_spec.block_size for g in kv_cache_groups
|
||||
)
|
||||
|
||||
vllm_config.validate_block_size()
|
||||
return scheduler_kv_cache_config
|
||||
|
||||
if model_arch == "MoonshotKimiaForCausalLM":
|
||||
pytest.skip(
|
||||
"Kimi-Audio requires SpeechToTextConfig "
|
||||
"which is not configured in test environment"
|
||||
)
|
||||
|
||||
if model_arch == "Terratorch":
|
||||
import importlib.util
|
||||
|
||||
if importlib.util.find_spec("terratorch") is None:
|
||||
pytest.skip(
|
||||
"terratorch is not installed; "
|
||||
"temporarily skipped while PyPI has `lightning` quarantined "
|
||||
"(see #41376)"
|
||||
)
|
||||
|
||||
if model_arch in ["DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"]:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability and capability.major < 9:
|
||||
pytest.skip(
|
||||
f"DeepseekV32 requires Hopper (9.0+) or Blackwell (10.0+) "
|
||||
f"for FLASHMLA_SPARSE backend. Current device has compute "
|
||||
f"capability {capability.major}.{capability.minor}"
|
||||
)
|
||||
|
||||
if model_arch == "DeepseekV4ForConditionalGeneration":
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not (current_platform.is_cuda() or current_platform.is_rocm()):
|
||||
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
|
||||
|
||||
with (
|
||||
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
|
||||
monkeypatch.context() as m,
|
||||
):
|
||||
if requires_spawn_multiprocessing() or model_arch == "Glm5NextForCausalLM":
|
||||
# The EngineCore subprocess re-imports the class and does not
|
||||
# inherit the KV-cache patch above, so it OOMs. Run in-process
|
||||
# so the patch applies.
|
||||
m.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
# FIXME: A hack to bypass FA3 assertion because our CI's L4 GPU
|
||||
# has cc==8.9 which hasn't supported FA3 yet. Remove this hack when
|
||||
# L4 supports FA3.
|
||||
# Step1ForCausalLM requires TRITON_ATTN for use_alibi_sqrt support.
|
||||
attention_config = (
|
||||
{"backend": "TRITON_ATTN"}
|
||||
if model_arch in ("GptOssForCausalLM", "Step1ForCausalLM")
|
||||
else None
|
||||
)
|
||||
if model_arch == "WhisperForConditionalGeneration":
|
||||
m.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
kwargs = {}
|
||||
if not model_info.enable_prefix_caching:
|
||||
kwargs["enable_prefix_caching"] = False
|
||||
|
||||
LLM(
|
||||
model_info.default,
|
||||
tokenizer=model_info.tokenizer,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
dtype=model_info.dtype,
|
||||
speculative_config={
|
||||
"model": model_info.speculative_model,
|
||||
"method": model_info.speculative_method,
|
||||
"num_speculative_tokens": 1,
|
||||
}
|
||||
if model_info.speculative_model
|
||||
else None,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
max_model_len=model_info.max_model_len,
|
||||
max_num_batched_tokens=model_info.max_num_batched_tokens,
|
||||
# these tests seem to produce leftover memory
|
||||
gpu_memory_utilization=0.80,
|
||||
load_format="dummy",
|
||||
model_impl="transformers"
|
||||
if model_arch in _TRANSFORMERS_BACKEND_MODELS
|
||||
else "vllm",
|
||||
hf_overrides=hf_overrides_fn,
|
||||
max_num_seqs=model_info.max_num_seqs,
|
||||
attention_config=attention_config,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_arch", MINIMAL_MODEL_ARCH_LIST)
|
||||
def test_can_initialize_small_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test initializing small subset of supported models."""
|
||||
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_arch", OTHER_MODEL_ARCH_LIST)
|
||||
def test_can_initialize_large_subset(model_arch: str, monkeypatch: pytest.MonkeyPatch):
|
||||
"""Test initializing large subset of supported models.
|
||||
|
||||
This test covers the complement of the tests covered in the "small subset"
|
||||
test.
|
||||
"""
|
||||
if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel"):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("HY V4 ROCm initialization requires #54405")
|
||||
|
||||
can_initialize(model_arch, monkeypatch, HF_EXAMPLE_MODELS)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_arch", AUTO_EXAMPLE_MODELS.get_supported_archs())
|
||||
def test_implicit_converted_models(model_arch: str, monkeypatch: pytest.MonkeyPatch):
|
||||
can_initialize(model_arch, monkeypatch, AUTO_EXAMPLE_MODELS)
|
||||
@@ -1,237 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
import torch.cuda
|
||||
|
||||
from vllm.model_executor.models import (
|
||||
is_pooling_model,
|
||||
is_text_generation_model,
|
||||
supports_multimodal,
|
||||
)
|
||||
from vllm.model_executor.models.adapters import (
|
||||
as_embedding_model,
|
||||
as_seq_cls_model,
|
||||
)
|
||||
from vllm.model_executor.models.registry import (
|
||||
_MULTIMODAL_MODELS,
|
||||
_SPECULATIVE_DECODING_MODELS,
|
||||
_TEXT_GENERATION_MODELS,
|
||||
ModelRegistry,
|
||||
_LazyRegisteredModel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ..utils import create_new_process_for_each_test
|
||||
from .registry import HF_EXAMPLE_MODELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_arch", ModelRegistry.get_supported_archs())
|
||||
def test_registry_imports(model_arch):
|
||||
# Skip if transformers version is incompatible
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch)
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
if model_arch == "Terratorch":
|
||||
import importlib.util
|
||||
|
||||
if importlib.util.find_spec("terratorch") is None:
|
||||
pytest.skip(
|
||||
"terratorch is not installed; "
|
||||
"temporarily skipped while PyPI has `lightning` quarantined "
|
||||
"(see #41376)"
|
||||
)
|
||||
|
||||
# DSpark draft model is supported on CUDA and ROCm; stubbed to None on XPU.
|
||||
if model_arch == "DSparkDraftModel" and not (
|
||||
current_platform.is_cuda() or current_platform.is_rocm()
|
||||
):
|
||||
pytest.skip("DSparkDraftModel is only supported on CUDA and ROCm")
|
||||
|
||||
if model_arch in ("Dots3NoteForCausalLM", "Dots3NoteMTPModel") and not (
|
||||
current_platform.is_cuda()
|
||||
):
|
||||
pytest.skip("Dots3 NOTE is only supported on CUDA")
|
||||
|
||||
if model_arch in ("HYV4ForCausalLM", "HYV4MTPModel") and not (
|
||||
current_platform.is_cuda() or current_platform.is_rocm()
|
||||
):
|
||||
pytest.skip("HY V4 is only supported on CUDA and ROCm")
|
||||
|
||||
if (
|
||||
model_arch == "DeepseekV4ForConditionalGeneration"
|
||||
and not current_platform.is_cuda_alike()
|
||||
):
|
||||
pytest.skip("Deepseek V4 vision is only supported on CUDA and ROCm")
|
||||
|
||||
# Ensure all model classes can be imported successfully
|
||||
model_cls = ModelRegistry._try_load_model_cls(model_arch)
|
||||
assert model_cls is not None
|
||||
|
||||
if model_arch in _SPECULATIVE_DECODING_MODELS:
|
||||
return # Ignore these models which do not have a unified format
|
||||
|
||||
if model_arch in _TEXT_GENERATION_MODELS or model_arch in _MULTIMODAL_MODELS:
|
||||
assert is_text_generation_model(model_cls)
|
||||
|
||||
# All vLLM models should be convertible to a pooling model
|
||||
assert is_pooling_model(as_seq_cls_model(model_cls))
|
||||
assert is_pooling_model(as_embedding_model(model_cls))
|
||||
|
||||
if model_arch in _MULTIMODAL_MODELS:
|
||||
assert supports_multimodal(model_cls)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,is_mm,score_type",
|
||||
[
|
||||
("LlamaForCausalLM", False, "bi-encoder"),
|
||||
("LlavaForConditionalGeneration", True, "bi-encoder"),
|
||||
("DeepseekV41ForCausalLM", True, "bi-encoder"),
|
||||
("BertForSequenceClassification", False, "cross-encoder"),
|
||||
("RobertaForSequenceClassification", False, "cross-encoder"),
|
||||
("XLMRobertaForSequenceClassification", False, "cross-encoder"),
|
||||
("GteNewModel", False, "bi-encoder"),
|
||||
("GteNewForSequenceClassification", False, "cross-encoder"),
|
||||
("HF_ColBERT", False, "late-interaction"),
|
||||
],
|
||||
)
|
||||
def test_registry_model_property(model_arch, is_mm, score_type):
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
|
||||
assert model_info.supports_multimodal is is_mm
|
||||
assert model_info.score_type == score_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,is_pp",
|
||||
[
|
||||
("DeepseekV2ForCausalLM", True),
|
||||
("Qwen2VLForConditionalGeneration", True),
|
||||
],
|
||||
)
|
||||
def test_registry_is_pp(model_arch, is_pp):
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
|
||||
assert model_info.supports_pp is is_pp
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
("Qwen4ExpForCausalLM", True),
|
||||
("Qwen4ExpForConditionalGeneration", True),
|
||||
("Qwen4ExpMTP", True),
|
||||
("LlamaForCausalLM", False),
|
||||
],
|
||||
)
|
||||
def test_registry_supports_hc_sp(model_arch, supported):
|
||||
"""HC SP is opt-in for each model implementation, including MTP."""
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
assert model_info.supports_hc_sp is supported
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch",
|
||||
["LlavaForConditionalGeneration", "Qwen2VLForConditionalGeneration"],
|
||||
)
|
||||
def test_registry_inspect_does_not_init_cuda(model_arch):
|
||||
"""Inspecting a model must not import it: these architectures initialize
|
||||
CUDA at import time, so the inspection path has to stay out-of-process."""
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Requires a CUDA-like platform")
|
||||
|
||||
assert ModelRegistry._try_inspect_model_cls(model_arch) is not None
|
||||
assert not torch.cuda.is_initialized()
|
||||
|
||||
ModelRegistry._try_load_model_cls(model_arch)
|
||||
if not torch.cuda.is_initialized():
|
||||
warnings.warn(
|
||||
"This model no longer initializes CUDA on import. "
|
||||
"Please test using a different one.",
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_arch,supported",
|
||||
[
|
||||
# ReplaySSM is opt-in per model.
|
||||
("NemotronHForCausalLM", True),
|
||||
("KimiLinearForCausalLM", not current_platform.is_rocm()),
|
||||
("KimiK3ForConditionalGeneration", not current_platform.is_rocm()),
|
||||
("Mamba2ForCausalLM", False),
|
||||
("Zamba2ForCausalLM", False),
|
||||
],
|
||||
)
|
||||
def test_registry_supports_replayssm(model_arch, supported):
|
||||
model_info = ModelRegistry._try_inspect_model_cls(model_arch)
|
||||
assert model_info is not None
|
||||
assert model_info.supports_replayssm is supported
|
||||
|
||||
|
||||
def test_lazy_modelinfo_package_hash_includes_submodules(tmp_path):
|
||||
package_dir = tmp_path / "model_package"
|
||||
package_dir.mkdir()
|
||||
init_file = package_dir / "__init__.py"
|
||||
init_file.write_text("from .model import Model\n", encoding="utf-8")
|
||||
model_file = package_dir / "model.py"
|
||||
model_file.write_text("class Model: pass\n", encoding="utf-8")
|
||||
|
||||
first_hash = _LazyRegisteredModel._get_modelinfo_module_hash(init_file)
|
||||
|
||||
model_file.write_text("class Model:\n supports_pp = True\n", encoding="utf-8")
|
||||
second_hash = _LazyRegisteredModel._get_modelinfo_module_hash(init_file)
|
||||
|
||||
assert first_hash != second_hash
|
||||
|
||||
|
||||
def test_lazy_modelinfo_package_attempts_cache_load(monkeypatch):
|
||||
cached_model_info = object()
|
||||
loaded_hashes = []
|
||||
|
||||
def fake_load_cache(self, module_hash):
|
||||
loaded_hashes.append(module_hash)
|
||||
return cached_model_info
|
||||
|
||||
monkeypatch.setattr(
|
||||
_LazyRegisteredModel,
|
||||
"_load_modelinfo_from_cache",
|
||||
fake_load_cache,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.registry._run_in_subprocess",
|
||||
lambda _: pytest.fail("Package-backed model should use the cache path"),
|
||||
)
|
||||
|
||||
registered_model = _LazyRegisteredModel(
|
||||
module_name="vllm.model_executor.models.transformers",
|
||||
class_name="TransformersForCausalLM",
|
||||
)
|
||||
|
||||
result = registered_model.inspect_model_cls()
|
||||
|
||||
assert result is cached_model_info
|
||||
assert len(loaded_hashes) == 1
|
||||
assert loaded_hashes[0]
|
||||
|
||||
|
||||
def test_hf_registry_coverage():
|
||||
untested_archs = (
|
||||
ModelRegistry.get_supported_archs() - HF_EXAMPLE_MODELS.get_supported_archs()
|
||||
)
|
||||
|
||||
assert not untested_archs, (
|
||||
"Please add the following architectures to "
|
||||
f"`tests/models/registry.py`: {untested_archs}"
|
||||
)
|
||||
@@ -1,241 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
_merge_multimodal_embeddings,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
class ModuleWithBatchNorm(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.bn = torch.nn.BatchNorm1d(2)
|
||||
|
||||
def forward(self, x):
|
||||
return self.bn(x)
|
||||
|
||||
|
||||
class ModuleWithNestedBatchNorm(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.nested_mod = ModuleWithBatchNorm()
|
||||
|
||||
def forward(self, x):
|
||||
return self.nested_mod(x)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_module_with_batchnorm_can_load():
|
||||
"""Ensure the auto weight loader can load batchnorm stats."""
|
||||
mod = ModuleWithBatchNorm()
|
||||
# Run some data through the module with batchnorm
|
||||
mod(torch.Tensor([[1, 2], [3, 4]]))
|
||||
|
||||
# Try to load the weights to a new instance
|
||||
def weight_generator():
|
||||
yield from mod.state_dict().items()
|
||||
|
||||
new_mod = ModuleWithBatchNorm()
|
||||
|
||||
assert not torch.all(new_mod.bn.running_mean == mod.bn.running_mean)
|
||||
assert not torch.all(new_mod.bn.running_var == mod.bn.running_var)
|
||||
assert new_mod.bn.num_batches_tracked.item() == 0
|
||||
|
||||
loader = AutoWeightsLoader(new_mod)
|
||||
loader.load_weights(weight_generator())
|
||||
|
||||
# Ensure the stats are updated
|
||||
assert torch.all(new_mod.bn.running_mean == mod.bn.running_mean)
|
||||
assert torch.all(new_mod.bn.running_var == mod.bn.running_var)
|
||||
assert new_mod.bn.num_batches_tracked.item() == 1
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_module_with_child_containing_batchnorm_can_autoload():
|
||||
"""Ensure the auto weight loader can load nested modules batchnorm stats."""
|
||||
mod = ModuleWithNestedBatchNorm()
|
||||
# Run some data through the module with batchnorm
|
||||
mod(torch.Tensor([[1, 2], [3, 4]]))
|
||||
|
||||
# Try to load the weights to a new instance
|
||||
def weight_generator():
|
||||
yield from mod.state_dict().items()
|
||||
|
||||
new_mod = ModuleWithNestedBatchNorm()
|
||||
|
||||
assert not torch.all(
|
||||
new_mod.nested_mod.bn.running_mean == mod.nested_mod.bn.running_mean
|
||||
)
|
||||
assert not torch.all(
|
||||
new_mod.nested_mod.bn.running_var == mod.nested_mod.bn.running_var
|
||||
)
|
||||
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 0
|
||||
|
||||
loader = AutoWeightsLoader(new_mod)
|
||||
loader.load_weights(weight_generator())
|
||||
|
||||
# Ensure the stats are updated
|
||||
assert torch.all(
|
||||
new_mod.nested_mod.bn.running_mean == mod.nested_mod.bn.running_mean
|
||||
)
|
||||
assert torch.all(new_mod.nested_mod.bn.running_var == mod.nested_mod.bn.running_var)
|
||||
assert new_mod.nested_mod.bn.num_batches_tracked.item() == 1
|
||||
|
||||
|
||||
VOCAB_SIZE = 16
|
||||
HIDDEN_SIZE = 2
|
||||
|
||||
|
||||
class ModuleWithTiedWeights(torch.nn.Module):
|
||||
"""Mimics how models tie `lm_head` to the input embeddings."""
|
||||
|
||||
def __init__(self, tie: bool):
|
||||
super().__init__()
|
||||
self.model = torch.nn.Module()
|
||||
self.model.embed_tokens = VocabParallelEmbedding(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
self.lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
if tie:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
|
||||
|
||||
def make_embedding_weights(value: float) -> torch.Tensor:
|
||||
return torch.full((VOCAB_SIZE, HIDDEN_SIZE), value)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
@pytest.mark.parametrize("tie", [True, False])
|
||||
def test_module_skip_tied_weights(tie: bool):
|
||||
"""Tied weights must be loaded once, under the first of their names."""
|
||||
mod = ModuleWithTiedWeights(tie)
|
||||
|
||||
weights = [
|
||||
("model.embed_tokens.weight", make_embedding_weights(1.0)),
|
||||
("lm_head.weight", make_embedding_weights(2.0)),
|
||||
]
|
||||
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
if tie:
|
||||
assert loaded == {"model.embed_tokens.weight"}
|
||||
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 1.0)
|
||||
else:
|
||||
assert loaded == {"model.embed_tokens.weight", "lm_head.weight"}
|
||||
assert torch.all(mod.lm_head.weight[:VOCAB_SIZE] == 2.0)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.usefixtures("dist_init")
|
||||
def test_module_skip_tied_weights_without_canonical():
|
||||
"""Skipping a tied weight must not leave the shared weight uninitialized."""
|
||||
mod = ModuleWithTiedWeights(tie=True)
|
||||
|
||||
weights = [("lm_head.weight", make_embedding_weights(2.0))]
|
||||
with pytest.raises(ValueError, match="model.embed_tokens.weight"):
|
||||
AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
|
||||
class ModuleWithSharedParam(torch.nn.Module):
|
||||
"""Mimics an MoE router shared between the MLP and its fused experts."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.experts = torch.nn.Module()
|
||||
self.experts.gate = torch.nn.Linear(2, 2, bias=False)
|
||||
self.gate = self.experts.gate
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_module_load_shared_params_that_are_not_tied_embeddings():
|
||||
"""Only tied embeddings are skipped; other shared params must still load."""
|
||||
mod = ModuleWithSharedParam()
|
||||
|
||||
weights = [("gate.weight", torch.Tensor([[1, 2], [3, 4]]))]
|
||||
loaded = AutoWeightsLoader(mod).load_weights(iter(weights))
|
||||
|
||||
assert loaded == {"gate.weight"}
|
||||
assert torch.all(mod.gate.weight == torch.Tensor([[1, 2], [3, 4]]))
|
||||
|
||||
|
||||
class raise_if_cuda_sync:
|
||||
def __enter__(self):
|
||||
self.previous_debug_mode = torch.cuda.get_sync_debug_mode()
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
|
||||
def __exit__(self, exception_type, exception_value, traceback):
|
||||
torch.cuda.set_sync_debug_mode(self.previous_debug_mode)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
def test_merge_multimodal_embeddings_no_sync():
|
||||
inputs_embeds = torch.zeros(
|
||||
[5, 10], dtype=torch.bfloat16, device=f"{DEVICE_TYPE}:0"
|
||||
)
|
||||
multimodal_embeddings = [
|
||||
torch.ones([3, 10], dtype=torch.bfloat16, device=f"{DEVICE_TYPE}:0")
|
||||
]
|
||||
is_multimodal = torch.tensor([True, False, True, True, False], device="cpu")
|
||||
with raise_if_cuda_sync():
|
||||
_merge_multimodal_embeddings(
|
||||
inputs_embeds, multimodal_embeddings, is_multimodal
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_get_rename_mapper_keeps_only_renames():
|
||||
"""`None` means "do not load", which is meaningless to the consumers of
|
||||
this mapper (LoRA name parsing, quantization config layer lists), and
|
||||
applying it would silently shrink their lists."""
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_regex={re.compile(r"^drop_regex\."): None},
|
||||
orig_to_new_substr={"drop_substr": None, "keep_substr": "kept"},
|
||||
orig_to_new_stacked={".q_proj": (".qkv_proj", "q")},
|
||||
orig_to_new_prefix={"drop_prefix.": None, "keep_prefix.": "kept."},
|
||||
orig_to_new_suffix={".drop_suffix": None},
|
||||
)
|
||||
renames = mapper.get_rename_mapper()
|
||||
|
||||
assert renames.orig_to_new_regex == {}
|
||||
assert renames.orig_to_new_substr == {"keep_substr": "kept"}
|
||||
assert renames.orig_to_new_stacked == {}
|
||||
assert renames.orig_to_new_prefix == {"keep_prefix.": "kept."}
|
||||
assert renames.orig_to_new_suffix == {}
|
||||
|
||||
# Names the full mapper drops now survive unchanged.
|
||||
for name in ("drop_regex.w", "drop_substr.w", "drop_prefix.w", "w.drop_suffix"):
|
||||
assert mapper._map_name(name) is None
|
||||
assert renames._map_name(name) == name
|
||||
|
||||
|
||||
def test_weights_mapper_stacks_one_weight_into_several_shards():
|
||||
"""One checkpoint tensor can feed more than one shard of a stacked
|
||||
parameter, and a dropped name stays dropped rather than reappearing as a
|
||||
shard of the stacked one."""
|
||||
mapper = WeightsMapper(
|
||||
orig_to_new_substr={"layers.1.k_proj.": None},
|
||||
orig_to_new_stacked={".k_proj.": (".qkv_proj.", ["k", "v"])},
|
||||
)
|
||||
weight = torch.ones(2)
|
||||
weights = [(f"layers.{i}.k_proj.weight", weight) for i in (0, 1)]
|
||||
|
||||
mapped = list(mapper.apply(weights))
|
||||
|
||||
assert [(name, w.shard_id) for name, w in mapped] == [
|
||||
("layers.0.qkv_proj.weight", "k"),
|
||||
("layers.0.qkv_proj.weight", "v"),
|
||||
]
|
||||
# Each shard needs its own tensor object, but they alias one allocation.
|
||||
assert mapped[0][1] is not mapped[1][1]
|
||||
assert mapped[0][1].data_ptr() == mapped[1][1].data_ptr() == weight.data_ptr()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,786 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Test the functionality of the Transformers modeling backend."""
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import os
|
||||
import tempfile
|
||||
from functools import partial
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import AutoConfig, AutoModel, PreTrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.transformers.base import Base
|
||||
from vllm.model_executor.models.transformers.fusers import AttentionFuser
|
||||
from vllm.model_executor.models.transformers.fusers.attention import VLLM_ATTN_IMPL
|
||||
from vllm.model_executor.models.transformers.multimodal import MultiModalMixin
|
||||
from vllm.model_executor.models.utils import StageMissingLayer
|
||||
|
||||
from ...conftest import HfRunner, VllmRunner
|
||||
from ...utils import multi_gpu_test, prep_prompts
|
||||
from ..registry import HF_EXAMPLE_MODELS
|
||||
from ..utils import check_embeddings_close, check_logprobs_close
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
def enable_pickle(monkeypatch):
|
||||
"""`LLM.apply_model` requires pickling a function."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
def get_model(arch: str) -> str:
|
||||
model_info = HF_EXAMPLE_MODELS.get_hf_info(arch)
|
||||
model_info.check_transformers_version(on_fail="skip")
|
||||
return model_info.default
|
||||
|
||||
|
||||
def count_modules(model, cls: type) -> int:
|
||||
return sum(isinstance(m, cls) for m in model.modules())
|
||||
|
||||
|
||||
def get_num_fused(model) -> tuple[int, int]:
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
)
|
||||
|
||||
glu = count_modules(model, MergedColumnParallelLinear)
|
||||
qkv = count_modules(model, QKVParallelLinear)
|
||||
return glu, qkv
|
||||
|
||||
|
||||
def check_implementation(
|
||||
runner_ref: type[HfRunner | VllmRunner],
|
||||
runner_test: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
kwargs_ref: dict[str, Any] | None = None,
|
||||
kwargs_test: dict[str, Any] | None = None,
|
||||
num_fused: tuple[int, int] = (1, 1),
|
||||
**kwargs,
|
||||
):
|
||||
if kwargs_ref is None:
|
||||
kwargs_ref = {}
|
||||
if kwargs_test is None:
|
||||
kwargs_test = {}
|
||||
|
||||
max_tokens = 32
|
||||
num_logprobs = 5
|
||||
|
||||
args = (example_prompts, max_tokens, num_logprobs)
|
||||
|
||||
with runner_test(model, **kwargs_test, **kwargs) as model_test:
|
||||
model_config = model_test.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
|
||||
num_layers = model_config.hf_config.get_text_config().num_hidden_layers
|
||||
expected_glu, expected_qkv = num_fused
|
||||
for num_glu, num_qkv in model_test.apply_model(get_num_fused):
|
||||
assert num_glu == expected_glu * num_layers
|
||||
assert num_qkv == expected_qkv * num_layers
|
||||
|
||||
tp_size = kwargs_test.get("tensor_parallel_size", 1)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
|
||||
counts = model_test.apply_model(partial(count_modules, cls=Attention))
|
||||
assert counts == [num_layers] * tp_size
|
||||
|
||||
outputs_test = model_test.generate_greedy_logprobs(*args)
|
||||
|
||||
with runner_ref(model, **kwargs_ref) as model_ref:
|
||||
if isinstance(model_ref, VllmRunner):
|
||||
outputs_ref = model_ref.generate_greedy_logprobs(*args)
|
||||
else:
|
||||
outputs_ref = model_ref.generate_greedy_logprobs_limit(*args)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=outputs_ref,
|
||||
outputs_1_lst=outputs_test,
|
||||
name_0="ref",
|
||||
name_1="test",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,model_impl,num_fused",
|
||||
[
|
||||
("meta-llama/Llama-3.2-1B-Instruct", "transformers", (1, 1)),
|
||||
("hmellor/Ilama-3.2-1B", "auto", (1, 1)), # CUSTOM CODE
|
||||
("allenai/OLMoE-1B-7B-0924", "transformers", (0, 1)), # MoE
|
||||
],
|
||||
) # trust_remote_code=True by default
|
||||
def test_models(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
model_impl: str,
|
||||
num_fused: tuple[int, int],
|
||||
) -> None:
|
||||
check_implementation(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
model,
|
||||
num_fused=num_fused,
|
||||
model_impl=model_impl,
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_attention(vllm_runner: type[VllmRunner]) -> None:
|
||||
prompts, _, _ = prep_prompts(4, (800, 801))
|
||||
kwargs_ref = {"max_model_len": 8192, "enforce_eager": True}
|
||||
kwargs_test = {"model_impl": "transformers", **kwargs_ref}
|
||||
check_implementation(
|
||||
vllm_runner,
|
||||
vllm_runner,
|
||||
prompts,
|
||||
model="hmellor/tiny-random-Gemma2ForCausalLM",
|
||||
kwargs_ref=kwargs_ref,
|
||||
kwargs_test=kwargs_test,
|
||||
)
|
||||
|
||||
|
||||
def get_sinks(model) -> dict[int, torch.Tensor]:
|
||||
"""The sink tensor each attention layer was handed, keyed by layer index."""
|
||||
sinks = {}
|
||||
for i, (prefix, fuser) in model.attention_fusers.items():
|
||||
if (sink := fuser.sinks(model.get_submodule(prefix))) is not None:
|
||||
sinks[i] = sink.float().cpu()
|
||||
return sinks
|
||||
|
||||
|
||||
def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None:
|
||||
"""Learnable attention sinks must reach the attention layers.
|
||||
|
||||
Only the attention impl can apply them, so if they are not passed to
|
||||
`Attention` they are dropped and every softmax is subtly wrong.
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.has_device_capability(90):
|
||||
pytest.skip("Attention sinks need FlashAttention 3 or TRTLLM attention")
|
||||
|
||||
model = "tiny-random/gpt-oss-bf16"
|
||||
with vllm_runner(
|
||||
model, model_impl="transformers", max_model_len=1024, enforce_eager=True
|
||||
) as model_test:
|
||||
assert model_test.llm.llm_engine.model_config.using_transformers_backend()
|
||||
sinks = model_test.apply_model(get_sinks)[0]
|
||||
|
||||
with hf_runner(model, dtype="bfloat16") as model_ref:
|
||||
layers = model_ref.model.model.get_decoder().layers
|
||||
expected = {i: layer.self_attn.sinks.float() for i, layer in enumerate(layers)}
|
||||
|
||||
assert sinks.keys() == expected.keys()
|
||||
for i, sink in sinks.items():
|
||||
torch.testing.assert_close(sink, expected[i].cpu())
|
||||
|
||||
|
||||
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
|
||||
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
|
||||
args = (example_prompts, 32, 5)
|
||||
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
|
||||
|
||||
with vllm_runner(
|
||||
model, model_impl="transformers", trust_remote_code=False, **kwargs
|
||||
) as model_test:
|
||||
model_config = model_test.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
num_layers = model_config.hf_config.get_text_config().num_hidden_layers
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
|
||||
counts = model_test.apply_model(partial(count_modules, cls=MLAAttention))
|
||||
assert counts == [num_layers]
|
||||
outputs_test = model_test.generate_greedy_logprobs(*args)
|
||||
|
||||
with vllm_runner(model, model_impl="auto") as model_ref:
|
||||
outputs_ref = model_ref.generate_greedy_logprobs(*args)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=outputs_ref,
|
||||
outputs_1_lst=outputs_test,
|
||||
name_0="native",
|
||||
name_1="transformers",
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_distributed(
|
||||
hf_runner: type[HfRunner],
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts,
|
||||
):
|
||||
kwargs = {"model_impl": "transformers", "tensor_parallel_size": 2}
|
||||
check_implementation(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
kwargs_test=kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model, quantization_kwargs",
|
||||
[
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {}),
|
||||
("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", {}),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("max_tokens", [32])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_quantization(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
quantization_kwargs: dict[str, str],
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
with vllm_runner(
|
||||
model,
|
||||
model_impl="auto",
|
||||
enforce_eager=True,
|
||||
**quantization_kwargs,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
model_impl="transformers",
|
||||
enforce_eager=True,
|
||||
**quantization_kwargs,
|
||||
) as vllm_model:
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
|
||||
transformers_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, max_tokens=max_tokens, num_logprobs=num_logprobs
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=transformers_outputs,
|
||||
outputs_1_lst=vllm_outputs,
|
||||
name_0="transformers",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
# Layers live in `layers`
|
||||
"Qwen/Qwen3-Embedding-0.6B",
|
||||
# Layers live in `model.layers`
|
||||
"meta-llama/Llama-3.2-1B-Instruct",
|
||||
],
|
||||
)
|
||||
def test_embed_loading(vllm_runner, model):
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
runner="pooling",
|
||||
model_impl="transformers",
|
||||
) as model_test:
|
||||
model_config = model_test.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"arch", ["TransformersEmbeddingModel", "TransformersForSequenceClassification"]
|
||||
)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_pooling(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
example_prompts,
|
||||
arch,
|
||||
monkeypatch,
|
||||
use_v2_model_runner,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
model = get_model(arch)
|
||||
|
||||
vllm_kwargs = dict(max_model_len=None, model_impl="transformers")
|
||||
|
||||
hf_kwargs = dict()
|
||||
if arch == "TransformersEmbeddingModel":
|
||||
hf_kwargs["is_sentence_transformer"] = True
|
||||
elif arch == "TransformersForSequenceClassification":
|
||||
from transformers import AutoModelForSequenceClassification
|
||||
|
||||
hf_kwargs["auto_cls"] = AutoModelForSequenceClassification
|
||||
|
||||
# The example_prompts has ending "\n", for example:
|
||||
# "Write a short story about a robot that dreams for the first time.\n"
|
||||
# sentence_transformers will strip the input texts, see:
|
||||
# https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
|
||||
# This makes the input_ids different between hf_model and vllm_model.
|
||||
# So we need to strip the input texts to avoid test failing.
|
||||
example_prompts = [str(s).strip() for s in example_prompts]
|
||||
|
||||
with (
|
||||
vllm_runner(model, **vllm_kwargs) as vllm_model,
|
||||
hf_runner(model, **hf_kwargs) as hf_model,
|
||||
):
|
||||
model_config = vllm_model.llm.llm_engine.model_config
|
||||
assert model_config.using_transformers_backend()
|
||||
|
||||
if arch == "TransformersEmbeddingModel":
|
||||
vllm_outputs = vllm_model.embed(example_prompts)
|
||||
hf_outputs = hf_model.encode(example_prompts)
|
||||
elif arch == "TransformersForSequenceClassification":
|
||||
vllm_outputs = vllm_model.classify(example_prompts)
|
||||
hf_outputs = hf_model.classify(example_prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
)
|
||||
|
||||
|
||||
VOCAB_SIZE = 64
|
||||
PER_LAYER_VOCAB_SIZE = 32
|
||||
NUM_POSITIONS = 16
|
||||
HIDDEN_SIZE = 8
|
||||
EMBED_SCALE = 3.0
|
||||
|
||||
|
||||
class ScaledWordEmbedding(nn.Embedding):
|
||||
"""Mirrors Transformers' `*ScaledWordEmbedding` classes."""
|
||||
|
||||
def __init__(
|
||||
self, num_embeddings, embedding_dim, padding_idx=None, embed_scale=1.0
|
||||
):
|
||||
super().__init__(num_embeddings, embedding_dim, padding_idx)
|
||||
self.scalar_embed_scale = embed_scale
|
||||
self.register_buffer("embed_scale", torch.tensor(embed_scale), persistent=False)
|
||||
|
||||
def forward(self, input_ids):
|
||||
return super().forward(input_ids) * self.embed_scale.to(self.weight.dtype)
|
||||
|
||||
|
||||
class ComposedWordEmbedding(nn.Module):
|
||||
"""Scales embeddings, but wraps `nn.Embedding` instead of inheriting from it."""
|
||||
|
||||
def __init__(self, num_embeddings, embedding_dim, embed_scale=1.0):
|
||||
super().__init__()
|
||||
self.embed = nn.Embedding(num_embeddings, embedding_dim)
|
||||
self.embed_scale = embed_scale
|
||||
|
||||
def forward(self, input_ids):
|
||||
return self.embed(input_ids) * self.embed_scale
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tp_init():
|
||||
"""Single rank tensor parallel state, so vLLM layers can be constructed."""
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import ensure_current_vllm_config
|
||||
|
||||
fd, temp_file = tempfile.mkstemp()
|
||||
os.close(fd)
|
||||
try:
|
||||
with ensure_current_vllm_config():
|
||||
init_distributed_environment(
|
||||
world_size=1,
|
||||
rank=0,
|
||||
distributed_init_method=f"file://{temp_file}",
|
||||
local_rank=0,
|
||||
backend=current_platform.dist_backend,
|
||||
)
|
||||
initialize_model_parallel(1, 1)
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
os.unlink(temp_file)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vpe(tp_init):
|
||||
"""`VocabParallelEmbedding`, imported late so collection does not import vLLM."""
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
|
||||
return VocabParallelEmbedding
|
||||
|
||||
|
||||
def replace(embedding):
|
||||
"""Replace `embedding` and fill the new weights with recognisable values."""
|
||||
from vllm.model_executor.models.transformers.utils import replace_embedding_class
|
||||
|
||||
new_embedding = replace_embedding_class(embedding)
|
||||
for _, param in new_embedding.named_parameters():
|
||||
param.data = torch.arange(param.numel(), dtype=param.dtype).view(param.shape)
|
||||
return new_embedding
|
||||
|
||||
|
||||
def assert_scaled(vpe, module):
|
||||
"""`module`'s output is its own unscaled output times `EMBED_SCALE`."""
|
||||
input_ids = torch.arange(VOCAB_SIZE)
|
||||
unscaled = vpe.forward(module, input_ids)
|
||||
torch.testing.assert_close(module(input_ids), unscaled * EMBED_SCALE)
|
||||
|
||||
|
||||
def test_replace_plain_embedding(vpe):
|
||||
"""A plain `nn.Embedding` is replaced outright, leaving no subclass behind."""
|
||||
assert type(replace(nn.Embedding(VOCAB_SIZE, HIDDEN_SIZE))) is vpe
|
||||
|
||||
|
||||
def test_replace_infers_shape_and_dtype(tp_init):
|
||||
"""Shape and dtype come from the replaced module, not from the config."""
|
||||
embedding = nn.Embedding(VOCAB_SIZE * 2, HIDDEN_SIZE + 1, dtype=torch.float16)
|
||||
new_embedding = replace(embedding)
|
||||
|
||||
assert new_embedding.num_embeddings == VOCAB_SIZE * 2
|
||||
assert new_embedding.org_vocab_size == VOCAB_SIZE * 2
|
||||
assert new_embedding.embedding_dim == HIDDEN_SIZE + 1
|
||||
assert new_embedding.weight.dtype == torch.float16
|
||||
|
||||
|
||||
def test_replace_inherited_embedding(vpe):
|
||||
"""Subclasses keep their extra state and their scaled `forward`."""
|
||||
new_embedding = replace(
|
||||
ScaledWordEmbedding(
|
||||
VOCAB_SIZE, HIDDEN_SIZE, padding_idx=0, embed_scale=EMBED_SCALE
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(new_embedding, vpe)
|
||||
assert new_embedding.scalar_embed_scale == EMBED_SCALE
|
||||
assert "embed_scale" in new_embedding._non_persistent_buffers_set
|
||||
assert_scaled(vpe, new_embedding)
|
||||
|
||||
|
||||
def test_replace_is_idempotent(tp_init):
|
||||
"""A tied table is reached twice by the recursion, and must not be rebased twice."""
|
||||
from vllm.model_executor.models.transformers.utils import replace_embedding_class
|
||||
|
||||
new_embedding = replace(ScaledWordEmbedding(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=2))
|
||||
again = replace_embedding_class(new_embedding)
|
||||
|
||||
assert again is new_embedding
|
||||
assert type(again) is type(new_embedding)
|
||||
|
||||
|
||||
class FakeVocabModel(nn.Module):
|
||||
"""The embedding tables a model can hold, only one of which is a vocab table.
|
||||
|
||||
`embed_tokens_per_layer` mirrors Gemma 3n, whose per-layer table is sized by
|
||||
`vocab_size_per_layer_input` rather than `vocab_size`.
|
||||
"""
|
||||
|
||||
def __init__(self, per_layer: bool = False, composed: bool = False):
|
||||
super().__init__()
|
||||
embed_cls = ComposedWordEmbedding if composed else ScaledWordEmbedding
|
||||
self.embed_tokens = embed_cls(VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE)
|
||||
if per_layer:
|
||||
self.embed_tokens_per_layer = ScaledWordEmbedding(
|
||||
PER_LAYER_VOCAB_SIZE, HIDDEN_SIZE, embed_scale=EMBED_SCALE
|
||||
)
|
||||
# Not a vocab table: indexed by position, and read as a whole by models
|
||||
# which interpolate it (e.g. `SiglipVisionEmbeddings`)
|
||||
self.position_embedding = nn.Embedding(NUM_POSITIONS, HIDDEN_SIZE)
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.embed_tokens
|
||||
|
||||
|
||||
def replace_vocab_embeddings(model, **config_kwargs):
|
||||
"""Replace `model`'s vocab tables as `recursive_replace` would, and return them."""
|
||||
from vllm.model_executor.models.transformers.utils import attrsetter
|
||||
|
||||
stub = nn.Module()
|
||||
stub.model = model
|
||||
stub.config = PreTrainedConfig(
|
||||
vocab_size=VOCAB_SIZE, num_positions=NUM_POSITIONS, **config_kwargs
|
||||
)
|
||||
embeddings = Base._vocab_embeddings(stub)
|
||||
|
||||
replaced = []
|
||||
for name, module in list(model.named_modules()):
|
||||
if module in embeddings:
|
||||
replaced.append(replace(module))
|
||||
attrsetter(name)(model, replaced[-1])
|
||||
return replaced
|
||||
|
||||
|
||||
def test_only_vocab_tables_are_replaced(vpe):
|
||||
"""Position embeddings share the `nn.Embedding` type, but not the treatment."""
|
||||
model = FakeVocabModel()
|
||||
|
||||
assert replace_vocab_embeddings(model) == [model.embed_tokens]
|
||||
assert isinstance(model.embed_tokens, vpe)
|
||||
assert type(model.position_embedding) is nn.Embedding
|
||||
|
||||
|
||||
def test_per_layer_vocab_table_is_replaced(vpe):
|
||||
"""A second vocab table is found by its size, without a bespoke accessor."""
|
||||
model = FakeVocabModel(per_layer=True)
|
||||
replaced = replace_vocab_embeddings(
|
||||
model, vocab_size_per_layer_input=PER_LAYER_VOCAB_SIZE
|
||||
)
|
||||
|
||||
assert replaced == [model.embed_tokens, model.embed_tokens_per_layer]
|
||||
assert all(isinstance(module, vpe) for module in replaced)
|
||||
|
||||
|
||||
def test_composed_input_embeddings_are_replaced(vpe):
|
||||
"""A wrapper is left alone; only the `nn.Embedding` it composes is replaced.
|
||||
|
||||
`CausalMixin` ties `lm_head` to whichever `VocabParallelEmbedding` it finds under
|
||||
`get_input_embeddings()`, reading a `.weight` the wrapper does not have.
|
||||
"""
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
|
||||
model = FakeVocabModel(composed=True)
|
||||
wrapper = model.embed_tokens
|
||||
replaced = replace_vocab_embeddings(model)
|
||||
|
||||
assert model.embed_tokens is wrapper
|
||||
assert replaced == [m for m in wrapper.modules() if isinstance(m, vpe)]
|
||||
|
||||
lm_head = ParallelLMHead(VOCAB_SIZE, HIDDEN_SIZE)
|
||||
assert lm_head.tie_weights(replaced[0]).weight is replaced[0].weight
|
||||
|
||||
|
||||
def test_missing_input_embeddings_are_skipped(tp_init):
|
||||
"""Pipeline ranks without the embeddings have a `PPMissingLayer` in their place."""
|
||||
from vllm.model_executor.models.utils import PPMissingLayer
|
||||
|
||||
model = FakeVocabModel()
|
||||
model.embed_tokens = PPMissingLayer()
|
||||
|
||||
assert replace_vocab_embeddings(model) == []
|
||||
|
||||
|
||||
MULTIMODAL_MODEL = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
|
||||
|
||||
class MarkingStub(SupportsMultiModal, nn.Module):
|
||||
"""Just enough of the backend to exercise component marking."""
|
||||
|
||||
_mark_model_components = MultiModalMixin._mark_model_components
|
||||
_find_encoder_classes = MultiModalMixin._find_encoder_classes
|
||||
_from_config_kwargs = Base._from_config_kwargs
|
||||
_pre_trained_model_classes = Base._pre_trained_model_classes
|
||||
|
||||
|
||||
def build_marked_model(
|
||||
image_limit: int, video_limit: int = 0, skip_tokenizer_init: bool = False
|
||||
):
|
||||
"""Build the HF model inside the marking context and return it."""
|
||||
model_config = ModelConfig(
|
||||
model=MULTIMODAL_MODEL,
|
||||
model_impl="transformers",
|
||||
limit_mm_per_prompt={"image": image_limit, "video": video_limit},
|
||||
)
|
||||
# Set after construction: building the config itself needs the tokenizer
|
||||
model_config.skip_tokenizer_init = skip_tokenizer_init
|
||||
stub = MarkingStub()
|
||||
stub.config = AutoConfig.from_pretrained(MULTIMODAL_MODEL)
|
||||
stub.model_config = model_config
|
||||
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
with stub._mark_model_components(vllm_config), torch.device("meta"):
|
||||
stub.model = AutoModel.from_config(**stub._from_config_kwargs)
|
||||
return stub.model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_limit", "video_limit", "skipped"),
|
||||
[(0, 0, True), (4, 0, False), (0, 4, False)],
|
||||
)
|
||||
def test_tower_weights_skipped_when_modality_disabled(
|
||||
image_limit, video_limit, skipped
|
||||
):
|
||||
"""Only `--limit-mm-per-prompt image=0,video=0` drops the tower they share."""
|
||||
vision_tower = build_marked_model(image_limit, video_limit).vision_tower
|
||||
assert isinstance(vision_tower, StageMissingLayer) is skipped
|
||||
|
||||
|
||||
def test_marking_skipped_without_tokenizer():
|
||||
"""Marking needs the HF processor, which needs a tokenizer, so it is skipped.
|
||||
|
||||
The tower is built as normal rather than the model failing to load.
|
||||
"""
|
||||
vision_tower = build_marked_model(0, skip_tokenizer_init=True).vision_tower
|
||||
assert not isinstance(vision_tower, StageMissingLayer)
|
||||
|
||||
|
||||
NUM_LAYERS = 4
|
||||
|
||||
|
||||
def build_model(model_type: str, **overrides) -> nn.Module:
|
||||
"""A tiny HF model of `NUM_LAYERS` layers, built on the meta device."""
|
||||
try:
|
||||
config = AutoConfig.for_model(
|
||||
model_type,
|
||||
num_hidden_layers=NUM_LAYERS,
|
||||
hidden_size=64,
|
||||
intermediate_size=128,
|
||||
num_attention_heads=4,
|
||||
num_key_value_heads=2,
|
||||
head_dim=16,
|
||||
vocab_size=VOCAB_SIZE,
|
||||
**overrides,
|
||||
)
|
||||
except ValueError:
|
||||
pytest.skip(f"The installed transformers has no {model_type!r} model")
|
||||
with torch.device("meta"):
|
||||
return AutoModel.from_config(config)
|
||||
|
||||
|
||||
ATTENTION_MODEL_TYPES = [
|
||||
# The default `head_size**-0.5` happens to be right here
|
||||
"llama",
|
||||
# Scales the query by a learnable per-dim weight, so it declares 1.0
|
||||
"gemma4_text",
|
||||
# MLA, and declares yarn's `mscale`
|
||||
"deepseek_v3",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
|
||||
def test_attention_dispatch_is_matched(model_type: str):
|
||||
"""Exactly the decoder layers' attention modules match an `AttentionFuser`.
|
||||
|
||||
That match is what `recursive_replace` records, and so what
|
||||
`_create_attention_instances` attaches vLLM's attention layer to.
|
||||
"""
|
||||
model = build_model(model_type)
|
||||
matched = {
|
||||
name
|
||||
for name, module in model.named_modules()
|
||||
if AttentionFuser.match(None, module) is not None
|
||||
}
|
||||
assert matched == {f"layers.{i}.self_attn" for i in range(NUM_LAYERS)}
|
||||
|
||||
|
||||
def test_attention_layer_index_is_the_modules_own():
|
||||
"""The layer served comes from the module, not from its position in the stack.
|
||||
|
||||
LongCat Flash gives each decoder layer two attention sublayers numbered
|
||||
`2i` and `2i + 1`, so `num_hidden_layers` is twice the length of the stack
|
||||
and the enclosing layer's position is not the index the KV cache is keyed by.
|
||||
A vision tower is excluded separately, by `validate`: only the text config is
|
||||
patched to dispatch to vLLM, so a tower's attention is left to Transformers.
|
||||
"""
|
||||
model = build_model("longcat_flash", num_layers=2)
|
||||
text_config = model.config.get_text_config()
|
||||
assert len(model.layers) * 2 == text_config.num_hidden_layers
|
||||
|
||||
attentions = [
|
||||
model.get_submodule(f"layers.{i}.self_attn.{j}") for i in (0, 1) for j in (0, 1)
|
||||
]
|
||||
fusers = [AttentionFuser.match(None, attn) for attn in attentions]
|
||||
assert all(fuser is not None for fuser in fusers)
|
||||
assert [f.layer_index(a) for f, a in zip(fusers, attentions)] == [0, 1, 2, 3]
|
||||
|
||||
# Not dispatching to vLLM yet, as a vision tower never would be. The vLLM
|
||||
# config is not consulted, only the one the module was built with.
|
||||
assert not any(f.validate(a, None) for f, a in zip(fusers, attentions))
|
||||
text_config._attn_implementation = VLLM_ATTN_IMPL
|
||||
assert all(f.validate(a, None) for f, a in zip(fusers, attentions))
|
||||
|
||||
|
||||
def test_attention_dispatch_is_required(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Models selected by the permissive registry fail clearly without dispatch."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.models.transformers.base.get_pp_indices",
|
||||
lambda *_: (0, 1),
|
||||
)
|
||||
model = SimpleNamespace(
|
||||
text_config=SimpleNamespace(num_hidden_layers=1),
|
||||
pp_group=SimpleNamespace(rank_in_group=0, world_size=1),
|
||||
attention_fusers={},
|
||||
_get_attn_cls=lambda: None,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Layer 0 does not dispatch through the Transformers attention interface",
|
||||
):
|
||||
Base._create_attention_instances(model)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", ATTENTION_MODEL_TYPES)
|
||||
def test_attention_scale_is_the_declared_one(model_type: str):
|
||||
"""The scale comes off the module, not from the `head_size**-0.5` default.
|
||||
|
||||
Two of the three model types declare something the default would silently
|
||||
replace, so they are what makes this more than a tautology.
|
||||
"""
|
||||
attention = build_model(model_type).layers[0].self_attn
|
||||
fuser = AttentionFuser.match(None, attention)
|
||||
assert fuser is not None
|
||||
assert fuser.scale(attention) == attention.scaling
|
||||
|
||||
|
||||
def test_attention_scale_is_the_argument_not_the_attribute():
|
||||
"""OPT applies the scale to the query itself, then declares a literal 1.0.
|
||||
|
||||
Its `self.scaling` is the `head_size**-0.5` it has already applied, so
|
||||
reading the attribute rather than the argument the module hands the
|
||||
interface would scale twice.
|
||||
"""
|
||||
model = build_model("opt", ffn_dim=128, word_embed_proj_dim=64)
|
||||
attention = model.get_submodule("decoder.layers.0.self_attn")
|
||||
fuser = AttentionFuser.match(None, attention)
|
||||
assert fuser is not None
|
||||
assert fuser.scale(attention) == 1.0 != attention.scaling
|
||||
|
||||
|
||||
def test_attention_scale_rejects_unresolvable_expression():
|
||||
"""Unresolvable declared scales must not silently use the Llama default."""
|
||||
|
||||
class Attention(nn.Module):
|
||||
head_dim = 16
|
||||
|
||||
fuser = AttentionFuser(
|
||||
source_cls="Attention",
|
||||
scale_expr=ast.parse("self.head_dim**-0.5 * self.factor", mode="eval").body,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Cannot resolve attention scaling expression"):
|
||||
fuser.scale(Attention())
|
||||
|
||||
|
||||
class _ImageOnlyMRoPEModel:
|
||||
"""`get_rope_index` without `video_grid_thw` or `**kwargs` (e.g. HunYuanVL)."""
|
||||
|
||||
def get_rope_index(self, input_ids, image_grid_thw):
|
||||
seq_len = input_ids.shape[-1]
|
||||
positions = torch.arange(seq_len).view(1, 1, -1).expand(4, 1, -1)
|
||||
return positions, torch.tensor([0])
|
||||
|
||||
|
||||
def test_get_mrope_input_positions_omits_unsupported_grid_kwargs():
|
||||
"""Optional grids the model can't accept must not be passed when empty."""
|
||||
mixin = SimpleNamespace(model=_ImageOnlyMRoPEModel())
|
||||
|
||||
positions, delta = MultiModalMixin.get_mrope_input_positions(mixin, [1, 2, 3], [])
|
||||
|
||||
assert positions.shape == (4, 3)
|
||||
assert delta == 0
|
||||
@@ -1,611 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import warnings
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config.model import AttnTypeStr, ModelConfig, ModelDType, RunnerOption
|
||||
from vllm.config.pooler import SequencePoolingType, TokenPoolingType
|
||||
from vllm.logprobs import Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers import cached_tokenizer_from_config
|
||||
|
||||
from .. import ci_envs
|
||||
from .registry import HF_EXAMPLE_MODELS
|
||||
|
||||
TokensText = tuple[list[int], str]
|
||||
|
||||
|
||||
def check_outputs_equal(
|
||||
*,
|
||||
outputs_0_lst: Sequence[TokensText],
|
||||
outputs_1_lst: Sequence[TokensText],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
):
|
||||
"""Compare the two sequences generated by different models,
|
||||
which should be equal.
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
for prompt_idx, (outputs_0, outputs_1) in enumerate(
|
||||
zip(outputs_0_lst, outputs_1_lst)
|
||||
):
|
||||
output_ids_0, output_str_0 = outputs_0
|
||||
output_ids_1, output_str_1 = outputs_1
|
||||
|
||||
# The text and token outputs should exactly match
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\n{name_0}:\t{output_str_0!r}"
|
||||
f"\n{name_1}:\t{output_str_1!r}"
|
||||
)
|
||||
|
||||
assert output_str_0 == output_str_1, fail_msg
|
||||
assert output_ids_0 == output_ids_1, fail_msg
|
||||
|
||||
|
||||
# Representation of generated sequence as a tuple of
|
||||
# * Token ID list
|
||||
# * String
|
||||
# * List of top sample logprobs for each sampled token
|
||||
#
|
||||
# Assumes prompt logprobs were not requested.
|
||||
TokensTextLogprobs = tuple[
|
||||
list[int], str, list[dict[int, float]] | SampleLogprobs | None
|
||||
]
|
||||
|
||||
# Allow for tokens to be represented as str's rather than IDs;
|
||||
# tuple of
|
||||
# * Token string representations list
|
||||
# * String
|
||||
# * Optional list of top sample logprobs for each sampled token
|
||||
#
|
||||
# Assumes prompt logprobs were not requested.
|
||||
TextTextLogprobs = tuple[
|
||||
list[str], str, list[dict[str, float]] | list[dict[str, Logprob]] | None
|
||||
]
|
||||
|
||||
# Representation of generated sequence as a tuple of
|
||||
# * Token ID list
|
||||
# * String
|
||||
# * Optional list of top sample logprobs for each sampled token
|
||||
# * Optional list of top prompt logprobs for each prompt token
|
||||
#
|
||||
# Allows prompt logprobs to be requested.
|
||||
TokensTextLogprobsPromptLogprobs = tuple[
|
||||
list[int],
|
||||
str,
|
||||
list[dict[int, float]] | SampleLogprobs | None,
|
||||
list[dict[int, float] | None] | PromptLogprobs | None,
|
||||
]
|
||||
|
||||
|
||||
def check_logprobs_close(
|
||||
*,
|
||||
outputs_0_lst: Sequence[
|
||||
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
|
||||
],
|
||||
outputs_1_lst: Sequence[
|
||||
TokensTextLogprobs | TokensTextLogprobsPromptLogprobs | TextTextLogprobs
|
||||
],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
num_outputs_0_skip_tokens: int = 0,
|
||||
warn_on_mismatch: bool = True,
|
||||
always_check_logprobs: bool = False,
|
||||
) -> None:
|
||||
"""Compare the logprobs of two sequences generated by different models,
|
||||
which should be similar but not necessarily equal.
|
||||
|
||||
How sample logprobs are compared:
|
||||
* `always_check_logprobs == True`: set of highest-logprob token ids
|
||||
must match between seq0 and seq1 at all sampled token offsets
|
||||
* `always_check_logprobs == False`: highest-logprob token ids are
|
||||
only compared at sampled token offsets for which generated token
|
||||
ids don't match
|
||||
|
||||
Prompt logprobs must be provided either for both input sequences, or
|
||||
for neither. If prompt logprobs are provided, then highest-logprob
|
||||
prompt token ids must match between seq0 and seq1 at all prompt token
|
||||
offsets.
|
||||
|
||||
Args:
|
||||
outputs_0_lst: First sequence to compare
|
||||
outputs_0_lst: Second sequence to compare
|
||||
name_0: sequence #0 name
|
||||
name_1: sequence #1 name
|
||||
num_outputs_0_skip_tokens: If > 0, specifies the number of initial
|
||||
sequence #0 tokens & logprobs to discard
|
||||
before comparison, i.e. all
|
||||
of sequence #1 will be compared to
|
||||
sequence #0 beginning at index
|
||||
num_outputs_0_skip_tokens
|
||||
warn_on_mismatch: Issue a warning if there is token-wise or text-wise
|
||||
mismatch between the two sequences
|
||||
always_check_logprobs: If true, check logprobs even when tokens match
|
||||
|
||||
"""
|
||||
assert len(outputs_0_lst) == len(outputs_1_lst)
|
||||
|
||||
# Loop through responses to each prompt.
|
||||
for prompt_idx, (outputs_0, outputs_1) in enumerate(
|
||||
zip(outputs_0_lst, outputs_1_lst)
|
||||
):
|
||||
assert len(outputs_0) == len(outputs_1)
|
||||
if len(outputs_0) == 3:
|
||||
assert len(outputs_1) == 3
|
||||
# Break out tokens, text & sample logprobs
|
||||
# (prompt logprobs were not provided)
|
||||
output_ids_0, output_str_0, logprobs_0 = outputs_0
|
||||
output_ids_1, output_str_1, logprobs_1 = outputs_1
|
||||
elif len(outputs_0) == 4:
|
||||
assert len(outputs_1) == 4
|
||||
# Break out tokens, text, sample logprobs & prompt logprobs
|
||||
(
|
||||
output_ids_0,
|
||||
output_str_0,
|
||||
logprobs_0,
|
||||
prompt_logprobs_0,
|
||||
) = outputs_0
|
||||
(
|
||||
output_ids_1,
|
||||
output_str_1,
|
||||
logprobs_1,
|
||||
prompt_logprobs_1,
|
||||
) = outputs_1
|
||||
|
||||
# Test prompt logprobs closeness
|
||||
if prompt_logprobs_0 is not None and prompt_logprobs_1 is not None:
|
||||
# Both sequences' prompt logprobs lists are not `None`
|
||||
# (although individual list elements may be `None`);
|
||||
# for each token's logprobs:
|
||||
for idx, (logprobs_elem_0, logprobs_elem_1) in enumerate(
|
||||
zip(prompt_logprobs_0, prompt_logprobs_1)
|
||||
):
|
||||
fail_msg = (
|
||||
f"Prompt logprobs test:"
|
||||
f"\n{name_0}:\tPrompt index {idx}\t{logprobs_elem_0}"
|
||||
f"\n{name_1}:\tPrompt index {idx}\t{logprobs_elem_1}"
|
||||
)
|
||||
|
||||
if logprobs_elem_0 is None:
|
||||
# If the seq 0 token's logprobs are `None`,
|
||||
# the seq 1 token's logprobs must be `None`
|
||||
assert logprobs_elem_1 is None, fail_msg
|
||||
else:
|
||||
# If the seq 0 token's logprobs are not `None`,
|
||||
# the seq 1 token's logprobs must not be `None`
|
||||
assert logprobs_elem_1 is not None, fail_msg
|
||||
# Logprobs check: top-k token choices must be the same
|
||||
assert set(logprobs_elem_0.keys()) == set(
|
||||
logprobs_elem_1.keys()
|
||||
), fail_msg
|
||||
else:
|
||||
# Both sequence logprobs lists must be `None`
|
||||
fail_msg = (
|
||||
f"Prompt logprobs test:"
|
||||
f"\n{name_0}:\tlogprobs\t{prompt_logprobs_0}"
|
||||
f"\n{name_1}:\tlogprobs\t{prompt_logprobs_1}"
|
||||
)
|
||||
|
||||
assert prompt_logprobs_0 is None and prompt_logprobs_1 is None, fail_msg
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Outputs tuple must have 3 or 4 elements but "
|
||||
f"{len(outputs_0)} elements were provided: "
|
||||
f"{outputs_0}"
|
||||
)
|
||||
|
||||
if logprobs_0 is None:
|
||||
logprobs_0 = [None] * len(output_ids_0)
|
||||
if logprobs_1 is None:
|
||||
logprobs_1 = [None] * len(output_ids_1)
|
||||
|
||||
# Skip specified number of initial sequence #0 tokens
|
||||
# & logprobs, leaving output text as-is for simplicity
|
||||
# (text mismatches may generate warnings but do not
|
||||
# cause the test to fail.)
|
||||
if num_outputs_0_skip_tokens < 0:
|
||||
raise ValueError("num_outputs_0_skip_tokens must be non-negative")
|
||||
output_ids_0 = output_ids_0[num_outputs_0_skip_tokens:]
|
||||
logprobs_0 = logprobs_0[num_outputs_0_skip_tokens:]
|
||||
|
||||
# Loop through generated tokens.
|
||||
for idx, (output_id_0, output_id_1) in enumerate(
|
||||
zip(output_ids_0, output_ids_1)
|
||||
):
|
||||
is_tok_mismatch = output_id_0 != output_id_1
|
||||
|
||||
# If generated tokens don't match
|
||||
# or it is desired to always check logprobs,
|
||||
# then
|
||||
if is_tok_mismatch or always_check_logprobs:
|
||||
logprobs_elem_0 = logprobs_0[idx]
|
||||
logprobs_elem_1 = logprobs_1[idx]
|
||||
|
||||
# Each predicted token must be in top N logprobs of the other
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\nMatched tokens:\t{output_ids_0[:idx]}"
|
||||
f"\n{name_0}:\t{output_str_0!r}\t{logprobs_elem_0}"
|
||||
f"\n{name_1}:\t{output_str_1!r}\t{logprobs_elem_1}"
|
||||
)
|
||||
|
||||
assert logprobs_elem_0 is not None, fail_msg
|
||||
assert logprobs_elem_1 is not None, fail_msg
|
||||
assert output_id_0 in logprobs_elem_1, fail_msg
|
||||
assert output_id_1 in logprobs_elem_0, fail_msg
|
||||
|
||||
if warn_on_mismatch and is_tok_mismatch:
|
||||
with warnings.catch_warnings():
|
||||
# This ensures that repeated warnings are shown
|
||||
# in the output, not just the first occurrence
|
||||
warnings.simplefilter("always")
|
||||
|
||||
warnings.warn(fail_msg, stacklevel=2)
|
||||
|
||||
# Break out since sequences will now diverge.
|
||||
break
|
||||
else:
|
||||
if output_str_0 != output_str_1 and warn_on_mismatch:
|
||||
# The token outputs exactly match,
|
||||
# so the text outputs should exactly match as well
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\n{name_0}:\t{output_str_0!r}"
|
||||
f"\n{name_1}:\t{output_str_1!r}"
|
||||
)
|
||||
|
||||
with warnings.catch_warnings():
|
||||
# This ensures that repeated warnings are shown
|
||||
# in the output, not just the first occurrence
|
||||
warnings.simplefilter("always")
|
||||
|
||||
warnings.warn(fail_msg, stacklevel=2)
|
||||
|
||||
|
||||
def build_model_context(
|
||||
model_id: str,
|
||||
runner: RunnerOption = "auto",
|
||||
dtype: ModelDType = "auto",
|
||||
model_config_kwargs: dict[str, Any] | None = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
limit_mm_per_prompt: Mapping[str, int | Mapping[str, int]] | None = None,
|
||||
mm_processor_cache_gb: int = 0,
|
||||
):
|
||||
"""Creates an InputProcessingContext for a given model.
|
||||
|
||||
Args:
|
||||
model_id: ID of the model being considered.
|
||||
mm_processor_kwargs: optional processor kwargs for to be leveraged
|
||||
in the input processor, mapper, dummy data creation, etc.
|
||||
limit_mm_per_prompt: Multimodal limits.
|
||||
|
||||
Returns:
|
||||
InputProcessingContext for the model being considered.
|
||||
|
||||
"""
|
||||
model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id)
|
||||
model_info.check_available_online(on_fail="skip")
|
||||
model_info.check_transformers_version(
|
||||
on_fail="skip",
|
||||
check_max_version=False,
|
||||
check_version_reason="vllm",
|
||||
)
|
||||
|
||||
model_config_kwargs = model_config_kwargs or {}
|
||||
limit_mm_per_prompt = {
|
||||
modality: dict(limit) if isinstance(limit, Mapping) else limit
|
||||
for modality, limit in (limit_mm_per_prompt or {}).items()
|
||||
}
|
||||
model_config = ModelConfig(
|
||||
model_id,
|
||||
runner=runner,
|
||||
tokenizer=model_info.tokenizer or model_id,
|
||||
tokenizer_mode=model_info.tokenizer_mode,
|
||||
revision=model_info.revision,
|
||||
trust_remote_code=model_info.trust_remote_code,
|
||||
dtype=dtype,
|
||||
seed=0,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
limit_mm_per_prompt=limit_mm_per_prompt,
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
hf_overrides=model_info.hf_overrides,
|
||||
skip_tokenizer_init=model_info.require_embed_inputs,
|
||||
enable_prompt_embeds=model_info.require_embed_inputs,
|
||||
enable_mm_embeds=model_info.require_embed_inputs,
|
||||
enforce_eager=model_info.enforce_eager,
|
||||
**model_config_kwargs,
|
||||
)
|
||||
|
||||
return InputProcessingContext(
|
||||
model_config,
|
||||
tokenizer=cached_tokenizer_from_config(model_config),
|
||||
)
|
||||
|
||||
|
||||
def check_embeddings_close(
|
||||
*,
|
||||
embeddings_0_lst: Sequence[list[float]],
|
||||
embeddings_1_lst: Sequence[list[float]],
|
||||
name_0: str,
|
||||
name_1: str,
|
||||
tol: float = 1e-3,
|
||||
) -> None:
|
||||
assert len(embeddings_0_lst) == len(embeddings_1_lst)
|
||||
|
||||
for prompt_idx, (embeddings_0, embeddings_1) in enumerate(
|
||||
zip(embeddings_0_lst, embeddings_1_lst)
|
||||
):
|
||||
assert len(embeddings_0) == len(embeddings_1), (
|
||||
f"Length mismatch: {len(embeddings_0)} vs. {len(embeddings_1)}"
|
||||
)
|
||||
|
||||
sim = F.cosine_similarity(
|
||||
torch.tensor(embeddings_0), torch.tensor(embeddings_1), dim=0
|
||||
)
|
||||
|
||||
fail_msg = (
|
||||
f"Test{prompt_idx}:"
|
||||
f"\nCosine similarity: \t{sim:.4f}"
|
||||
f"\n{name_0}:\t{embeddings_0[:16]!r}"
|
||||
f"\n{name_1}:\t{embeddings_1[:16]!r}"
|
||||
)
|
||||
|
||||
assert sim >= 1 - tol, fail_msg
|
||||
|
||||
|
||||
def matryoshka_fy(tensor: torch.Tensor, dimensions: int):
|
||||
tensor = torch.tensor(tensor)
|
||||
tensor = tensor[..., :dimensions]
|
||||
tensor = F.normalize(tensor, p=2, dim=1)
|
||||
return tensor
|
||||
|
||||
|
||||
def softmax(data):
|
||||
if data.shape[-1] == 1:
|
||||
return F.sigmoid(data)
|
||||
else:
|
||||
return F.softmax(data, dim=-1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelInfo:
|
||||
name: str
|
||||
revision: str | None = None
|
||||
architecture: str = ""
|
||||
dtype: str = "auto"
|
||||
max_model_len: int | None = None
|
||||
hf_dtype: str = "float32"
|
||||
hf_overrides: dict[str, Any] | None = None
|
||||
seq_pooling_type: SequencePoolingType | None = None
|
||||
tok_pooling_type: TokenPoolingType | None = None
|
||||
attn_type: AttnTypeStr | None = None
|
||||
is_prefix_caching_supported: bool | None = None
|
||||
is_chunked_prefill_supported: bool | None = None
|
||||
enable_test: bool = True
|
||||
|
||||
|
||||
@dataclass
|
||||
class EmbedModelInfo(ModelInfo):
|
||||
mteb_score: float | None = None
|
||||
# Per-model MTEB score tolerance; falls back to MTEB_EMBED_TOL.
|
||||
mteb_tol: float | None = None
|
||||
is_matryoshka: bool = False
|
||||
matryoshka_dimensions: list[int] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RerankModelInfo(ModelInfo):
|
||||
mteb_score: float | None = None
|
||||
# Per-model MTEB score tolerance; falls back to MTEB_RERANK_TOL.
|
||||
mteb_tol: float | None = None
|
||||
chat_template_name: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerateModelInfo(ModelInfo):
|
||||
hf_dtype: str = "auto"
|
||||
hf_ppl: float | None = None
|
||||
|
||||
|
||||
def get_vllm_extra_kwargs(model_info: ModelInfo, vllm_extra_kwargs):
|
||||
# A model family has many models with the same architecture,
|
||||
# and we don't need to test each one.
|
||||
if not ci_envs.VLLM_CI_NO_SKIP and not model_info.enable_test:
|
||||
import pytest
|
||||
|
||||
pytest.skip("Skipping test.")
|
||||
|
||||
# Allow vllm to test using the given dtype, such as float32
|
||||
vllm_extra_kwargs = vllm_extra_kwargs or {}
|
||||
vllm_extra_kwargs["dtype"] = ci_envs.VLLM_CI_DTYPE or model_info.dtype
|
||||
|
||||
# Allow vllm to test using hf_overrides
|
||||
if model_info.hf_overrides is not None:
|
||||
vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
|
||||
|
||||
# Allow changing the head dtype used by vllm in tests
|
||||
if ci_envs.VLLM_CI_HEAD_DTYPE is not None:
|
||||
if "hf_overrides" not in vllm_extra_kwargs:
|
||||
vllm_extra_kwargs["hf_overrides"] = {}
|
||||
vllm_extra_kwargs["hf_overrides"]["head_dtype"] = ci_envs.VLLM_CI_HEAD_DTYPE
|
||||
|
||||
# Allow control over whether tests use enforce_eager
|
||||
if ci_envs.VLLM_CI_ENFORCE_EAGER is not None:
|
||||
vllm_extra_kwargs["enforce_eager"] = ci_envs.VLLM_CI_ENFORCE_EAGER
|
||||
|
||||
return vllm_extra_kwargs
|
||||
|
||||
|
||||
def dummy_hf_overrides(
|
||||
hf_config: PreTrainedConfig,
|
||||
*,
|
||||
model_arch: str = "",
|
||||
exist_overrides: dict[str, Any] | None = None,
|
||||
use_original_num_layers: bool = False,
|
||||
) -> PreTrainedConfig:
|
||||
"""Dummy HF overrides function used to create dummy model
|
||||
with only minimum nums of layer.
|
||||
"""
|
||||
# Copy because this helper is called more than once
|
||||
# while loading config, and we `.pop()`
|
||||
exist_overrides = (exist_overrides or {}).copy()
|
||||
text_config_override = exist_overrides.pop("text_config", None)
|
||||
hf_config.update(exist_overrides)
|
||||
|
||||
text_config = hf_config.get_text_config()
|
||||
if text_config_override is not None:
|
||||
# multimodal test models may override *some* text-model fields
|
||||
text_config.update(text_config_override)
|
||||
|
||||
# Ensure at least 2 expert per group
|
||||
# Since `grouped_topk` assumes top-2
|
||||
n_group = getattr(text_config, "n_group", None)
|
||||
# Kimi uses `num_expert_group` instead of `n_group`.
|
||||
if n_group is None:
|
||||
n_group = getattr(text_config, "num_expert_group", None)
|
||||
# InternS1Pro uses `router_n_groups` instead of `n_group`.
|
||||
if n_group is None:
|
||||
n_group = getattr(text_config, "router_n_groups", None)
|
||||
num_experts = n_group * 2 if n_group is not None else 2
|
||||
|
||||
# we use three layers for Gemma-3n to check
|
||||
# both normal layer and kv_shared_layer
|
||||
if use_original_num_layers:
|
||||
# Use the original number of layers from the config
|
||||
num_layers = getattr(text_config, "num_layers", 1)
|
||||
num_hidden_layers = getattr(text_config, "num_hidden_layers", 1)
|
||||
else:
|
||||
# Use minimal layers for testing
|
||||
num_layers = 1
|
||||
num_hidden_layers = (
|
||||
3
|
||||
if model_arch
|
||||
in (
|
||||
"Gemma3nForConditionalGeneration",
|
||||
"Gemma4ForCausalLM",
|
||||
"Gemma4ForConditionalGeneration",
|
||||
"Gemma4MTPModel",
|
||||
"DiffusionGemmaForBlockDiffusion",
|
||||
)
|
||||
else 1
|
||||
)
|
||||
|
||||
update_dict = {
|
||||
"num_layers": num_layers,
|
||||
# For Gemma-3n
|
||||
"num_kv_shared_layers": 1,
|
||||
}
|
||||
|
||||
_hf_config = hf_config
|
||||
|
||||
class DummyConfig:
|
||||
hf_config = _hf_config
|
||||
hf_text_config = text_config
|
||||
|
||||
# Keep architecture conversion on the default multimodal path; this
|
||||
# helper only needs HF-derived fields, not deployment MM limits.
|
||||
@staticmethod
|
||||
def _supports_multimodal_for_mm_prefix() -> bool:
|
||||
return True
|
||||
|
||||
model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
|
||||
# Only set MoE related config when the model has MoE layers.
|
||||
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
|
||||
if model_arch_config.num_experts > 0:
|
||||
num_experts_per_tok = 2
|
||||
if model_arch in (
|
||||
"Llama4ForConditionalGeneration",
|
||||
"Llama4ForCausalLM",
|
||||
"EagleLlama4ForCausalLM",
|
||||
):
|
||||
num_experts_per_tok = 1
|
||||
elif model_arch == "InternS1ProForConditionalGeneration":
|
||||
assert n_group is not None
|
||||
num_experts_per_tok = n_group
|
||||
update_dict.update(
|
||||
{
|
||||
"num_experts": num_experts,
|
||||
"num_experts_per_tok": num_experts_per_tok,
|
||||
# Kimi uses `num_experts_per_token`.
|
||||
"num_experts_per_token": num_experts_per_tok,
|
||||
"num_local_experts": num_experts,
|
||||
# Otherwise there will not be any expert layers
|
||||
"first_k_dense_replace": 0,
|
||||
# To avoid OOM on DeepSeek-V3
|
||||
"n_routed_experts": num_experts,
|
||||
}
|
||||
)
|
||||
|
||||
# Update num_hidden_layers for non-Longcat architectures (Longcat derives it
|
||||
# from num_layers for its dual-attention layers).
|
||||
if model_arch not in (
|
||||
"LongcatFlashForCausalLM",
|
||||
"LongCatFlashMTPModel",
|
||||
"LongcatFlashNgramForCausalLM",
|
||||
):
|
||||
update_dict["num_hidden_layers"] = num_hidden_layers
|
||||
|
||||
text_config.update(update_dict)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
# HY V4 uses this to decide which sparse-attention indexers are shared.
|
||||
if indexer_types := getattr(text_config, "indexer_types", None):
|
||||
text_config.update({"indexer_types": indexer_types[:num_hidden_layers]})
|
||||
|
||||
# Update n_layers and moe configs for Moondream3 model
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
text_config.update(
|
||||
{
|
||||
"n_layers": num_hidden_layers,
|
||||
"moe_num_experts": num_experts,
|
||||
"moe_experts_per_token": 2,
|
||||
"moe_start_layer": num_hidden_layers,
|
||||
}
|
||||
)
|
||||
|
||||
if hasattr(hf_config, "vision_config"):
|
||||
vision_config = hf_config.vision_config
|
||||
vision_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# Keep per-layer metadata consistent with the reduced layer count.
|
||||
if layer_types := getattr(vision_config, "layer_types", None):
|
||||
vision_config.update({"layer_types": layer_types[:1]})
|
||||
|
||||
if model_arch in ("Moondream3ForCausalLM", "HfMoondream"):
|
||||
vision_config.update({"enc_n_layers": 1})
|
||||
|
||||
# e.g.: ibm-granite/granite-speech-3.3-2b
|
||||
if hasattr(hf_config, "encoder_config"):
|
||||
hf_config.encoder_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
# e.g.: Qwen/Qwen2-Audio-7B-Instruct
|
||||
# audio_config may exist but be None (e.g. audio-less Gemma4 variants).
|
||||
if getattr(hf_config, "audio_config", None) is not None:
|
||||
hf_config.audio_config.update(
|
||||
{
|
||||
"num_layers": 1,
|
||||
"num_hidden_layers": 1,
|
||||
"encoder_layers": 1,
|
||||
}
|
||||
)
|
||||
|
||||
return hf_config
|
||||
Reference in New Issue
Block a user