Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 22:06:10 +07:00
327 changed files with 13704 additions and 5433 deletions
@@ -8,10 +8,6 @@ import torch
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA-only dispatch"
)
def _require_deep_gemm() -> None:
from vllm.utils.deep_gemm import has_deep_gemm
@@ -20,17 +16,10 @@ def _require_deep_gemm() -> None:
pytest.skip("kpool indexer requires DeepGEMM")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
def _build(indexer_cls, backend: str):
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
op = SparseAttnIndexerKpool(
return indexer_cls(
k_cache=None,
quant_block_size=128,
scale_fmt="ue8m0",
@@ -40,4 +29,73 @@ def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> N
max_total_seq_len=8192,
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
)
assert op.topk_backend == backend
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
and hand it to the shared SparseIndexerTopk dispatcher, rather than
hard-coding its own cooperative/persistent/per_row choice."""
_require_deep_gemm()
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
backend: str,
) -> None:
"""The AMD kpool indexer must go through the same dispatcher, so the
AITER decode top-k is reachable from kernel_config."""
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
@pytest.mark.parametrize(
"configured,full_cudagraph,expected",
[
("auto", False, "aiter"),
("auto", True, "auto"),
("aiter", True, "aiter"),
("per_row", False, "per_row"),
],
)
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
monkeypatch: pytest.MonkeyPatch,
configured: str,
full_cudagraph: bool,
expected: str,
) -> None:
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
use the context length to pick AITER and must stay on the in-tree kernel.
Explicit backends keep working, and outside FULL the long-context heuristic
still narrows to AITER."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.models.glm5next.amd import sparse_indexer
monkeypatch.setattr(
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
)
monkeypatch.setattr(
rocm_aiter_ops,
"is_indexer_top_k_supported",
lambda **kwargs: True,
raising=False,
)
assert (
sparse_indexer._kpool_decode_topk_backend(
configured,
num_rows=8,
max_valid_seq_len=64 * 1024,
select_k=512,
index_kpool=4,
full_cudagraph=full_cudagraph,
)
== expected
)
+108
View File
@@ -0,0 +1,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import sys
from collections.abc import Callable
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock, patch
import pytest
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.layers.fused_moe import RoutedExperts
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.quantization.mxfp4 import (
_moe_weight_override_is_int4,
_use_k3_situ_int4_gfx942,
)
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
pytestmark = pytest.mark.cpu_test
class _FakeRocmModule(ModuleType):
on_gfx942: Callable[[], bool]
class _FakeAiterOpsModule(ModuleType):
rocm_aiter_ops: SimpleNamespace
def _vllm_config(moe_weight: str | None):
quantization_config = (
None
if moe_weight is None
else QuantizationConfigArgs(moe={"weight": moe_weight})
)
return SimpleNamespace(
model_config=SimpleNamespace(quantization_config=quantization_config)
)
@pytest.mark.parametrize(
("moe_weight", "expected"),
[
(None, False),
("mxfp4", False),
("int4_per_group_32", True),
],
)
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
monkeypatch.setattr(
"vllm.config.get_current_vllm_config",
lambda: _vllm_config(moe_weight),
)
assert _moe_weight_override_is_int4() is expected
@pytest.mark.parametrize("int4_requested", [False, True])
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
moe = SimpleNamespace(
activation=MoEActivation.SITU,
activation_situ_linear_beta=25.0,
)
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
fake_rocm.on_gfx942 = lambda: True
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
with (
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
patch.dict(
sys.modules,
{
"vllm.platforms.rocm": fake_rocm,
"vllm._aiter_ops": fake_aiter_ops,
},
),
patch(
"vllm.model_executor.layers.quantization.mxfp4."
"_moe_weight_override_is_int4",
return_value=int4_requested,
),
):
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
def test_int4_per_group_32_is_not_an_online_moe_method():
# Load-time gfx942 requant. Must not match the online MoE table.
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
)
assert (
config.resolve_quant_method_cls(
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
)
is None
)
def test_unknown_online_moe_weight_still_raises():
config = OnlineQuantizationConfig(
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
)
with pytest.raises(ValueError, match="online quantization"):
config.resolve_quant_method_cls(
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
)
+375
View File
@@ -0,0 +1,375 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Kimi-K3 multi-turn prefix cache reuse with KV offload, P/D and DCP."""
import contextlib
import json
import random
from collections.abc import Iterator
from dataclasses import dataclass
from math import lcm
import pytest
import requests
from prometheus_client.parser import text_string_to_metric_families
from tests.models.utils import check_logprobs_close
from tests.utils import RemoteOpenAIServer, multi_gpu_marks
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
# Official Kimi-K3 config with 8 layers and no weights.
MODEL = "riverclouds/Kimi-K3-8L-dummy"
# The recipe's DSpark draft, reading target layers that exist in MODEL.
DRAFT = "riverclouds/Kimi-K3-DSpark-for-8L-dummy"
# Conversation: the first prompt spans FIRST_PROMPT_BLOCKS blocks plus an
# unaligned tail, and each later turn appends TURN_TOKENS random tokens.
NUM_TURNS = 3
FIRST_PROMPT_BLOCKS = 3
UNALIGNED_TAIL = 37
TURN_TOKENS = 777
MAX_TOKENS = 8
NUM_LOGPROBS = 5
STARTUP_TIMEOUT = 1200
BASE_ARGS = [
"--trust-remote-code",
"--language-model-only",
"--load-format",
"dummy",
"--no-enable-flashinfer-autotune",
"--enable-prompt-tokens-details",
# Default max_num_seqs OOMs one GPU during warmup.
"--max-num-seqs",
"8",
]
pytestmark = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Kimi-K3 NVIDIA kernels require the SM100 family",
)
SPEC_CONFIG = {
"method": "dspark",
"model": DRAFT,
"num_speculative_tokens": 7,
"draft_sample_method": "probabilistic",
"rejection_sample_method": "block",
}
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
OFFLOAD = {
"kv_connector": "SimpleCPUOffloadConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {
"cpu_bytes_to_use_per_rank": 32 << 30,
"lazy_offload": False,
},
}
NIXL_OFFLOAD = {
"kv_connector": "MultiConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"connectors": [NIXL, OFFLOAD]},
}
@dataclass(frozen=True)
class Mode:
# A finer prefix_match_unit enables partial hits inside a Mamba block.
prefix_match_unit: int | None
spec: bool
@property
def name(self) -> str:
name = "block" if self.prefix_match_unit is None else "partial"
return f"{name}-dspark" if self.spec else name
MODES = [Mode(unit, spec) for spec in (False, True) for unit in (None, 128)]
@dataclass(frozen=True)
class Instance:
tp: int = 1
dp: int = 1
dcp: int = 1
kv_config: dict | None = None
prefix_caching: bool = True
@property
def num_gpus(self) -> int:
return self.tp * self.dp
def args(self, mode: Mode) -> list[str]:
args = BASE_ARGS + ["--tensor-parallel-size", str(self.tp)]
if mode.spec:
args += ["--speculative-config", json.dumps(SPEC_CONFIG)]
if self.dp > 1:
args += ["--data-parallel-size", str(self.dp), "--enable-expert-parallel"]
if self.dcp > 1:
args += ["--decode-context-parallel-size", str(self.dcp)]
if self.kv_config is not None:
args += ["--kv-transfer-config", json.dumps(self.kv_config)]
if not self.prefix_caching:
return args + ["--no-enable-prefix-caching"]
args.append("--enable-prefix-caching")
if mode.prefix_match_unit is not None:
args += ["--prefix-match-unit", str(mode.prefix_match_unit)]
return args
@dataclass(frozen=True)
class Deployment:
decode: Instance
prefill: Instance | None = None
# Reset GPU prefix caches each turn so hits must come from CPU.
offload: bool = False
@property
def instances(self) -> tuple[Instance, ...]:
return (self.prefill, self.decode) if self.prefill else (self.decode,)
@property
def num_gpus(self) -> int:
return sum(i.num_gpus for i in self.instances)
DEPLOYMENTS = {
"plain": Deployment(Instance()),
"offload": Deployment(Instance(kv_config=OFFLOAD), offload=True),
"dcp2": Deployment(Instance(tp=2, dcp=2)),
"dcp2-offload": Deployment(Instance(tp=2, dcp=2, kv_config=OFFLOAD), offload=True),
"pd": Deployment(Instance(kv_config=NIXL), prefill=Instance(kv_config=NIXL)),
# NIXL rejects prefix caching on a hybrid decoder with a different TP.
"pd-tp2-dep2": Deployment(
Instance(dp=2, kv_config=NIXL, prefix_caching=False),
prefill=Instance(tp=2, kv_config=NIXL),
),
"pd-offload": Deployment(
Instance(kv_config=NIXL_OFFLOAD),
prefill=Instance(kv_config=NIXL_OFFLOAD),
offload=True,
),
}
def _known_failure(name: str, mode: Mode) -> str | None:
deployment = DEPLOYMENTS[name]
if mode.spec and deployment.decode.dcp > 1:
return "FlashInfer MLA DCP decode breaks on ragged spec batches (#59392)"
return None
def _case(name: str, mode: Mode):
marks = []
if (num_gpus := DEPLOYMENTS[name].num_gpus) > 1:
marks += multi_gpu_marks(num_gpus=num_gpus)
if reason := _known_failure(name, mode):
marks.append(pytest.mark.xfail(reason=reason, strict=True))
return pytest.param(name, mode, marks=marks, id=f"{name}-{mode.name}")
def _block_size(url: str) -> int:
response = requests.get(f"{url}/metrics", timeout=30)
response.raise_for_status()
for family in text_string_to_metric_families(response.text):
for sample in family.samples:
if sample.name == "vllm:cache_config_info":
return lcm(
*(
int(value)
for key in ("block_size", "mamba_block_size")
if (value := sample.labels.get(key, "None")) != "None"
)
)
raise AssertionError("missing vllm:cache_config_info metric")
def _complete(url: str, prompt: list[int], salt: str, **extra) -> dict:
body = {
"model": MODEL,
"prompt": prompt,
"max_tokens": MAX_TOKENS,
"temperature": 0,
"logprobs": NUM_LOGPROBS,
"return_tokens_as_token_ids": True,
"cache_salt": salt,
**extra,
}
response = requests.post(f"{url}/v1/completions", json=body, timeout=600)
response.raise_for_status()
return response.json()
def _pd_complete(
prefill_url: str, decode_url: str, prompt: list[int], salt: str
) -> tuple[dict, dict]:
remote_decode = {
"do_remote_decode": True,
"do_remote_prefill": False,
"remote_engine_id": None,
"remote_block_ids": None,
"remote_host": None,
"remote_port": None,
}
prefill = _complete(
prefill_url, prompt, salt, max_tokens=1, kv_transfer_params=remote_decode
)
decode = _complete(
decode_url, prompt, salt, kv_transfer_params=prefill["kv_transfer_params"]
)
return prefill, decode
def _cached(response: dict) -> int:
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
def _token_id(token: str) -> int:
return int(token.removeprefix("token_id:"))
def _tokens_text_logprobs(response: dict):
choice = response["choices"][0]
ids = [_token_id(t) for t in choice["logprobs"]["tokens"]]
top = [
{_token_id(t): lp for t, lp in d.items()}
for d in choice["logprobs"]["top_logprobs"]
]
return ids, choice["text"], top
def _reset_gpu_prefix_cache(url: str) -> None:
response = requests.post(
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
)
response.raise_for_status()
@dataclass(frozen=True)
class Turn:
prompt_len: int
cached: int
expected_cached: int
recompute_cached: int
output: tuple
recompute: tuple
@property
def matches_recompute(self) -> bool:
return self.output[0] == self.recompute[0]
def _expected_cached(prev_prompt_len: int, hit_unit: int, spec: bool) -> int:
"""A turn must reuse the previous prompt up to its last checkpoint.
EAGLE-style drafts drop the trailing unit, so the checkpoint is one earlier.
"""
if not prev_prompt_len:
return 0
checkpoint = (prev_prompt_len - 1) // hit_unit * hit_unit
return max(checkpoint - hit_unit, 0) if spec else checkpoint
@contextlib.contextmanager
def _serve(deployment: Deployment, mode: Mode) -> Iterator[list[RemoteOpenAIServer]]:
with contextlib.ExitStack() as stack:
# Sequential shutdown waits on memory still held by sibling servers.
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
next_gpu = 0
for instance in deployment.instances:
env = {}
if deployment.offload:
# Enables /reset_prefix_cache.
env["VLLM_SERVER_DEV_MODE"] = "1"
if deployment.prefill is not None:
gpus = range(next_gpu, next_gpu + instance.num_gpus)
next_gpu += instance.num_gpus
env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus))
env["VLLM_SSM_CONV_STATE_LAYOUT"] = "DS"
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(get_open_port())
servers.append(
RemoteOpenAIServer(
MODEL,
instance.args(mode),
env_dict=env,
max_wait_seconds=STARTUP_TIMEOUT,
)
)
yield servers
def _run_conversation(
deployment: Deployment,
servers: list[RemoteOpenAIServer],
mode: Mode,
) -> list[Turn]:
# Under P/D the prefiller computes the prompt, so hits are read from it.
compute_url, decode_url = servers[0].url_root, servers[-1].url_root
block_size = _block_size(compute_url)
hit_unit = mode.prefix_match_unit or block_size
rng = random.Random(0)
def new_tokens(n: int) -> list[int]:
return [rng.randint(1000, 150000) for _ in range(n)]
prompt = new_tokens(FIRST_PROMPT_BLOCKS * block_size + UNALIGNED_TAIL)
prev_len = 0
turns = []
for turn in range(NUM_TURNS):
if deployment.offload:
for server in servers:
_reset_gpu_prefix_cache(server.url_root)
if deployment.prefill is not None:
computed, output = _pd_complete(compute_url, decode_url, prompt, "conv")
else:
computed = output = _complete(decode_url, prompt, "conv")
recompute = _complete(decode_url, prompt, f"recompute-{turn}")
turns.append(
Turn(
prompt_len=len(prompt),
cached=_cached(computed),
expected_cached=_expected_cached(prev_len, hit_unit, mode.spec),
recompute_cached=_cached(recompute),
output=_tokens_text_logprobs(output),
recompute=_tokens_text_logprobs(recompute),
)
)
prev_len = len(prompt)
prompt = prompt + new_tokens(TURN_TOKENS)
return turns
def _format(turns: list[Turn]) -> str:
rows = ["turn prompt cached expected matches_recompute"]
rows += [
f"{i:>4} {t.prompt_len:>6} {t.cached:>6} {t.expected_cached:>8} "
f"{t.matches_recompute}"
for i, t in enumerate(turns)
]
return "\n".join(rows)
@pytest.mark.parametrize(
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
)
def test_turns_reuse_prefix_and_match_recompute(name: str, mode: Mode) -> None:
deployment = DEPLOYMENTS[name]
with _serve(deployment, mode) as servers:
turns = _run_conversation(deployment, servers, mode)
table = _format(turns)
print(f"\n{name}-{mode.name}\n{table}")
assert all(t.recompute_cached == 0 for t in turns), table
assert all(t.cached >= t.expected_cached for t in turns), table
# Dummy weights differ across TP sizes.
if len({i.tp for i in deployment.instances}) == 1:
check_logprobs_close(
outputs_0_lst=[t.recompute for t in turns],
outputs_1_lst=[t.output for t in turns],
name_0="recompute",
name_1=name,
)
@@ -5,16 +5,15 @@ from transformers import GraniteConfig
from vllm.model_executor.models.granite import granite_layer_attn_params
from ...utils import check_logprobs_close, check_transformers_version
from ...utils import check_logprobs_close
# model -> minimum transformers version, or None if unconstrained
MODELS = {
MODELS = [
# TODO(sang): Sliding window should be tested separately.
"ibm/PowerLM-3b": None,
"ibm/PowerMoE-3b": None,
"ibm-granite/granite-swash-2b": "5.15.1",
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
}
"ibm/PowerLM-3b",
"ibm/PowerMoE-3b",
"ibm-granite/granite-swash-2b",
"ibm-granite/granite-swash-3b-a600m",
]
@pytest.mark.parametrize("model", MODELS)
@@ -31,8 +30,6 @@ def test_models(
max_tokens: int,
num_logprobs: int,
) -> None:
check_transformers_version(model, min_transformers_version=MODELS[model])
with hf_runner(model, dtype=dtype) as hf_model:
hf_outputs = hf_model.generate_greedy_logprobs_limit(
example_prompts, max_tokens, num_logprobs
@@ -799,7 +799,6 @@ VLM_TEST_SETTINGS = {
dtype="half",
num_logprobs=10,
patch_hf_runner=model_utils.ovis2_5_patch_hf_runner,
hf_model_kwargs={"revision": "refs/pr/5"},
),
"paddleocr_vl": VLMTestInfo(
models=["PaddlePaddle/PaddleOCR-VL"],
@@ -104,16 +104,7 @@ def run_test(
)
@pytest.mark.parametrize(
"model",
[
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
# instead of shipping a separate lm_head.weight tensor; this
# exercises that path (see tie_weights() in nemotron_parse.py).
"nvidia/NVIDIA-Nemotron-Parse-2.0",
],
)
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"])
@pytest.mark.parametrize("dtype", ["bfloat16"])
@pytest.mark.parametrize("num_logprobs", [5])
def test_models(
@@ -12,12 +12,22 @@ Run with:
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
"""
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from transformers import AutoTokenizer
from vllm.model_executor.models.deepseek_ocr import DeepseekOCRImagePixelInputs
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
from vllm.model_executor.models.deepseek_ocr import (
DeepseekOCRForCausalLM,
DeepseekOCRImagePixelInputs,
)
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
from vllm.transformers_utils.processors.deepseek_ocr import (
DeepseekOCRProcessor,
ImageTransform,
)
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
@@ -155,3 +165,80 @@ class TestDeepseekOCRInputValidation:
def test_missing_images_raises_value_error(self, processor):
with pytest.raises(ValueError, match="prompt and images"):
processor(prompt="<image>\nDescribe this image.", images=None)
def _balanced_normalized_pixels(
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
) -> torch.Tensor:
"""Build a pixel tensor whose values cancel to an exact sum of 0.
Matches the normalized half-black / half-white case from the DeepSeek-OCR
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
"""
assert width % 2 == 0
pixel_values = torch.ones(batch, channels, height, width)
pixel_values[..., : width // 2] = -1.0
assert pixel_values.sum().item() == 0.0
return pixel_values
class TestDeepseekOCRZeroSumPixelsAccepted:
"""Zero-sum normalized pixels must still produce validated image inputs.
Returning None from ``_parse_and_validate_image_input`` makes
``embed_multimodal`` return None and trips the worker encoder-output
sanity check, killing EngineCore. The zero-sum shortcut is therefore
removed; balanced images must parse like any other valid tensor.
"""
def test_ocr_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_ocr2_accepts_balanced_normalized_pixels(self):
pixel_values = _balanced_normalized_pixels()
images_crop = torch.zeros(0, 3, 40, 40)
images_spatial_crop = torch.tensor([[1, 1]])
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
model,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert image_input.data is pixel_values
def test_image_transform_half_black_half_white_sums_to_zero(self):
"""A 1024×1024 half-black/half-white PNG through ImageTransform
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
must still accept it.
"""
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
pixel_values = ImageTransform()(image).unsqueeze(0)
assert pixel_values.sum().item() == 0.0
images_crop = torch.zeros(0, 3, 640, 640)
images_spatial_crop = torch.tensor([[1, 1]])
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
None,
pixel_values=pixel_values,
images_crop=images_crop,
images_spatial_crop=images_spatial_crop,
)
assert image_input is not None
assert torch.equal(image_input.data, pixel_values)
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
torch.testing.assert_close(
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
)
def _image_info(**kwargs):
ctx = build_model_context(
"zai-org/GLM-5.3-Flash",
limit_mm_per_prompt={"image": 1},
**kwargs,
)
return MULTIMODAL_REGISTRY.create_processor(
ctx.model_config,
tokenizer=ctx.tokenizer,
).info
def test_image_encoder_cache_covers_full_token_budget():
"""The most-features probe must reach the processor's token ceiling.
The inherited square probe refits to 89x89 = 7921 tokens under the
max_image_tokens=8000 budget, so the encoder cache came up short and
ordinary non-square images in 7922-8000 tokens were refused with
HTTP 400 (#59539).
"""
info = _image_info()
assert info.get_max_image_tokens() == 8000
# The shapes from the issue, with the token counts the processor
# actually produces; every one must fit the cache.
for width, height, expected_tokens in [
(4032, 3024, 7931), # phone photo, 4:3
(3840, 2160, 7973), # 4K frame, 16:9
(3508, 2480, 7950), # A4 at 300 dpi
(2600, 2400, 7998), # 13:12
(3000, 3000, 7921), # square worst case before the fix
]:
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
assert num_tokens == expected_tokens
assert num_tokens <= info.get_max_image_tokens()
# The profiling dummy covers the real worst case, not the square one.
size = info.get_image_size_with_most_features()
assert size.width * size.height == 2240 * 2800
def test_image_encoder_cache_follows_max_pixels_override():
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
assert info.get_max_image_tokens() == 100
size = info.get_image_size_with_most_features()
assert (
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
== 100
)
@@ -4,11 +4,8 @@ import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
from vllm.multimodal import MULTIMODAL_REGISTRY
from .transformers_backend import PROCESSOR_CLASSES, create_processor
AUDIO_MODEL_SETTINGS = {
"ibm-granite/granite-speech-3.3-2b": {
"prompt": (
@@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = {
}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize(
"model_id",
[
@@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = {
"zai-org/GLM-ASR-Nano-2512",
],
)
def test_audio_multimodal_processor(model_id, processor_cls):
def test_audio_multimodal_processor(model_id):
settings = AUDIO_MODEL_SETTINGS[model_id]
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
audio = np.zeros(16000, dtype=np.float32)
mm_data = {"audio": (audio, 16000)}
@@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("separator", [" and ", ""])
def test_audio_multiple_inputs(separator, processor_cls):
def test_audio_multiple_inputs(separator):
"""Multiple audios per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
model_id = "ibm-granite/granite-speech-3.3-2b"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
audio_token = mm_processor.info.get_hf_processor().audio_token
# One token per audio; the processor expands each to its placeholder run.
@@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls):
)
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
def process():
return mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
# The legacy path reads placeholders off contiguous runs of the audio token, so
# it cannot tell adjacent ones apart and says so instead of merging them
if processor_cls is LegacyMultiModalProcessor and not separator:
with pytest.raises(ValueError, match="Separate them in the prompt"):
process()
return
result = process()
result = mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
hf_processor_mm_kwargs={},
)
assert len(result["mm_placeholders"]["audio"]) == 2
assert len(result["mm_kwargs"]["audio"]) == 2
@@ -5,23 +5,16 @@ from unittest.mock import patch
import pytest
from vllm.assets.image import ImageAsset
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
OffsetsMultiModalProcessor,
)
from .transformers_backend import (
PROCESSOR_CLASSES,
create_cached_processor,
create_processor,
offsets_only,
)
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
def test_multimodal_processor(model_id, processor_cls):
mm_processor = create_processor(model_id, processor_cls)
def test_multimodal_processor(model_id):
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image_pil = ImageAsset("cherry_blossom").pil_image
mm_data = {"image": image_pil}
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
)
def _process_two_images(processor_cls, separator: str):
def _process_two_images(separator: str):
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
prompt = (
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_multiple_inputs(processor_cls):
def test_image_multiple_inputs():
"""Multiple images per prompt are each detected as a separate placeholder
and multi-modal item by the Transformers modelling backend."""
result = _process_two_images(processor_cls, separator="\n and ")
result = _process_two_images(separator="\n and ")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_image_adjacent_inputs(processor_cls):
def test_image_adjacent_inputs():
"""Adjacent images stay separate placeholders rather than merging into one."""
result = _process_two_images(processor_cls, separator="")
result = _process_two_images(separator="")
assert len(result["mm_placeholders"]["image"]) == 2
assert len(result["mm_kwargs"]["image"]) == 2
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_batch_padding_removed_from_image_items(processor_cls):
def test_batch_padding_removed_from_image_items():
"""Emu3 pads every image up to the largest in the batch, which would leave an
item's data dependent on what it was processed with and so uncacheable."""
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
)
image_token = mm_processor.info.get_hf_processor().image_token
images = [
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
assert len(shapes) == 2
def _process_one_gemma3_image(processor_cls):
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
hf_processor = mm_processor.info.get_hf_processor()
result = mm_processor(
prompt=f"{hf_processor.boi_token} What is this?",
def _process_one_gemma3_image():
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
)
boi_token = mm_processor.info.get_hf_processor().boi_token
return mm_processor(
prompt=f"{boi_token} What is this?",
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)
return hf_processor, result
@offsets_only
def test_non_embedding_tokens_excluded_from_placeholders():
"""Gemma3 wraps each image in text that carries no embeddings, which must be
inside the placeholder range but masked out of it."""
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
result = _process_one_gemma3_image()
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.is_embed is not None
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
def test_legacy_placeholders_hold_only_image_tokens():
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
marker in the unexpanded prompt, not the token the expansion repeats."""
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
(placeholder,) = result["mm_placeholders"]["image"]
assert placeholder.length == hf_processor.image_seq_length
prompt_ids = result["prompt_token_ids"]
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
assert set(covered) == {hf_processor.tokenizer.image_token_id}
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
def test_tokens_structuring_an_image_are_masked_not_dropped():
"""SmolVLM splits each image into tiles introduced by tokens carrying no
embeddings. Those belong inside the placeholder and masked out, because the token
count the processor reports is over the whole span. Idefics3 also refuses a prompt
holding `<image>` when no images are passed, which is how the offsets path has to
tokenize it before splicing in the expansion."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
holding `<image>` when no images are passed, which is how the prompt has to be
tokenized before splicing in the expansion."""
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<image>What is this?",
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
@offsets_only
def test_missing_replacement_offsets_names_the_processor():
"""A processor that reports no replacement offsets cannot be served, which must
be said plainly rather than surfacing later as a field config mismatch."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
hf_processor_cls = type(mm_processor.info.get_hf_processor())
hf_call = hf_processor_cls.__call__
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
)
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_text_only_prompt(processor_cls):
def test_text_only_prompt():
"""An image model still accepts a prompt with no images."""
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
mm_processor = create_processor(model_id, processor_cls)
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
result = mm_processor(
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
assert not result["mm_placeholders"]
@offsets_only
def test_repeated_image_hits_the_processor_cache():
"""Check that mm caching is actually working."""
mm_processor, cache = create_cached_processor(
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
)
model_config.multimodal_config.mm_processor_cache_gb = 4
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
cache = MultiModalProcessorOnlyCache(model_config)
image = ImageAsset("cherry_blossom").pil_image
def process():
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
assert first["mm_hashes"] == second["mm_hashes"]
@offsets_only
@pytest.mark.parametrize(
("model_id", "prompt"),
[
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
"<image> and more text",
marks=pytest.mark.xfail(
reason="Emu3Processor prepends its BOS token only when images are "
"passed, so the unexpanded prompt the offsets path tokenizes never "
"passed, so the unexpanded prompt vLLM tokenizes never "
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
strict=False,
),
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
],
)
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
"""The offsets path splices the expansion into a prompt tokenized without any
multi-modal data, so its token ids have to come out the same as the ones the HF
processor produces itself, which is what the legacy path returns."""
prompt_ids = []
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
mm_processor = create_processor(model_id, processor_cls)
prompt_ids.append(
mm_processor(
prompt=prompt,
mm_items=mm_processor.info.parse_mm_data(
{"image": ImageAsset("cherry_blossom").pil_image}
),
hf_processor_mm_kwargs={},
)["prompt_token_ids"]
)
"""The prompt is tokenized without any multi-modal data and the expansion spliced
in, so its token ids have to come out the same as the ones the HF processor
produces itself."""
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
info = mm_processor.info
image = ImageAsset("cherry_blossom").pil_image
legacy_ids, offsets_ids = prompt_ids
assert legacy_ids == offsets_ids
hf_processor = info.get_hf_processor()
prompt_ids = info.get_tokenizer().encode(
prompt, **info.default_tok_params.get_encode_kwargs()
)
hf_ids = info.ctx.call_hf_processor(
hf_processor,
dict(text=hf_processor.decode(prompt_ids), images=[image]),
dict(truncation=False, add_special_tokens=False),
)["input_ids"][0].tolist()
result = mm_processor(
prompt=prompt,
mm_items=info.parse_mm_data({"image": image}),
hf_processor_mm_kwargs={},
)
assert result["prompt_token_ids"] == hf_ids
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
def test_nested_image_fields_split_per_image(processor_cls):
def test_nested_image_fields_split_per_image():
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
belonging to each image one dimension further in. Slicing the batch dimension
instead handed the first image every row and the second an empty tensor."""
mm_processor = create_processor(
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
mm_processor = MULTIMODAL_REGISTRY.create_processor(
ModelConfig(model=model_id, model_impl="transformers")
)
image = ImageAsset("cherry_blossom").pil_image
result = mm_processor(
@@ -1,51 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helpers for testing both processors of the Transformers modelling backend."""
import pytest
from vllm.config import ModelConfig
from vllm.model_executor.models.transformers.multimodal import (
LegacyMultiModalProcessor,
MultiModalDummyInputsBuilder,
MultiModalProcessingInfo,
MultiModalProcessor,
OffsetsMultiModalProcessor,
)
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
from vllm.multimodal.processing import InputProcessingContext
from vllm.tokenizers.registry import cached_tokenizer_from_config
offsets_only = pytest.mark.skipif(
MultiModalProcessor is not OffsetsMultiModalProcessor,
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
)
PROCESSOR_CLASSES = [
pytest.param(LegacyMultiModalProcessor, id="legacy"),
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
]
def create_processor(model_id: str, processor_cls):
"""Build a processor directly, because the registry only ever builds the one the
installed transformers version selects, leaving the other path untested."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
return processor_cls(info, MultiModalDummyInputsBuilder(info))
def create_cached_processor(model_id: str, processor_cls):
"""Build a processor backed by a real multi-modal processor cache, and hand the
cache back so a test can check it was actually used."""
model_config = ModelConfig(model=model_id, model_impl="transformers")
model_config.multimodal_config.mm_processor_cache_gb = 4
ctx = InputProcessingContext(
model_config, cached_tokenizer_from_config(model_config)
)
info = MultiModalProcessingInfo(ctx)
cache = MultiModalProcessorOnlyCache(model_config)
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
+15 -81
View File
@@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"CohereLabs/North-Mini-Code",
trust_remote_code=True,
is_available_online=False,
min_transformers_version="5.9.0",
),
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm"),
# FIXME: databricks/dbrx-instruct has been deleted
"DbrxForCausalLM": _HfExamplesInfo(
"databricks/dbrx-instruct", is_available_online=False
@@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
),
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
"ExaoneMoeForCausalLM": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
),
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
@@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
"Gemma4ForCausalLM": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.0.0",
),
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
@@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
"zai-org/GLM-4.7-Flash",
min_transformers_version="5.0.0",
),
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
),
"GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False),
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
"GPTBigCodeForCausalLM": _HfExamplesInfo(
@@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
"ibm-research/moe-7b-1b-active-shared-experts"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
),
"GraniteSWAForCausalLM": _HfExamplesInfo(
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
),
"GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"),
"GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"),
"HrmTextForCausalLM": _HfExamplesInfo(
"sapientinc/HRM-Text-1B",
min_transformers_version="5.9.0",
),
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
@@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
min_transformers_version="5.9.0",
),
"InklingForCausalLM": _HfExamplesInfo(
"thinkingmachines/Inkling-NVFP4",
@@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"IQuestLoopCoderForCausalLM": _HfExamplesInfo(
"IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
),
"Jais2ForCausalLM": _HfExamplesInfo(
"inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
),
"Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"),
"JambaForCausalLM": _HfExamplesInfo(
"ai21labs/AI21-Jamba-1.5-Mini",
extras={
@@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
"Lfm2MoeForCausalLM": _HfExamplesInfo(
"LiquidAI/LFM2-8B-A1B",
min_transformers_version="5.0.0",
use_original_num_layers=True,
# Initialize at least one MoE layer
hf_overrides={"num_hidden_layers": 4},
@@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
),
"MiniCPM4ForCausalLM": _HfExamplesInfo(
"openbmb/MiniCPM4.1-8B",
min_transformers_version="4.56",
max_transformers_version="4.57",
transformers_version_reason={
"hf": "HF remote code imports removed `is_torch_fx_available`; "
@@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"Qwen3NextForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct",
extras={"tiny-random": "tiny-random/qwen3-next-moe"},
min_transformers_version="4.56.3",
),
"Rnj1ForCausalLM": _HfExamplesInfo(
"EssentialAI/rnj-1-instruct",
@@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
),
"OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo(
"openai/privacy-filter",
min_transformers_version="5.6.0.dev0",
),
"RobertaForTokenClassification": _HfExamplesInfo(
"Jean-Baptiste/roberta-large-ner-english"
@@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
"AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/audio-flamingo-3-hf",
min_transformers_version="5.3.0",
transformers_version_reason={
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
},
@@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025",
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
min_transformers_version="5.9.0",
),
"CohereCompassForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/North-Micro-Vision-Instruct",
min_transformers_version="5.16.0",
),
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Nano",
extras={"super": "nvidia/Cosmos3-Super"},
max_model_len=4096,
min_transformers_version="4.57",
),
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
"nvidia/Cosmos3-Edge",
max_model_len=4096,
min_transformers_version="5.15",
use_original_num_layers=True,
hf_overrides={
"text_config": {
@@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
"LGAI-EXAONE/EXAONE-4.5-33B",
min_transformers_version="5.6.0",
),
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
"allendou/FireRedASR2-LLM-vllm",
@@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-E2B-it",
min_transformers_version="5.5.0",
),
# TODO: update min_transformers_version when Gemma4 Unified lands in
# a stable transformers release.
"Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
"google/gemma-4-12B-it",
min_transformers_version="5.8.0",
is_available_online=False,
),
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-ASR-Nano-2512",
min_transformers_version="5.0.0",
),
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-vision-4.1-4b",
@@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
"ibm-granite/granite-speech-4.1-2b-plus",
min_transformers_version="5.8.0",
),
"GLM4VForCausalLM": _HfExamplesInfo(
"zai-org/glm-4v-9b",
@@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
"zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"H2OVLChatModel": _HfExamplesInfo(
@@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
"tencent/HunyuanOCR",
min_transformers_version="5.13",
),
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
"HuggingFaceM4/Idefics3-8B-Llama3",
@@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"Lfm2VlForConditionalGeneration": _HfExamplesInfo(
"LiquidAI/LFM2-VL-450M",
min_transformers_version="5.0.0",
),
"Llama4ForConditionalGeneration": _HfExamplesInfo(
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
@@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
"openbmb/MiniCPM-V-4_6",
min_transformers_version="5.7.0",
),
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
"MiniMaxAI/MiniMax-M3",
@@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
"allenai/Molmo2-8B",
extras={"olmo": "allenai/Molmo2-O-7B"},
min_transformers_version="4.51",
trust_remote_code=True,
# required by current PrefixLM implementation
max_num_batched_tokens=31872,
@@ -1296,19 +1260,15 @@ _MULTIMODAL_EXAMPLE_MODELS = {
trust_remote_code=True,
max_transformers_version="4.57",
transformers_version_reason={
"vllm": "Custom processor code is not compatible with Transformers v5."
"hf": "HF remote code reads `is_parallelizable`, which was removed "
"in Transformers v5."
},
),
"Ovis2_6ForCausalLM": _HfExamplesInfo(
"AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
),
"Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
"AIDC-AI/Ovis2.6-30B-A3B",
trust_remote_code=True,
max_transformers_version="4.57",
transformers_version_reason={
"vllm": "Custom processor code is not compatible with Transformers v5."
},
"AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
),
"PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
"PaddlePaddle/PaddleOCR-VL",
@@ -1353,7 +1313,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
"baidu/Qianfan-OCR",
min_transformers_version="5.6.0",
),
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen2-Audio-7B-Instruct"
@@ -1368,12 +1327,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen3VLForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-VL-4B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-VL-30B-A3B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3.5-0.8B",
@@ -1391,23 +1348,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
),
"Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
),
"Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
"Qwen/Qwen3-ForcedAligner-0.6B",
max_model_len=4096,
min_transformers_version="4.57",
hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
),
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
@@ -1456,7 +1409,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
),
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
"microsoft/VibeVoice-ASR-HF",
min_transformers_version="5.13.0",
),
"VoxtralForConditionalGeneration": _HfExamplesInfo(
"mistralai/Voxtral-Mini-3B-2507",
@@ -1530,7 +1482,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
use_original_num_layers=True, # DFlash requires all layers
max_model_len=8192, # Reduce for CI
max_num_seqs=32,
min_transformers_version="4.56.3", # Required for Qwen3Next
),
"MuseGlimmerAssistantModel": _HfExamplesInfo(
"meta-models/Muse-Glimmer-30B",
@@ -1576,7 +1527,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
is_available_online=False,
use_original_num_layers=True,
min_transformers_version="4.57",
),
"Gemma4DSparkModel": _HfExamplesInfo(
"google/gemma-4-12B-it",
@@ -1737,7 +1687,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"google/gemma-4-E4B-it",
speculative_model="google/gemma-4-E4B-it-assistant",
trust_remote_code=True,
min_transformers_version="5.8.0",
),
"ErnieMTPModel": _HfExamplesInfo(
"baidu/ERNIE-4.5-21B-A3B-PT",
@@ -1747,13 +1696,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"ExaoneMoeMTP": _HfExamplesInfo(
"LGAI-EXAONE/K-EXAONE-236B-A23B",
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
min_transformers_version="5.1.0",
enable_prefix_caching=False,
),
"Exaone4_5_MTP": _HfExamplesInfo(
"LGAI-EXAONE/EXAONE-4.5-33B",
speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
min_transformers_version="5.6.0",
),
"ExtractHiddenStatesModel": _HfExamplesInfo(
"Qwen/Qwen3-8B",
@@ -1766,12 +1713,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
"zai-org/GLM-4.7-Flash",
speculative_model="zai-org/GLM-4.7-Flash",
min_transformers_version="5.0.0",
),
"GlmOcrMTPModel": _HfExamplesInfo(
"zai-org/GLM-OCR",
speculative_model="zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextMTPModel": _HfExamplesInfo(
"zai-org/GLM-5.3-Flash",
@@ -1842,9 +1787,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
trust_remote_code=True,
is_available_online=False,
),
"Qwen3NextMTP": _HfExamplesInfo(
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
),
"Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"),
"Qwen3_5MTP": _HfExamplesInfo(
"Qwen/Qwen3.5-0.8B",
speculative_model="Qwen/Qwen3.5-0.8B",
@@ -1865,29 +1808,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
}
_TRANSFORMERS_BACKEND_MODELS = {
"TransformersEmbeddingModel": _HfExamplesInfo(
"BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
),
"TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
"TransformersForSequenceClassification": _HfExamplesInfo(
"papluca/xlm-roberta-base-language-detection",
min_transformers_version="5.0.0",
),
"TransformersForCausalLM": _HfExamplesInfo(
"hmellor/Ilama-3.2-1B", trust_remote_code=True
),
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
"TransformersMoEForCausalLM": _HfExamplesInfo(
"allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
),
"TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"),
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
),
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
),
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
"Qwen/Qwen3-VL-30B-A3B-Instruct"
),
"TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
"TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
"google/gemma-3-4b-it"
+313 -195
View File
@@ -1,116 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the flashinfer moe_ep backend plumbing.
Everything here runs without a GPU or a flashinfer install: the flashinfer
modules the helpers import lazily are replaced with capture fakes.
"""
import sys
from dataclasses import dataclass, field
from types import ModuleType, SimpleNamespace
from typing import Any
from dataclasses import replace
from types import SimpleNamespace
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.config.kernel import (
FLASHINFER_MOE_EP_BACKENDS,
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
MEGA_MOE_BACKENDS,
NATIVE_MEGA_MOE_BACKENDS,
PASSTHROUGH_ALL2ALL_BACKEND,
bind_passthrough_all2all_backend,
validate_flashinfer_moe_ep_model,
)
from vllm.utils.flashinfer_moe_ep import (
_E2M1_LUT,
FI_MOE_EP_BACKEND_SPECS,
_dequant_fp4_ue8m0_gran32,
build_fi_mega_config,
fi_moe_ep_backend_spec,
make_fi_moe_ep_bootstrap,
megakernel_runtime_requirements,
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
epilogue_from_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
make_mxfp4_moe_quant_config,
map_mxfp4_backend,
mxfp4_round_up_hidden_size_and_intermediate_size,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
backend_to_kernel_cls as mxfp4_backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
NvFp4MoeBackend,
convert_to_nvfp4_moe_kernel_format,
map_nvfp4_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
backend_to_kernel_cls as nvfp4_backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
PassThroughPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Static,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
@dataclass
class _FakeBootstrapConfig:
world_size: int
rank: int
process_group: Any = None
auto_bootstrap: bool = True
device: int | None = field(default=None, kw_only=True)
def test_flashinfer_backends_are_megakernels_outside_the_native_model_path():
assert FLASHINFER_MOE_EP_BACKENDS <= MEGA_MOE_BACKENDS
assert FLASHINFER_MOE_EP_BACKENDS.isdisjoint(NATIVE_MEGA_MOE_BACKENDS)
@dataclass
class _FakeDeepGemmMegaMoeConfig:
intermediate_size: int
top_k: int
activation_clamp: float | None
fast_math: bool
@dataclass
class _FakeNvfp4CutedslMegaMoeConfig:
intermediate_size: int
top_k: int
activation_clamp: float | None
fast_math: bool
@dataclass
class _FakeMegaConfig:
megakernel: Any
preprocess_weights: bool
quantize_input: bool
@pytest.fixture
def fake_flashinfer(monkeypatch):
"""Install a minimal fake flashinfer.moe_ep for the lazy imports."""
moe_ep = ModuleType("flashinfer.moe_ep")
core = ModuleType("flashinfer.moe_ep.core")
runtime = ModuleType("flashinfer.moe_ep.core.runtime")
flashinfer = ModuleType("flashinfer")
fake_attrs: dict[ModuleType, dict[str, Any]] = {
moe_ep: {
"BootstrapConfig": _FakeBootstrapConfig,
"DeepGemmMegaMoeConfig": _FakeDeepGemmMegaMoeConfig,
"Nvfp4CutedslMegaMoeConfig": _FakeNvfp4CutedslMegaMoeConfig,
"MegaConfig": _FakeMegaConfig,
"core": core,
},
runtime: {"TORCH_DIST": "torch_dist", "NVSHMEM": "nvshmem"},
flashinfer: {"moe_ep": moe_ep},
core: {"runtime": runtime},
}
for mod, attrs in fake_attrs.items():
for attr, value in attrs.items():
setattr(mod, attr, value)
for name, mod in {
"flashinfer": flashinfer,
"flashinfer.moe_ep": moe_ep,
"flashinfer.moe_ep.core": core,
"flashinfer.moe_ep.core.runtime": runtime,
}.items():
monkeypatch.setitem(sys.modules, name, mod)
return moe_ep
def test_fi_backend_strings_are_registered_mega_moe_backends():
assert set(FI_MOE_EP_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
assert FLASHINFER_MOE_EP_BACKENDS < MEGA_MOE_BACKENDS
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
instead of silently falling through to the generic FusedMoE path."""
def test_only_deep_gemm_backend_is_dsv4_specific():
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_CUTEDSL,
["MixtralForCausalLM"],
)
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
validate_flashinfer_moe_ep_model(moe_backend, ["DeepseekV4ForCausalLM"])
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_DEEP_GEMM,
["MixtralForCausalLM"],
)
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_DEEP_GEMM,
["DeepseekV4ForCausalLM"],
)
@pytest.mark.parametrize(
@@ -118,8 +92,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
)
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
models validate their own constraints at construction time."""
"""Models validate native deep_gemm mega constraints at construction time."""
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
@@ -128,11 +101,7 @@ def test_non_fi_backend_ignores_architectures():
@pytest.mark.parametrize("moe_backend", sorted(MEGA_MOE_BACKENDS))
def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
"""Every mega backend must qualify for sequence-parallel MoE at
TP>1/EP: the predicate once matched only the native backend string,
which silently ran the fi backends full-batch with an all-reduce on
every rank — 0.42-0.65x native e2e at TP8."""
def test_token_sharding_backends_enable_dsv4_sequence_parallel(moe_backend: str):
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
vllm_config = SimpleNamespace(
@@ -147,113 +116,262 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
assert _use_sequence_parallel(vllm_config)
def test_fi_moe_ep_backend_spec_kernel_and_nvshmem_contract():
dg = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
assert dg.megakernel == "deep_gemm_mega"
assert not dg.needs_nvshmem
cd = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
assert cd.megakernel == "nvfp4_cutedsl"
assert cd.needs_nvshmem
with pytest.raises(ValueError, match="not a flashinfer moe_ep backend"):
fi_moe_ep_backend_spec("deep_gemm_mega_moe")
def test_dsv4_requests_pre_fc2_router_weight_placement():
dsv4 = SimpleNamespace(routing_method=RoutingMethodType.DeepseekV4)
default = SimpleNamespace(routing_method=RoutingMethodType.Default)
assert fi_ep.apply_topk_in_fc1(dsv4)
assert not fi_ep.apply_topk_in_fc1(default)
def test_megakernel_runtime_requirements(fake_flashinfer):
dg = megakernel_runtime_requirements(
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
def test_backend_specs_preserve_weight_format_contracts():
cutedsl = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_CUTEDSL)
assert cutedsl.kernel == "cutedsl"
assert cutedsl.weight_formats == frozenset({"nvfp4", "mxfp4"})
deep_gemm = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_DEEP_GEMM)
assert deep_gemm.kernel == "deep_gemm"
assert deep_gemm.weight_formats == frozenset({"mxfp4"})
def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
fields = dict(
moe_backend=moe_backend,
moe_parallel_config=SimpleNamespace(
use_ep=True, use_batched_activation_format=False
),
is_act_and_mul=True,
activation=MoEActivation.SILU,
has_hash_routing=False,
routing_method=RoutingMethodType.DeepseekV4,
router_logits_dtype=torch.float32,
hidden_dim=256,
is_lora_enabled=False,
skip_final_all_reduce=False,
in_dtype=torch.bfloat16,
has_bias=False,
swiglu_alpha=None,
swiglu_beta=None,
)
assert dg == frozenset({"torch_dist"})
fields.update(overrides)
return SimpleNamespace(**fields)
cd = megakernel_runtime_requirements(
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
def _is_supported(moe, weight_key, activation_key) -> tuple[bool, str]:
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
weight_key,
activation_key,
mk.FusedMoEActivationFormat.Standard,
)
assert cd == frozenset({"torch_dist", "nvshmem"})
return supported, reason or ""
def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
device it already bound via BootstrapConfig.device."""
import vllm.utils.flashinfer_moe_ep as mod
pg = object()
@pytest.fixture
def megakernel_host(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
"""A vLLM config without DBO/EPLB/weight transfer, on a supported device."""
config = SimpleNamespace(
weight_transfer_config=None,
parallel_config=SimpleNamespace(enable_dbo=False, enable_eplb=False),
)
monkeypatch.setattr(fi_ep, "get_current_vllm_config", lambda: config)
monkeypatch.setattr(
mod,
"get_ep_group",
lambda: SimpleNamespace(world_size=4, rank_in_group=2, device_group=pg),
FlashInferMoeEpExperts, "_supports_current_device", staticmethod(lambda: True)
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 3)
bootstrap = make_fi_moe_ep_bootstrap()
assert bootstrap.world_size == 4
assert bootstrap.rank == 2
assert bootstrap.process_group is pg
assert bootstrap.auto_bootstrap is False
assert bootstrap.device == 3
return config
def test_build_fi_mega_config_selects_kernel_config(fake_flashinfer):
dg = build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=7.0,
megakernel="deep_gemm_mega",
def test_megakernel_requires_expert_parallel(megakernel_host):
"""The oracle rejects the megakernel through the experts' support check."""
moe = _megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL)
assert _is_supported(moe, kNvfp4Static, kNvfp4Dynamic) == (True, "")
moe.moe_parallel_config.use_ep = False
supported, reason = _is_supported(moe, kNvfp4Static, kNvfp4Dynamic)
assert not supported and "parallel config" in reason
def test_megakernel_reports_its_own_constraints(megakernel_host):
"""Constraints the framework does not know about surface as reasons."""
_, reason = _is_supported(
_megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL), kNvfp4Static, None
)
assert isinstance(dg.megakernel, _FakeDeepGemmMegaMoeConfig)
assert dg.megakernel.intermediate_size == 2048
assert dg.megakernel.top_k == 8
assert dg.megakernel.activation_clamp == 7.0
assert dg.preprocess_weights and dg.quantize_input
assert "A16 activations" in reason
cd = build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=None,
megakernel="nvfp4_cutedsl",
deep_gemm = _megakernel_moe(
FLASHINFER_MOE_EP_DEEP_GEMM, routing_method=RoutingMethodType.Renormalize
)
assert isinstance(cd.megakernel, _FakeNvfp4CutedslMegaMoeConfig)
_, reason = _is_supported(deep_gemm, kMxfp4Static, None)
assert "routing method" in reason
with pytest.raises(ValueError, match="Unsupported fi_moe_ep megakernel"):
build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=None,
megakernel="deep_gemm",
megakernel_host.parallel_config.enable_eplb = True
_, reason = _is_supported(
_megakernel_moe(FLASHINFER_MOE_EP_DEEP_GEMM), kMxfp4Static, None
)
assert reason.endswith("EPLB")
def test_megakernels_are_oracle_backends():
"""Quant methods reach the megakernel through the NVFP4/MXFP4 oracles."""
nvfp4 = map_nvfp4_backend(FLASHINFER_MOE_EP_CUTEDSL)
assert nvfp4 is NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
assert nvfp4_backend_to_kernel_cls(nvfp4) == [FlashInferMoeEpExperts]
assert map_mxfp4_backend(FLASHINFER_MOE_EP_DEEP_GEMM) == [
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
]
for backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
assert mxfp4_backend_to_kernel_cls(backend) == [FlashInferMoeEpExperts]
# The megakernel takes the model's own dimensions.
assert mxfp4_round_up_hidden_size_and_intermediate_size(
backend, 2880, 2880
) == (2880, 2880)
quant_config = make_mxfp4_moe_quant_config(
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL, w1_scale=None, w2_scale=None
)
assert quant_config is not None and quant_config.use_nvfp4_w4a4
def test_megakernel_is_a_modular_kernel_with_pass_through_stages():
"""The megakernel plugs into ``FusedMoEKernel`` like any other experts impl.
Routing stays outside (top-k ids in), dispatch and combine happen inside,
so prepare passes tokens through untouched and finalize has nothing to do.
"""
moe = make_dummy_moe_config(
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
)
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
kernel = mk.FusedMoEKernel(
PassThroughPrepareAndFinalize(),
FlashInferMoeEpExperts(moe, quant_config),
)
assert not kernel.is_monolithic
assert kernel.prepare_finalize.output_is_reduced()
# Generic stages: the int32 top-k ids the megakernel wants are its concern.
assert kernel.prepare_finalize.topk_indices_dtype() is None
assert kernel.fused_experts.expects_unquantized_inputs
assert isinstance(
kernel.fused_experts.finalize_weight_and_reduce_impl(), TopKWeightAndReduceNoOP
)
assert kernel.fused_experts.workspace_shapes(
16, 256, 256, 2, 8, 8, None, MoEActivation.SILU
) == ((0,), (0,), (16, 256))
tokens = torch.zeros(4, 256, dtype=torch.bfloat16)
prepared = kernel.prepare_finalize.prepare(
tokens,
torch.ones(4, 2),
torch.zeros(4, 2, dtype=torch.int32),
8,
None,
False,
quant_config,
)
assert prepared[0] is tokens and all(item is None for item in prepared[1:])
def test_epilogue_takes_the_weight_global_scales_from_the_quant_config():
"""The canonical ``g1/g2_alphas`` become the per-expert fc1/fc2 alphas and
keep aliasing the layer's tensors, so EPLB permutations reach the kernel."""
g1_alphas = torch.tensor([1.0, 2.0])
g2_alphas = torch.tensor([3.0, 4.0])
quant_config = nvfp4_moe_quant_config(
g1_alphas=g1_alphas,
g2_alphas=g2_alphas,
a1_gscale=torch.ones(2),
a2_gscale=torch.ones(2),
w1_scale=torch.ones(2, 4, 1),
w2_scale=torch.ones(2, 2, 1),
)
epilogue = epilogue_from_quant_config(quant_config)
assert epilogue.fc1_alpha is not None and epilogue.fc2_alpha is not None
assert epilogue.fc1_alpha.data_ptr() == g1_alphas.data_ptr()
assert epilogue.fc2_alpha.data_ptr() == g2_alphas.data_ptr()
# Activations are quantized dynamically: no norm constants.
assert epilogue.input_norm_const == 1.0 and epilogue.fc1_norm_const is None
def test_epilogue_rejects_unloaded_weight_global_scales():
quant_config = nvfp4_moe_quant_config(
g1_alphas=torch.tensor([1.0, float("nan")]),
g2_alphas=torch.ones(2),
a1_gscale=torch.ones(2),
a2_gscale=torch.ones(2),
w1_scale=torch.ones(2, 4, 1),
w2_scale=torch.ones(2, 2, 1),
)
with pytest.raises(ValueError, match="g1_alphas"):
epilogue_from_quant_config(quant_config)
def test_mxfp4_checkpoints_keep_the_default_epilogue():
"""MXFP4 has no per-tensor global scales; the megakernel needs no alphas."""
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
assert epilogue_from_quant_config(quant_config) == fi_ep.FlashInferMoeEpEpilogue()
def test_nvfp4_kernel_format_passes_megakernel_weights_through():
"""The megakernel takes the canonical tensors as they are; the oracle must
not reject or reshape them."""
tensors = tuple(torch.arange(4.0).reshape(2, 2) + i for i in range(8))
converted = convert_to_nvfp4_moe_kernel_format(
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
SimpleNamespace(),
*tensors,
is_act_and_mul=True,
)
assert all(out is src for out, src in zip(converted, tensors))
def test_passthrough_all2all_backend_selects_the_pass_through_stages():
"""``all2all_backend="passthrough"`` is how the oracles get the
pass-through prepare/finalize."""
moe = make_dummy_moe_config(
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
)
moe.moe_parallel_config = replace(
moe.moe_parallel_config, all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND
)
assert not moe.moe_parallel_config.use_all2all_kernels
prepare_finalize = maybe_make_prepare_finalize(
moe, FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
)
assert isinstance(prepare_finalize, PassThroughPrepareAndFinalize)
def test_passthrough_all2all_backend_is_not_a_cli_choice():
"""Users cannot pick the passthrough all2all backend explicitly."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
with pytest.raises(SystemExit):
parser.parse_args(["--all2all-backend", PASSTHROUGH_ALL2ALL_BACKEND])
parser.parse_args(["--all2all-backend", "deepep_low_latency"])
def test_megakernel_moe_backend_binds_the_all2all_backend():
"""A megakernel deployment gets its all2all backend by default."""
parallel = SimpleNamespace(all2all_backend="allgather_reducescatter")
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
assert parallel.all2all_backend == PASSTHROUGH_ALL2ALL_BACKEND
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
with pytest.raises(ValueError, match="cannot be paired"):
bind_passthrough_all2all_backend(
FLASHINFER_MOE_EP_DEEP_GEMM,
SimpleNamespace(all2all_backend="deepep_low_latency"),
)
with pytest.raises(ValueError, match="only valid with"):
bind_passthrough_all2all_backend(
"flashinfer_trtllm",
SimpleNamespace(all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND),
)
def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
from vllm.models.deepseek_v4.nvidia.fi_moe import ckpt_uses_nvfp4_experts
nvfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo="NVFP4"))
assert ckpt_uses_nvfp4_experts(nvfp4)
mxfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo=None))
assert not ckpt_uses_nvfp4_experts(mxfp4)
no_algo = SimpleNamespace(quant_config=SimpleNamespace())
assert not ckpt_uses_nvfp4_experts(no_algo)
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
"""One 32-element scale group per row: low nibble is the even element,
high nibble the odd one, ue8m0 scale applies to the whole group."""
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
out = _dequant_fp4_ue8m0_gran32(packed, sf)
assert out.shape == (2, 32)
assert out.dtype == torch.bfloat16
expected = torch.empty(2, 32)
for row in range(2):
for col in range(16):
byte = int(packed[row, col])
expected[row, 2 * col] = _E2M1_LUT[byte & 0x0F]
expected[row, 2 * col + 1] = _E2M1_LUT[byte >> 4]
expected[row] *= 2.0**row
assert torch.equal(out, expected.to(torch.bfloat16))
untouched = SimpleNamespace(all2all_backend="deepep_low_latency")
bind_passthrough_all2all_backend("flashinfer_trtllm", untouched)
assert untouched.all2all_backend == "deepep_low_latency"
+1 -1
View File
@@ -657,7 +657,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.use_mega_moe = True
moe.use_native_mega_moe = True
moe.gate = FakeGate()
moe.experts = FakeExperts()
moe.shared_experts = FakeSharedExperts()
+1 -1
View File
@@ -241,7 +241,7 @@ def test_v41_dspark_loads_linear_scales(
linear_scale_name=scale_name,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
confidence_head=None,
),
named_parameters=lambda: [(runtime_name, param)],
-11
View File
@@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
import transformers
from packaging.version import Version
installed = Version(transformers.__version__)
required = Version("5.15.0.dev0")
if installed < required:
pytest.skip(
"MLA models with the Transformers modeling backend require "
f"transformers>={required}, but got {installed}"
)
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
args = (example_prompts, 32, 5)
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
-14
View File
@@ -609,17 +609,3 @@ def dummy_hf_overrides(
)
return hf_config
def check_transformers_version(
model: str,
min_transformers_version: str | None = None,
max_transformers_version: str | None = None,
):
from .registry import _HfExamplesInfo
return _HfExamplesInfo(
model,
min_transformers_version=min_transformers_version,
max_transformers_version=max_transformers_version,
).check_transformers_version(on_fail="skip")