Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -8,10 +8,6 @@ import torch
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA-only dispatch"
|
||||
)
|
||||
|
||||
|
||||
def _require_deep_gemm() -> None:
|
||||
from vllm.utils.deep_gemm import has_deep_gemm
|
||||
@@ -20,17 +16,10 @@ def _require_deep_gemm() -> None:
|
||||
pytest.skip("kpool indexer requires DeepGEMM")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
def _build(indexer_cls, backend: str):
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
op = SparseAttnIndexerKpool(
|
||||
return indexer_cls(
|
||||
k_cache=None,
|
||||
quant_block_size=128,
|
||||
scale_fmt="ue8m0",
|
||||
@@ -40,4 +29,73 @@ def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> N
|
||||
max_total_seq_len=8192,
|
||||
topk_indices_buffer=torch.empty(8, 2176, dtype=torch.int32, device="cuda"),
|
||||
)
|
||||
assert op.topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "persistent", "cooperative", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend(backend: str) -> None:
|
||||
"""The kpool indexer must read kernel_config.sparse_indexer_topk_backend
|
||||
and hand it to the shared SparseIndexerTopk dispatcher, rather than
|
||||
hard-coding its own cooperative/persistent/per_row choice."""
|
||||
_require_deep_gemm()
|
||||
from vllm.models.glm5next.nvidia.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize("backend", ["auto", "aiter", "per_row", "torch"])
|
||||
def test_kpool_indexer_dispatches_through_shared_topk_backend_rocm(
|
||||
backend: str,
|
||||
) -> None:
|
||||
"""The AMD kpool indexer must go through the same dispatcher, so the
|
||||
AITER decode top-k is reachable from kernel_config."""
|
||||
from vllm.models.glm5next.amd.sparse_indexer import SparseAttnIndexerKpool
|
||||
|
||||
assert _build(SparseAttnIndexerKpool, backend).topk_backend == backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only dispatch")
|
||||
@pytest.mark.parametrize(
|
||||
"configured,full_cudagraph,expected",
|
||||
[
|
||||
("auto", False, "aiter"),
|
||||
("auto", True, "auto"),
|
||||
("aiter", True, "aiter"),
|
||||
("per_row", False, "per_row"),
|
||||
],
|
||||
)
|
||||
def test_kpool_decode_topk_backend_never_auto_selects_aiter_under_full_cudagraph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
configured: str,
|
||||
full_cudagraph: bool,
|
||||
expected: str,
|
||||
) -> None:
|
||||
"""A FULL cudagraph bakes in one kernel for every replay, so "auto" cannot
|
||||
use the context length to pick AITER and must stay on the in-tree kernel.
|
||||
Explicit backends keep working, and outside FULL the long-context heuristic
|
||||
still narrows to AITER."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.models.glm5next.amd import sparse_indexer
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: True, raising=False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_indexer_top_k_supported",
|
||||
lambda **kwargs: True,
|
||||
raising=False,
|
||||
)
|
||||
|
||||
assert (
|
||||
sparse_indexer._kpool_decode_topk_backend(
|
||||
configured,
|
||||
num_rows=8,
|
||||
max_valid_seq_len=64 * 1024,
|
||||
select_k=512,
|
||||
index_kpool=4,
|
||||
full_cudagraph=full_cudagraph,
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import (
|
||||
_moe_weight_override_is_int4,
|
||||
_use_k3_situ_int4_gfx942,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _FakeRocmModule(ModuleType):
|
||||
on_gfx942: Callable[[], bool]
|
||||
|
||||
|
||||
class _FakeAiterOpsModule(ModuleType):
|
||||
rocm_aiter_ops: SimpleNamespace
|
||||
|
||||
|
||||
def _vllm_config(moe_weight: str | None):
|
||||
quantization_config = (
|
||||
None
|
||||
if moe_weight is None
|
||||
else QuantizationConfigArgs(moe={"weight": moe_weight})
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(quantization_config=quantization_config)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("moe_weight", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
("mxfp4", False),
|
||||
("int4_per_group_32", True),
|
||||
],
|
||||
)
|
||||
def test_moe_weight_override_is_int4(monkeypatch, moe_weight, expected):
|
||||
monkeypatch.setattr(
|
||||
"vllm.config.get_current_vllm_config",
|
||||
lambda: _vllm_config(moe_weight),
|
||||
)
|
||||
assert _moe_weight_override_is_int4() is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("int4_requested", [False, True])
|
||||
def test_k3_situ_int4_gfx942_requires_explicit_override(int4_requested):
|
||||
moe = SimpleNamespace(
|
||||
activation=MoEActivation.SITU,
|
||||
activation_situ_linear_beta=25.0,
|
||||
)
|
||||
fake_rocm = _FakeRocmModule("vllm.platforms.rocm")
|
||||
fake_rocm.on_gfx942 = lambda: True
|
||||
fake_aiter_ops = _FakeAiterOpsModule("vllm._aiter_ops")
|
||||
fake_aiter_ops.rocm_aiter_ops = SimpleNamespace(is_fused_moe_enabled=lambda: True)
|
||||
with (
|
||||
patch("vllm.platforms.current_platform.is_rocm", return_value=True),
|
||||
patch.dict(
|
||||
sys.modules,
|
||||
{
|
||||
"vllm.platforms.rocm": fake_rocm,
|
||||
"vllm._aiter_ops": fake_aiter_ops,
|
||||
},
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.mxfp4."
|
||||
"_moe_weight_override_is_int4",
|
||||
return_value=int4_requested,
|
||||
),
|
||||
):
|
||||
assert _use_k3_situ_int4_gfx942(moe) is int4_requested
|
||||
|
||||
|
||||
def test_int4_per_group_32_is_not_an_online_moe_method():
|
||||
# Load-time gfx942 requant. Must not match the online MoE table.
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "int4_per_group_32"})
|
||||
)
|
||||
assert (
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_unknown_online_moe_weight_still_raises():
|
||||
config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(moe={"weight": "fp8_per_token"})
|
||||
)
|
||||
with pytest.raises(ValueError, match="online quantization"):
|
||||
config.resolve_quant_method_cls(
|
||||
Mock(spec=RoutedExperts), "model.layers.0.mlp.experts"
|
||||
)
|
||||
@@ -0,0 +1,375 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Kimi-K3 multi-turn prefix cache reuse with KV offload, P/D and DCP."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import random
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from math import lcm
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
from prometheus_client.parser import text_string_to_metric_families
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from tests.utils import RemoteOpenAIServer, multi_gpu_marks
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
# Official Kimi-K3 config with 8 layers and no weights.
|
||||
MODEL = "riverclouds/Kimi-K3-8L-dummy"
|
||||
# The recipe's DSpark draft, reading target layers that exist in MODEL.
|
||||
DRAFT = "riverclouds/Kimi-K3-DSpark-for-8L-dummy"
|
||||
|
||||
# Conversation: the first prompt spans FIRST_PROMPT_BLOCKS blocks plus an
|
||||
# unaligned tail, and each later turn appends TURN_TOKENS random tokens.
|
||||
NUM_TURNS = 3
|
||||
FIRST_PROMPT_BLOCKS = 3
|
||||
UNALIGNED_TAIL = 37
|
||||
TURN_TOKENS = 777
|
||||
MAX_TOKENS = 8
|
||||
NUM_LOGPROBS = 5
|
||||
STARTUP_TIMEOUT = 1200
|
||||
|
||||
BASE_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--language-model-only",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--no-enable-flashinfer-autotune",
|
||||
"--enable-prompt-tokens-details",
|
||||
# Default max_num_seqs OOMs one GPU during warmup.
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Kimi-K3 NVIDIA kernels require the SM100 family",
|
||||
)
|
||||
|
||||
SPEC_CONFIG = {
|
||||
"method": "dspark",
|
||||
"model": DRAFT,
|
||||
"num_speculative_tokens": 7,
|
||||
"draft_sample_method": "probabilistic",
|
||||
"rejection_sample_method": "block",
|
||||
}
|
||||
|
||||
NIXL = {"kv_connector": "NixlConnector", "kv_role": "kv_both"}
|
||||
# SimpleCPUOffloadConnector as in the Kimi-K3 recipe.
|
||||
OFFLOAD = {
|
||||
"kv_connector": "SimpleCPUOffloadConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {
|
||||
"cpu_bytes_to_use_per_rank": 32 << 30,
|
||||
"lazy_offload": False,
|
||||
},
|
||||
}
|
||||
NIXL_OFFLOAD = {
|
||||
"kv_connector": "MultiConnector",
|
||||
"kv_role": "kv_both",
|
||||
"kv_connector_extra_config": {"connectors": [NIXL, OFFLOAD]},
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Mode:
|
||||
# A finer prefix_match_unit enables partial hits inside a Mamba block.
|
||||
prefix_match_unit: int | None
|
||||
spec: bool
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
name = "block" if self.prefix_match_unit is None else "partial"
|
||||
return f"{name}-dspark" if self.spec else name
|
||||
|
||||
|
||||
MODES = [Mode(unit, spec) for spec in (False, True) for unit in (None, 128)]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Instance:
|
||||
tp: int = 1
|
||||
dp: int = 1
|
||||
dcp: int = 1
|
||||
kv_config: dict | None = None
|
||||
prefix_caching: bool = True
|
||||
|
||||
@property
|
||||
def num_gpus(self) -> int:
|
||||
return self.tp * self.dp
|
||||
|
||||
def args(self, mode: Mode) -> list[str]:
|
||||
args = BASE_ARGS + ["--tensor-parallel-size", str(self.tp)]
|
||||
if mode.spec:
|
||||
args += ["--speculative-config", json.dumps(SPEC_CONFIG)]
|
||||
if self.dp > 1:
|
||||
args += ["--data-parallel-size", str(self.dp), "--enable-expert-parallel"]
|
||||
if self.dcp > 1:
|
||||
args += ["--decode-context-parallel-size", str(self.dcp)]
|
||||
if self.kv_config is not None:
|
||||
args += ["--kv-transfer-config", json.dumps(self.kv_config)]
|
||||
if not self.prefix_caching:
|
||||
return args + ["--no-enable-prefix-caching"]
|
||||
args.append("--enable-prefix-caching")
|
||||
if mode.prefix_match_unit is not None:
|
||||
args += ["--prefix-match-unit", str(mode.prefix_match_unit)]
|
||||
return args
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Deployment:
|
||||
decode: Instance
|
||||
prefill: Instance | None = None
|
||||
# Reset GPU prefix caches each turn so hits must come from CPU.
|
||||
offload: bool = False
|
||||
|
||||
@property
|
||||
def instances(self) -> tuple[Instance, ...]:
|
||||
return (self.prefill, self.decode) if self.prefill else (self.decode,)
|
||||
|
||||
@property
|
||||
def num_gpus(self) -> int:
|
||||
return sum(i.num_gpus for i in self.instances)
|
||||
|
||||
|
||||
DEPLOYMENTS = {
|
||||
"plain": Deployment(Instance()),
|
||||
"offload": Deployment(Instance(kv_config=OFFLOAD), offload=True),
|
||||
"dcp2": Deployment(Instance(tp=2, dcp=2)),
|
||||
"dcp2-offload": Deployment(Instance(tp=2, dcp=2, kv_config=OFFLOAD), offload=True),
|
||||
"pd": Deployment(Instance(kv_config=NIXL), prefill=Instance(kv_config=NIXL)),
|
||||
# NIXL rejects prefix caching on a hybrid decoder with a different TP.
|
||||
"pd-tp2-dep2": Deployment(
|
||||
Instance(dp=2, kv_config=NIXL, prefix_caching=False),
|
||||
prefill=Instance(tp=2, kv_config=NIXL),
|
||||
),
|
||||
"pd-offload": Deployment(
|
||||
Instance(kv_config=NIXL_OFFLOAD),
|
||||
prefill=Instance(kv_config=NIXL_OFFLOAD),
|
||||
offload=True,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _known_failure(name: str, mode: Mode) -> str | None:
|
||||
deployment = DEPLOYMENTS[name]
|
||||
if mode.spec and deployment.decode.dcp > 1:
|
||||
return "FlashInfer MLA DCP decode breaks on ragged spec batches (#59392)"
|
||||
return None
|
||||
|
||||
|
||||
def _case(name: str, mode: Mode):
|
||||
marks = []
|
||||
if (num_gpus := DEPLOYMENTS[name].num_gpus) > 1:
|
||||
marks += multi_gpu_marks(num_gpus=num_gpus)
|
||||
if reason := _known_failure(name, mode):
|
||||
marks.append(pytest.mark.xfail(reason=reason, strict=True))
|
||||
return pytest.param(name, mode, marks=marks, id=f"{name}-{mode.name}")
|
||||
|
||||
|
||||
def _block_size(url: str) -> int:
|
||||
response = requests.get(f"{url}/metrics", timeout=30)
|
||||
response.raise_for_status()
|
||||
for family in text_string_to_metric_families(response.text):
|
||||
for sample in family.samples:
|
||||
if sample.name == "vllm:cache_config_info":
|
||||
return lcm(
|
||||
*(
|
||||
int(value)
|
||||
for key in ("block_size", "mamba_block_size")
|
||||
if (value := sample.labels.get(key, "None")) != "None"
|
||||
)
|
||||
)
|
||||
raise AssertionError("missing vllm:cache_config_info metric")
|
||||
|
||||
|
||||
def _complete(url: str, prompt: list[int], salt: str, **extra) -> dict:
|
||||
body = {
|
||||
"model": MODEL,
|
||||
"prompt": prompt,
|
||||
"max_tokens": MAX_TOKENS,
|
||||
"temperature": 0,
|
||||
"logprobs": NUM_LOGPROBS,
|
||||
"return_tokens_as_token_ids": True,
|
||||
"cache_salt": salt,
|
||||
**extra,
|
||||
}
|
||||
response = requests.post(f"{url}/v1/completions", json=body, timeout=600)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
|
||||
def _pd_complete(
|
||||
prefill_url: str, decode_url: str, prompt: list[int], salt: str
|
||||
) -> tuple[dict, dict]:
|
||||
remote_decode = {
|
||||
"do_remote_decode": True,
|
||||
"do_remote_prefill": False,
|
||||
"remote_engine_id": None,
|
||||
"remote_block_ids": None,
|
||||
"remote_host": None,
|
||||
"remote_port": None,
|
||||
}
|
||||
prefill = _complete(
|
||||
prefill_url, prompt, salt, max_tokens=1, kv_transfer_params=remote_decode
|
||||
)
|
||||
decode = _complete(
|
||||
decode_url, prompt, salt, kv_transfer_params=prefill["kv_transfer_params"]
|
||||
)
|
||||
return prefill, decode
|
||||
|
||||
|
||||
def _cached(response: dict) -> int:
|
||||
return response["usage"]["prompt_tokens_details"]["cached_tokens"]
|
||||
|
||||
|
||||
def _token_id(token: str) -> int:
|
||||
return int(token.removeprefix("token_id:"))
|
||||
|
||||
|
||||
def _tokens_text_logprobs(response: dict):
|
||||
choice = response["choices"][0]
|
||||
ids = [_token_id(t) for t in choice["logprobs"]["tokens"]]
|
||||
top = [
|
||||
{_token_id(t): lp for t, lp in d.items()}
|
||||
for d in choice["logprobs"]["top_logprobs"]
|
||||
]
|
||||
return ids, choice["text"], top
|
||||
|
||||
|
||||
def _reset_gpu_prefix_cache(url: str) -> None:
|
||||
response = requests.post(
|
||||
f"{url}/reset_prefix_cache", params={"reset_external": "false"}, timeout=60
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Turn:
|
||||
prompt_len: int
|
||||
cached: int
|
||||
expected_cached: int
|
||||
recompute_cached: int
|
||||
output: tuple
|
||||
recompute: tuple
|
||||
|
||||
@property
|
||||
def matches_recompute(self) -> bool:
|
||||
return self.output[0] == self.recompute[0]
|
||||
|
||||
|
||||
def _expected_cached(prev_prompt_len: int, hit_unit: int, spec: bool) -> int:
|
||||
"""A turn must reuse the previous prompt up to its last checkpoint.
|
||||
|
||||
EAGLE-style drafts drop the trailing unit, so the checkpoint is one earlier.
|
||||
"""
|
||||
if not prev_prompt_len:
|
||||
return 0
|
||||
checkpoint = (prev_prompt_len - 1) // hit_unit * hit_unit
|
||||
return max(checkpoint - hit_unit, 0) if spec else checkpoint
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _serve(deployment: Deployment, mode: Mode) -> Iterator[list[RemoteOpenAIServer]]:
|
||||
with contextlib.ExitStack() as stack:
|
||||
# Sequential shutdown waits on memory still held by sibling servers.
|
||||
servers: list[RemoteOpenAIServer] = []
|
||||
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
|
||||
next_gpu = 0
|
||||
for instance in deployment.instances:
|
||||
env = {}
|
||||
if deployment.offload:
|
||||
# Enables /reset_prefix_cache.
|
||||
env["VLLM_SERVER_DEV_MODE"] = "1"
|
||||
if deployment.prefill is not None:
|
||||
gpus = range(next_gpu, next_gpu + instance.num_gpus)
|
||||
next_gpu += instance.num_gpus
|
||||
env["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, gpus))
|
||||
env["VLLM_SSM_CONV_STATE_LAYOUT"] = "DS"
|
||||
env["VLLM_NIXL_SIDE_CHANNEL_PORT"] = str(get_open_port())
|
||||
servers.append(
|
||||
RemoteOpenAIServer(
|
||||
MODEL,
|
||||
instance.args(mode),
|
||||
env_dict=env,
|
||||
max_wait_seconds=STARTUP_TIMEOUT,
|
||||
)
|
||||
)
|
||||
yield servers
|
||||
|
||||
|
||||
def _run_conversation(
|
||||
deployment: Deployment,
|
||||
servers: list[RemoteOpenAIServer],
|
||||
mode: Mode,
|
||||
) -> list[Turn]:
|
||||
# Under P/D the prefiller computes the prompt, so hits are read from it.
|
||||
compute_url, decode_url = servers[0].url_root, servers[-1].url_root
|
||||
block_size = _block_size(compute_url)
|
||||
hit_unit = mode.prefix_match_unit or block_size
|
||||
rng = random.Random(0)
|
||||
|
||||
def new_tokens(n: int) -> list[int]:
|
||||
return [rng.randint(1000, 150000) for _ in range(n)]
|
||||
|
||||
prompt = new_tokens(FIRST_PROMPT_BLOCKS * block_size + UNALIGNED_TAIL)
|
||||
prev_len = 0
|
||||
turns = []
|
||||
for turn in range(NUM_TURNS):
|
||||
if deployment.offload:
|
||||
for server in servers:
|
||||
_reset_gpu_prefix_cache(server.url_root)
|
||||
if deployment.prefill is not None:
|
||||
computed, output = _pd_complete(compute_url, decode_url, prompt, "conv")
|
||||
else:
|
||||
computed = output = _complete(decode_url, prompt, "conv")
|
||||
recompute = _complete(decode_url, prompt, f"recompute-{turn}")
|
||||
turns.append(
|
||||
Turn(
|
||||
prompt_len=len(prompt),
|
||||
cached=_cached(computed),
|
||||
expected_cached=_expected_cached(prev_len, hit_unit, mode.spec),
|
||||
recompute_cached=_cached(recompute),
|
||||
output=_tokens_text_logprobs(output),
|
||||
recompute=_tokens_text_logprobs(recompute),
|
||||
)
|
||||
)
|
||||
prev_len = len(prompt)
|
||||
prompt = prompt + new_tokens(TURN_TOKENS)
|
||||
return turns
|
||||
|
||||
|
||||
def _format(turns: list[Turn]) -> str:
|
||||
rows = ["turn prompt cached expected matches_recompute"]
|
||||
rows += [
|
||||
f"{i:>4} {t.prompt_len:>6} {t.cached:>6} {t.expected_cached:>8} "
|
||||
f"{t.matches_recompute}"
|
||||
for i, t in enumerate(turns)
|
||||
]
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name, mode", [_case(name, mode) for name in DEPLOYMENTS for mode in MODES]
|
||||
)
|
||||
def test_turns_reuse_prefix_and_match_recompute(name: str, mode: Mode) -> None:
|
||||
deployment = DEPLOYMENTS[name]
|
||||
with _serve(deployment, mode) as servers:
|
||||
turns = _run_conversation(deployment, servers, mode)
|
||||
table = _format(turns)
|
||||
print(f"\n{name}-{mode.name}\n{table}")
|
||||
|
||||
assert all(t.recompute_cached == 0 for t in turns), table
|
||||
assert all(t.cached >= t.expected_cached for t in turns), table
|
||||
# Dummy weights differ across TP sizes.
|
||||
if len({i.tp for i in deployment.instances}) == 1:
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=[t.recompute for t in turns],
|
||||
outputs_1_lst=[t.output for t in turns],
|
||||
name_0="recompute",
|
||||
name_1=name,
|
||||
)
|
||||
@@ -5,16 +5,15 @@ from transformers import GraniteConfig
|
||||
|
||||
from vllm.model_executor.models.granite import granite_layer_attn_params
|
||||
|
||||
from ...utils import check_logprobs_close, check_transformers_version
|
||||
from ...utils import check_logprobs_close
|
||||
|
||||
# model -> minimum transformers version, or None if unconstrained
|
||||
MODELS = {
|
||||
MODELS = [
|
||||
# TODO(sang): Sliding window should be tested separately.
|
||||
"ibm/PowerLM-3b": None,
|
||||
"ibm/PowerMoE-3b": None,
|
||||
"ibm-granite/granite-swash-2b": "5.15.1",
|
||||
"ibm-granite/granite-swash-3b-a600m": "5.15.1",
|
||||
}
|
||||
"ibm/PowerLM-3b",
|
||||
"ibm/PowerMoE-3b",
|
||||
"ibm-granite/granite-swash-2b",
|
||||
"ibm-granite/granite-swash-3b-a600m",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
@@ -31,8 +30,6 @@ def test_models(
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
) -> None:
|
||||
check_transformers_version(model, min_transformers_version=MODELS[model])
|
||||
|
||||
with hf_runner(model, dtype=dtype) as hf_model:
|
||||
hf_outputs = hf_model.generate_greedy_logprobs_limit(
|
||||
example_prompts, max_tokens, num_logprobs
|
||||
|
||||
@@ -799,7 +799,6 @@ VLM_TEST_SETTINGS = {
|
||||
dtype="half",
|
||||
num_logprobs=10,
|
||||
patch_hf_runner=model_utils.ovis2_5_patch_hf_runner,
|
||||
hf_model_kwargs={"revision": "refs/pr/5"},
|
||||
),
|
||||
"paddleocr_vl": VLMTestInfo(
|
||||
models=["PaddlePaddle/PaddleOCR-VL"],
|
||||
|
||||
@@ -104,16 +104,7 @@ def run_test(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"nvidia/NVIDIA-Nemotron-Parse-v1.2",
|
||||
# v2.0's checkpoint ties lm_head to the decoder's input embeddings
|
||||
# instead of shipping a separate lm_head.weight tensor; this
|
||||
# exercises that path (see tie_weights() in nemotron_parse.py).
|
||||
"nvidia/NVIDIA-Nemotron-Parse-2.0",
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.2"])
|
||||
@pytest.mark.parametrize("dtype", ["bfloat16"])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
def test_models(
|
||||
|
||||
@@ -12,12 +12,22 @@ Run with:
|
||||
pytest tests/models/multimodal/processing/test_deepseek_ocr.py -v
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm.model_executor.models.deepseek_ocr import DeepseekOCRImagePixelInputs
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import DeepseekOCRProcessor
|
||||
from vllm.model_executor.models.deepseek_ocr import (
|
||||
DeepseekOCRForCausalLM,
|
||||
DeepseekOCRImagePixelInputs,
|
||||
)
|
||||
from vllm.model_executor.models.deepseek_ocr2 import DeepseekOCR2ForCausalLM
|
||||
from vllm.transformers_utils.processors.deepseek_ocr import (
|
||||
DeepseekOCRProcessor,
|
||||
ImageTransform,
|
||||
)
|
||||
|
||||
MODEL_ID = "deepseek-ai/DeepSeek-OCR"
|
||||
|
||||
@@ -155,3 +165,80 @@ class TestDeepseekOCRInputValidation:
|
||||
def test_missing_images_raises_value_error(self, processor):
|
||||
with pytest.raises(ValueError, match="prompt and images"):
|
||||
processor(prompt="<image>\nDescribe this image.", images=None)
|
||||
|
||||
|
||||
def _balanced_normalized_pixels(
|
||||
batch: int = 1, channels: int = 3, height: int = 64, width: int = 64
|
||||
) -> torch.Tensor:
|
||||
"""Build a pixel tensor whose values cancel to an exact sum of 0.
|
||||
|
||||
Matches the normalized half-black / half-white case from the DeepSeek-OCR
|
||||
processor (Normalize(0.5, 0.5) maps 0→-1 and 255→+1).
|
||||
"""
|
||||
assert width % 2 == 0
|
||||
pixel_values = torch.ones(batch, channels, height, width)
|
||||
pixel_values[..., : width // 2] = -1.0
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
return pixel_values
|
||||
|
||||
|
||||
class TestDeepseekOCRZeroSumPixelsAccepted:
|
||||
"""Zero-sum normalized pixels must still produce validated image inputs.
|
||||
|
||||
Returning None from ``_parse_and_validate_image_input`` makes
|
||||
``embed_multimodal`` return None and trips the worker encoder-output
|
||||
sanity check, killing EngineCore. The zero-sum shortcut is therefore
|
||||
removed; balanced images must parse like any other valid tensor.
|
||||
"""
|
||||
|
||||
def test_ocr_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_ocr2_accepts_balanced_normalized_pixels(self):
|
||||
pixel_values = _balanced_normalized_pixels()
|
||||
images_crop = torch.zeros(0, 3, 40, 40)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
model = SimpleNamespace(vision_config=SimpleNamespace(image_size=64))
|
||||
|
||||
image_input = DeepseekOCR2ForCausalLM._parse_and_validate_image_input(
|
||||
model,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
|
||||
assert image_input is not None
|
||||
assert image_input.data is pixel_values
|
||||
|
||||
def test_image_transform_half_black_half_white_sums_to_zero(self):
|
||||
"""A 1024×1024 half-black/half-white PNG through ImageTransform
|
||||
(ToTensor + Normalize(0.5, 0.5)) yields an exact zero sum, and parse
|
||||
must still accept it.
|
||||
"""
|
||||
image = Image.new("RGB", (1024, 1024), (0, 0, 0))
|
||||
image.paste(Image.new("RGB", (512, 1024), (255, 255, 255)), (512, 0))
|
||||
pixel_values = ImageTransform()(image).unsqueeze(0)
|
||||
assert pixel_values.sum().item() == 0.0
|
||||
|
||||
images_crop = torch.zeros(0, 3, 640, 640)
|
||||
images_spatial_crop = torch.tensor([[1, 1]])
|
||||
image_input = DeepseekOCRForCausalLM._parse_and_validate_image_input(
|
||||
None,
|
||||
pixel_values=pixel_values,
|
||||
images_crop=images_crop,
|
||||
images_spatial_crop=images_spatial_crop,
|
||||
)
|
||||
assert image_input is not None
|
||||
assert torch.equal(image_input.data, pixel_values)
|
||||
|
||||
@@ -188,3 +188,54 @@ def test_mm_device_do_normalize():
|
||||
torch.testing.assert_close(
|
||||
output, normalized_values.to(device), rtol=1e-5, atol=1e-6
|
||||
)
|
||||
|
||||
|
||||
def _image_info(**kwargs):
|
||||
ctx = build_model_context(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
**kwargs,
|
||||
)
|
||||
return MULTIMODAL_REGISTRY.create_processor(
|
||||
ctx.model_config,
|
||||
tokenizer=ctx.tokenizer,
|
||||
).info
|
||||
|
||||
|
||||
def test_image_encoder_cache_covers_full_token_budget():
|
||||
"""The most-features probe must reach the processor's token ceiling.
|
||||
|
||||
The inherited square probe refits to 89x89 = 7921 tokens under the
|
||||
max_image_tokens=8000 budget, so the encoder cache came up short and
|
||||
ordinary non-square images in 7922-8000 tokens were refused with
|
||||
HTTP 400 (#59539).
|
||||
"""
|
||||
info = _image_info()
|
||||
assert info.get_max_image_tokens() == 8000
|
||||
|
||||
# The shapes from the issue, with the token counts the processor
|
||||
# actually produces; every one must fit the cache.
|
||||
for width, height, expected_tokens in [
|
||||
(4032, 3024, 7931), # phone photo, 4:3
|
||||
(3840, 2160, 7973), # 4K frame, 16:9
|
||||
(3508, 2480, 7950), # A4 at 300 dpi
|
||||
(2600, 2400, 7998), # 13:12
|
||||
(3000, 3000, 7921), # square worst case before the fix
|
||||
]:
|
||||
num_tokens = info.get_num_image_tokens(image_width=width, image_height=height)
|
||||
assert num_tokens == expected_tokens
|
||||
assert num_tokens <= info.get_max_image_tokens()
|
||||
|
||||
# The profiling dummy covers the real worst case, not the square one.
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert size.width * size.height == 2240 * 2800
|
||||
|
||||
|
||||
def test_image_encoder_cache_follows_max_pixels_override():
|
||||
info = _image_info(mm_processor_kwargs={"max_pixels": 1568 * 100})
|
||||
assert info.get_max_image_tokens() == 100
|
||||
size = info.get_image_size_with_most_features()
|
||||
assert (
|
||||
info.get_num_image_tokens(image_width=size.width, image_height=size.height)
|
||||
== 100
|
||||
)
|
||||
|
||||
@@ -4,11 +4,8 @@ import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import LegacyMultiModalProcessor
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
from .transformers_backend import PROCESSOR_CLASSES, create_processor
|
||||
|
||||
AUDIO_MODEL_SETTINGS = {
|
||||
"ibm-granite/granite-speech-3.3-2b": {
|
||||
"prompt": (
|
||||
@@ -56,7 +53,6 @@ AUDIO_MODEL_SETTINGS = {
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize(
|
||||
"model_id",
|
||||
[
|
||||
@@ -78,10 +74,12 @@ AUDIO_MODEL_SETTINGS = {
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
],
|
||||
)
|
||||
def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
def test_audio_multimodal_processor(model_id):
|
||||
settings = AUDIO_MODEL_SETTINGS[model_id]
|
||||
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
audio = np.zeros(16000, dtype=np.float32)
|
||||
mm_data = {"audio": (audio, 16000)}
|
||||
@@ -112,13 +110,14 @@ def test_audio_multimodal_processor(model_id, processor_cls):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("separator", [" and ", ""])
|
||||
def test_audio_multiple_inputs(separator, processor_cls):
|
||||
def test_audio_multiple_inputs(separator):
|
||||
"""Multiple audios per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
model_id = "ibm-granite/granite-speech-3.3-2b"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
audio_token = mm_processor.info.get_hf_processor().audio_token
|
||||
# One token per audio; the processor expands each to its placeholder run.
|
||||
@@ -128,21 +127,11 @@ def test_audio_multiple_inputs(separator, processor_cls):
|
||||
)
|
||||
audios = [np.zeros(16000, dtype=np.float32), np.zeros(24000, dtype=np.float32)]
|
||||
|
||||
def process():
|
||||
return mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
# The legacy path reads placeholders off contiguous runs of the audio token, so
|
||||
# it cannot tell adjacent ones apart and says so instead of merging them
|
||||
if processor_cls is LegacyMultiModalProcessor and not separator:
|
||||
with pytest.raises(ValueError, match="Separate them in the prompt"):
|
||||
process()
|
||||
return
|
||||
|
||||
result = process()
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data({"audio": audios}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
|
||||
assert len(result["mm_placeholders"]["audio"]) == 2
|
||||
assert len(result["mm_kwargs"]["audio"]) == 2
|
||||
|
||||
@@ -5,23 +5,16 @@ from unittest.mock import patch
|
||||
import pytest
|
||||
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
|
||||
from .transformers_backend import (
|
||||
PROCESSOR_CLASSES,
|
||||
create_cached_processor,
|
||||
create_processor,
|
||||
offsets_only,
|
||||
)
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
||||
def test_multimodal_processor(model_id, processor_cls):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
def test_multimodal_processor(model_id):
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
image_pil = ImageAsset("cherry_blossom").pil_image
|
||||
mm_data = {"image": image_pil}
|
||||
@@ -63,9 +56,11 @@ def test_multimodal_processor(model_id, processor_cls):
|
||||
)
|
||||
|
||||
|
||||
def _process_two_images(processor_cls, separator: str):
|
||||
def _process_two_images(separator: str):
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
prompt = (
|
||||
@@ -80,30 +75,29 @@ def _process_two_images(processor_cls, separator: str):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_multiple_inputs(processor_cls):
|
||||
def test_image_multiple_inputs():
|
||||
"""Multiple images per prompt are each detected as a separate placeholder
|
||||
and multi-modal item by the Transformers modelling backend."""
|
||||
result = _process_two_images(processor_cls, separator="\n and ")
|
||||
result = _process_two_images(separator="\n and ")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_image_adjacent_inputs(processor_cls):
|
||||
def test_image_adjacent_inputs():
|
||||
"""Adjacent images stay separate placeholders rather than merging into one."""
|
||||
result = _process_two_images(processor_cls, separator="")
|
||||
result = _process_two_images(separator="")
|
||||
|
||||
assert len(result["mm_placeholders"]["image"]) == 2
|
||||
assert len(result["mm_kwargs"]["image"]) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
def test_batch_padding_removed_from_image_items():
|
||||
"""Emu3 pads every image up to the largest in the batch, which would leave an
|
||||
item's data dependent on what it was processed with and so uncacheable."""
|
||||
mm_processor = create_processor("BAAI/Emu3-Chat-hf", processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="BAAI/Emu3-Chat-hf", model_impl="transformers")
|
||||
)
|
||||
image_token = mm_processor.info.get_hf_processor().image_token
|
||||
|
||||
images = [
|
||||
@@ -128,53 +122,39 @@ def test_batch_padding_removed_from_image_items(processor_cls):
|
||||
assert len(shapes) == 2
|
||||
|
||||
|
||||
def _process_one_gemma3_image(processor_cls):
|
||||
mm_processor = create_processor("google/gemma-3-4b-it", processor_cls)
|
||||
hf_processor = mm_processor.info.get_hf_processor()
|
||||
result = mm_processor(
|
||||
prompt=f"{hf_processor.boi_token} What is this?",
|
||||
def _process_one_gemma3_image():
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model="google/gemma-3-4b-it", model_impl="transformers")
|
||||
)
|
||||
boi_token = mm_processor.info.get_hf_processor().boi_token
|
||||
return mm_processor(
|
||||
prompt=f"{boi_token} What is this?",
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
return hf_processor, result
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_non_embedding_tokens_excluded_from_placeholders():
|
||||
"""Gemma3 wraps each image in text that carries no embeddings, which must be
|
||||
inside the placeholder range but masked out of it."""
|
||||
_, result = _process_one_gemma3_image(OffsetsMultiModalProcessor)
|
||||
result = _process_one_gemma3_image()
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.is_embed is not None
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
def test_legacy_placeholders_hold_only_image_tokens():
|
||||
"""The legacy path spans whatever `mm_token_type_ids` attributes to the image,
|
||||
which for Gemma3 excludes the text wrapping it, unlike the replacement the offsets
|
||||
path spans. Gemma3 is also the sharp case for the mask: its `image_token_id` is the
|
||||
marker in the unexpanded prompt, not the token the expansion repeats."""
|
||||
hf_processor, result = _process_one_gemma3_image(LegacyMultiModalProcessor)
|
||||
|
||||
(placeholder,) = result["mm_placeholders"]["image"]
|
||||
assert placeholder.length == hf_processor.image_seq_length
|
||||
prompt_ids = result["prompt_token_ids"]
|
||||
covered = prompt_ids[placeholder.offset : placeholder.offset + placeholder.length]
|
||||
assert set(covered) == {hf_processor.tokenizer.image_token_id}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
def test_tokens_structuring_an_image_are_masked_not_dropped():
|
||||
"""SmolVLM splits each image into tiles introduced by tokens carrying no
|
||||
embeddings. Those belong inside the placeholder and masked out, because the token
|
||||
count the processor reports is over the whole span. Idefics3 also refuses a prompt
|
||||
holding `<image>` when no images are passed, which is how the offsets path has to
|
||||
tokenize it before splicing in the expansion."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
holding `<image>` when no images are passed, which is how the prompt has to be
|
||||
tokenized before splicing in the expansion."""
|
||||
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
result = mm_processor(
|
||||
prompt="<image>What is this?",
|
||||
@@ -189,12 +169,13 @@ def test_tokens_structuring_an_image_are_masked_not_dropped(processor_cls):
|
||||
assert 0 < int(placeholder.is_embed.sum()) < placeholder.length
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_missing_replacement_offsets_names_the_processor():
|
||||
"""A processor that reports no replacement offsets cannot be served, which must
|
||||
be said plainly rather than surfacing later as a field config mismatch."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, OffsetsMultiModalProcessor)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
hf_processor_cls = type(mm_processor.info.get_hf_processor())
|
||||
hf_call = hf_processor_cls.__call__
|
||||
|
||||
@@ -216,11 +197,12 @@ def test_missing_replacement_offsets_names_the_processor():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_text_only_prompt(processor_cls):
|
||||
def test_text_only_prompt():
|
||||
"""An image model still accepts a prompt with no images."""
|
||||
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
|
||||
result = mm_processor(
|
||||
prompt="<|im_start|>user Hello!<|im_end|><|im_start|>assistant\n",
|
||||
@@ -232,12 +214,14 @@ def test_text_only_prompt(processor_cls):
|
||||
assert not result["mm_placeholders"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
def test_repeated_image_hits_the_processor_cache():
|
||||
"""Check that mm caching is actually working."""
|
||||
mm_processor, cache = create_cached_processor(
|
||||
"llava-hf/llava-onevision-qwen2-0.5b-ov-hf", OffsetsMultiModalProcessor
|
||||
model_config = ModelConfig(
|
||||
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", model_impl="transformers"
|
||||
)
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
def process():
|
||||
@@ -255,7 +239,6 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
assert first["mm_hashes"] == second["mm_hashes"]
|
||||
|
||||
|
||||
@offsets_only
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "prompt"),
|
||||
[
|
||||
@@ -267,7 +250,7 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
"<image> and more text",
|
||||
marks=pytest.mark.xfail(
|
||||
reason="Emu3Processor prepends its BOS token only when images are "
|
||||
"passed, so the unexpanded prompt the offsets path tokenizes never "
|
||||
"passed, so the unexpanded prompt vLLM tokenizes never "
|
||||
"gets one. Fixed by huggingface/transformers#47924, unreleased.",
|
||||
strict=False,
|
||||
),
|
||||
@@ -275,33 +258,40 @@ def test_repeated_image_hits_the_processor_cache():
|
||||
],
|
||||
)
|
||||
def test_spliced_prompt_matches_hf_expansion(model_id, prompt):
|
||||
"""The offsets path splices the expansion into a prompt tokenized without any
|
||||
multi-modal data, so its token ids have to come out the same as the ones the HF
|
||||
processor produces itself, which is what the legacy path returns."""
|
||||
prompt_ids = []
|
||||
for processor_cls in (LegacyMultiModalProcessor, OffsetsMultiModalProcessor):
|
||||
mm_processor = create_processor(model_id, processor_cls)
|
||||
prompt_ids.append(
|
||||
mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=mm_processor.info.parse_mm_data(
|
||||
{"image": ImageAsset("cherry_blossom").pil_image}
|
||||
),
|
||||
hf_processor_mm_kwargs={},
|
||||
)["prompt_token_ids"]
|
||||
)
|
||||
"""The prompt is tokenized without any multi-modal data and the expansion spliced
|
||||
in, so its token ids have to come out the same as the ones the HF processor
|
||||
produces itself."""
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
info = mm_processor.info
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
|
||||
legacy_ids, offsets_ids = prompt_ids
|
||||
assert legacy_ids == offsets_ids
|
||||
hf_processor = info.get_hf_processor()
|
||||
prompt_ids = info.get_tokenizer().encode(
|
||||
prompt, **info.default_tok_params.get_encode_kwargs()
|
||||
)
|
||||
hf_ids = info.ctx.call_hf_processor(
|
||||
hf_processor,
|
||||
dict(text=hf_processor.decode(prompt_ids), images=[image]),
|
||||
dict(truncation=False, add_special_tokens=False),
|
||||
)["input_ids"][0].tolist()
|
||||
|
||||
result = mm_processor(
|
||||
prompt=prompt,
|
||||
mm_items=info.parse_mm_data({"image": image}),
|
||||
hf_processor_mm_kwargs={},
|
||||
)
|
||||
assert result["prompt_token_ids"] == hf_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("processor_cls", PROCESSOR_CLASSES)
|
||||
def test_nested_image_fields_split_per_image(processor_cls):
|
||||
def test_nested_image_fields_split_per_image():
|
||||
"""Idefics3 returns image fields with a leading batch dimension, putting the rows
|
||||
belonging to each image one dimension further in. Slicing the batch dimension
|
||||
instead handed the first image every row and the second an empty tensor."""
|
||||
mm_processor = create_processor(
|
||||
"HuggingFaceTB/SmolVLM-256M-Instruct", processor_cls
|
||||
model_id = "HuggingFaceTB/SmolVLM-256M-Instruct"
|
||||
mm_processor = MULTIMODAL_REGISTRY.create_processor(
|
||||
ModelConfig(model=model_id, model_impl="transformers")
|
||||
)
|
||||
image = ImageAsset("cherry_blossom").pil_image
|
||||
result = mm_processor(
|
||||
|
||||
@@ -1,51 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helpers for testing both processors of the Transformers modelling backend."""
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
LegacyMultiModalProcessor,
|
||||
MultiModalDummyInputsBuilder,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
OffsetsMultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal.cache import MultiModalProcessorOnlyCache
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
|
||||
offsets_only = pytest.mark.skipif(
|
||||
MultiModalProcessor is not OffsetsMultiModalProcessor,
|
||||
reason="Replacement offsets are only used from transformers 5.15.0 onwards",
|
||||
)
|
||||
|
||||
PROCESSOR_CLASSES = [
|
||||
pytest.param(LegacyMultiModalProcessor, id="legacy"),
|
||||
pytest.param(OffsetsMultiModalProcessor, id="offsets", marks=offsets_only),
|
||||
]
|
||||
|
||||
|
||||
def create_processor(model_id: str, processor_cls):
|
||||
"""Build a processor directly, because the registry only ever builds the one the
|
||||
installed transformers version selects, leaving the other path untested."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info))
|
||||
|
||||
|
||||
def create_cached_processor(model_id: str, processor_cls):
|
||||
"""Build a processor backed by a real multi-modal processor cache, and hand the
|
||||
cache back so a test can check it was actually used."""
|
||||
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
||||
model_config.multimodal_config.mm_processor_cache_gb = 4
|
||||
ctx = InputProcessingContext(
|
||||
model_config, cached_tokenizer_from_config(model_config)
|
||||
)
|
||||
info = MultiModalProcessingInfo(ctx)
|
||||
cache = MultiModalProcessorOnlyCache(model_config)
|
||||
return processor_cls(info, MultiModalDummyInputsBuilder(info)), cache
|
||||
+15
-81
@@ -233,9 +233,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"CohereLabs/North-Mini-Code",
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm", min_transformers_version="4.58"),
|
||||
"CwmForCausalLM": _HfExamplesInfo("facebook/cwm"),
|
||||
# FIXME: databricks/dbrx-instruct has been deleted
|
||||
"DbrxForCausalLM": _HfExamplesInfo(
|
||||
"databricks/dbrx-instruct", is_available_online=False
|
||||
@@ -266,9 +265,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Exaone4ForCausalLM": _HfExamplesInfo("LGAI-EXAONE/EXAONE-4.0-32B"),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B", min_transformers_version="5.1.0"
|
||||
),
|
||||
"ExaoneMoeForCausalLM": _HfExamplesInfo("LGAI-EXAONE/K-EXAONE-236B-A23B"),
|
||||
"FalconForCausalLM": _HfExamplesInfo("tiiuae/falcon-7b"),
|
||||
"FalconH1ForCausalLM": _HfExamplesInfo("tiiuae/Falcon-H1-0.5B-Base"),
|
||||
"FlexOlmoForCausalLM": _HfExamplesInfo("allenai/Flex-reddit-2x7B-1T"),
|
||||
@@ -279,7 +276,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Gemma3ForCausalLM": _HfExamplesInfo("google/gemma-3-1b-it"),
|
||||
"Gemma4ForCausalLM": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Gemma3nForCausalLM": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmForCausalLM": _HfExamplesInfo("zai-org/glm-4-9b-chat-hf"),
|
||||
@@ -287,11 +283,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Glm4MoeForCausalLM": _HfExamplesInfo("zai-org/GLM-4.5"),
|
||||
"Glm4MoeLiteForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.7-Flash",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
|
||||
),
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo("zai-org/GLM-5", is_available_online=False),
|
||||
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
|
||||
"GPTBigCodeForCausalLM": _HfExamplesInfo(
|
||||
@@ -316,15 +309,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"GraniteMoeSharedForCausalLM": _HfExamplesInfo(
|
||||
"ibm-research/moe-7b-1b-active-shared-experts"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-3b-a600m", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo(
|
||||
"ibm-granite/granite-swash-2b", min_transformers_version="5.15.1"
|
||||
),
|
||||
"GraniteMoeSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-3b-a600m"),
|
||||
"GraniteSWAForCausalLM": _HfExamplesInfo("ibm-granite/granite-swash-2b"),
|
||||
"HrmTextForCausalLM": _HfExamplesInfo(
|
||||
"sapientinc/HRM-Text-1B",
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"HunYuanDenseV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-7B-Instruct"),
|
||||
"HunYuanMoEV1ForCausalLM": _HfExamplesInfo("tencent/Hunyuan-A13B-Instruct"),
|
||||
@@ -333,7 +321,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"HYV4ForCausalLM": _HfExamplesInfo("tencent/Hy4-preview", trust_remote_code=True),
|
||||
"HyperCLOVAXForCausalLM": _HfExamplesInfo(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"InklingForCausalLM": _HfExamplesInfo(
|
||||
"thinkingmachines/Inkling-NVFP4",
|
||||
@@ -353,9 +340,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"IQuestLoopCoderForCausalLM": _HfExamplesInfo(
|
||||
"IQuestLab/IQuest-Coder-V1-40B-Loop-Instruct", trust_remote_code=True
|
||||
),
|
||||
"Jais2ForCausalLM": _HfExamplesInfo(
|
||||
"inceptionai/Jais-2-8B-Chat", min_transformers_version="4.58"
|
||||
),
|
||||
"Jais2ForCausalLM": _HfExamplesInfo("inceptionai/Jais-2-8B-Chat"),
|
||||
"JambaForCausalLM": _HfExamplesInfo(
|
||||
"ai21labs/AI21-Jamba-1.5-Mini",
|
||||
extras={
|
||||
@@ -375,7 +360,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Lfm2ForCausalLM": _HfExamplesInfo("LiquidAI/LFM2-1.2B"),
|
||||
"Lfm2MoeForCausalLM": _HfExamplesInfo(
|
||||
"LiquidAI/LFM2-8B-A1B",
|
||||
min_transformers_version="5.0.0",
|
||||
use_original_num_layers=True,
|
||||
# Initialize at least one MoE layer
|
||||
hf_overrides={"num_hidden_layers": 4},
|
||||
@@ -422,7 +406,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MiniCPM4ForCausalLM": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM4.1-8B",
|
||||
min_transformers_version="4.56",
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"hf": "HF remote code imports removed `is_torch_fx_available`; "
|
||||
@@ -523,7 +506,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"Qwen3NextForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct",
|
||||
extras={"tiny-random": "tiny-random/qwen3-next-moe"},
|
||||
min_transformers_version="4.56.3",
|
||||
),
|
||||
"Rnj1ForCausalLM": _HfExamplesInfo(
|
||||
"EssentialAI/rnj-1-instruct",
|
||||
@@ -722,7 +704,6 @@ _TOKEN_CLASSIFICATION_EXAMPLE_MODELS = {
|
||||
),
|
||||
"OpenAIPrivacyFilterForTokenClassification": _HfExamplesInfo(
|
||||
"openai/privacy-filter",
|
||||
min_transformers_version="5.6.0.dev0",
|
||||
),
|
||||
"RobertaForTokenClassification": _HfExamplesInfo(
|
||||
"Jean-Baptiste/roberta-large-ner-english"
|
||||
@@ -794,7 +775,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
|
||||
"AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/audio-flamingo-3-hf",
|
||||
min_transformers_version="5.3.0",
|
||||
transformers_version_reason={
|
||||
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
|
||||
},
|
||||
@@ -823,22 +803,18 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
|
||||
min_transformers_version="5.9.0",
|
||||
),
|
||||
"CohereCompassForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/North-Micro-Vision-Instruct",
|
||||
min_transformers_version="5.16.0",
|
||||
),
|
||||
"Cosmos3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Nano",
|
||||
extras={"super": "nvidia/Cosmos3-Super"},
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Cosmos3EdgeForConditionalGeneration": _HfExamplesInfo(
|
||||
"nvidia/Cosmos3-Edge",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="5.15",
|
||||
use_original_num_layers=True,
|
||||
hf_overrides={
|
||||
"text_config": {
|
||||
@@ -887,7 +863,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Exaone4_5_ForConditionalGeneration": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allendou/FireRedASR2-LLM-vllm",
|
||||
@@ -918,19 +893,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Gemma4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-E2B-it",
|
||||
min_transformers_version="5.5.0",
|
||||
),
|
||||
# TODO: update min_transformers_version when Gemma4 Unified lands in
|
||||
# a stable transformers release.
|
||||
"Gemma4UnifiedForConditionalGeneration": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
min_transformers_version="5.8.0",
|
||||
is_available_online=False,
|
||||
),
|
||||
"Gemma3nForConditionalGeneration": _HfExamplesInfo("google/gemma-3n-E2B-it"),
|
||||
"GlmAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-ASR-Nano-2512",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Granite4VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-vision-4.1-4b",
|
||||
@@ -943,7 +913,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"GraniteSpeechPlusForConditionalGeneration": _HfExamplesInfo(
|
||||
"ibm-granite/granite-speech-4.1-2b-plus",
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"GLM4VForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/glm-4v-9b",
|
||||
@@ -957,7 +926,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Glm4vMoeForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-4.5V"),
|
||||
"GlmOcrForConditionalGeneration": _HfExamplesInfo(
|
||||
"zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"H2OVLChatModel": _HfExamplesInfo(
|
||||
@@ -974,7 +942,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"HunYuanVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"tencent/HunyuanOCR",
|
||||
min_transformers_version="5.13",
|
||||
),
|
||||
"Idefics3ForConditionalGeneration": _HfExamplesInfo(
|
||||
"HuggingFaceM4/Idefics3-8B-Llama3",
|
||||
@@ -1087,7 +1054,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"Lfm2VlForConditionalGeneration": _HfExamplesInfo(
|
||||
"LiquidAI/LFM2-VL-450M",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"Llama4ForConditionalGeneration": _HfExamplesInfo(
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
@@ -1154,7 +1120,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"MiniCPMV4_6ForConditionalGeneration": _HfExamplesInfo(
|
||||
"openbmb/MiniCPM-V-4_6",
|
||||
min_transformers_version="5.7.0",
|
||||
),
|
||||
"MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
|
||||
"MiniMaxAI/MiniMax-M3",
|
||||
@@ -1201,7 +1166,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Molmo2ForConditionalGeneration": _HfExamplesInfo(
|
||||
"allenai/Molmo2-8B",
|
||||
extras={"olmo": "allenai/Molmo2-O-7B"},
|
||||
min_transformers_version="4.51",
|
||||
trust_remote_code=True,
|
||||
# required by current PrefixLM implementation
|
||||
max_num_batched_tokens=31872,
|
||||
@@ -1296,19 +1260,15 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"vllm": "Custom processor code is not compatible with Transformers v5."
|
||||
"hf": "HF remote code reads `is_parallelizable`, which was removed "
|
||||
"in Transformers v5."
|
||||
},
|
||||
),
|
||||
"Ovis2_6ForCausalLM": _HfExamplesInfo(
|
||||
"AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
|
||||
),
|
||||
"Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
|
||||
"AIDC-AI/Ovis2.6-30B-A3B",
|
||||
trust_remote_code=True,
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"vllm": "Custom processor code is not compatible with Transformers v5."
|
||||
},
|
||||
"AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
|
||||
),
|
||||
"PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
|
||||
"PaddlePaddle/PaddleOCR-VL",
|
||||
@@ -1353,7 +1313,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"QianfanOCRForConditionalGeneration": _HfExamplesInfo(
|
||||
"baidu/Qianfan-OCR",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen2-Audio-7B-Instruct"
|
||||
@@ -1368,12 +1327,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen3VLForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-4B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3VLMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
@@ -1391,23 +1348,19 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Qwen3ASRRealtimeGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]},
|
||||
),
|
||||
"Qwen3ASRForcedAlignerForTokenClassification": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ForcedAligner-0.6B",
|
||||
max_model_len=4096,
|
||||
min_transformers_version="4.57",
|
||||
hf_overrides={"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]},
|
||||
),
|
||||
"RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True),
|
||||
@@ -1456,7 +1409,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": _HfExamplesInfo(
|
||||
"microsoft/VibeVoice-ASR-HF",
|
||||
min_transformers_version="5.13.0",
|
||||
),
|
||||
"VoxtralForConditionalGeneration": _HfExamplesInfo(
|
||||
"mistralai/Voxtral-Mini-3B-2507",
|
||||
@@ -1530,7 +1482,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
use_original_num_layers=True, # DFlash requires all layers
|
||||
max_model_len=8192, # Reduce for CI
|
||||
max_num_seqs=32,
|
||||
min_transformers_version="4.56.3", # Required for Qwen3Next
|
||||
),
|
||||
"MuseGlimmerAssistantModel": _HfExamplesInfo(
|
||||
"meta-models/Muse-Glimmer-30B",
|
||||
@@ -1576,7 +1527,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="Qwen/Qwen3-Omni-30B-A3B-DSpark",
|
||||
is_available_online=False,
|
||||
use_original_num_layers=True,
|
||||
min_transformers_version="4.57",
|
||||
),
|
||||
"Gemma4DSparkModel": _HfExamplesInfo(
|
||||
"google/gemma-4-12B-it",
|
||||
@@ -1737,7 +1687,6 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"google/gemma-4-E4B-it",
|
||||
speculative_model="google/gemma-4-E4B-it-assistant",
|
||||
trust_remote_code=True,
|
||||
min_transformers_version="5.8.0",
|
||||
),
|
||||
"ErnieMTPModel": _HfExamplesInfo(
|
||||
"baidu/ERNIE-4.5-21B-A3B-PT",
|
||||
@@ -1747,13 +1696,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"ExaoneMoeMTP": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
speculative_model="LGAI-EXAONE/K-EXAONE-236B-A23B",
|
||||
min_transformers_version="5.1.0",
|
||||
enable_prefix_caching=False,
|
||||
),
|
||||
"Exaone4_5_MTP": _HfExamplesInfo(
|
||||
"LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
speculative_model="LGAI-EXAONE/EXAONE-4.5-33B",
|
||||
min_transformers_version="5.6.0",
|
||||
),
|
||||
"ExtractHiddenStatesModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-8B",
|
||||
@@ -1766,12 +1713,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
"Glm4MoeLiteMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-4.7-Flash",
|
||||
speculative_model="zai-org/GLM-4.7-Flash",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"GlmOcrMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-OCR",
|
||||
speculative_model="zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
@@ -1842,9 +1787,7 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
trust_remote_code=True,
|
||||
is_available_online=False,
|
||||
),
|
||||
"Qwen3NextMTP": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-Next-80B-A3B-Instruct", min_transformers_version="4.56.3"
|
||||
),
|
||||
"Qwen3NextMTP": _HfExamplesInfo("Qwen/Qwen3-Next-80B-A3B-Instruct"),
|
||||
"Qwen3_5MTP": _HfExamplesInfo(
|
||||
"Qwen/Qwen3.5-0.8B",
|
||||
speculative_model="Qwen/Qwen3.5-0.8B",
|
||||
@@ -1865,29 +1808,20 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
"TransformersEmbeddingModel": _HfExamplesInfo(
|
||||
"BAAI/bge-base-en-v1.5", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersEmbeddingModel": _HfExamplesInfo("BAAI/bge-base-en-v1.5"),
|
||||
"TransformersForSequenceClassification": _HfExamplesInfo(
|
||||
"papluca/xlm-roberta-base-language-detection",
|
||||
min_transformers_version="5.0.0",
|
||||
),
|
||||
"TransformersForCausalLM": _HfExamplesInfo(
|
||||
"hmellor/Ilama-3.2-1B", trust_remote_code=True
|
||||
),
|
||||
"TransformersMultiModalForCausalLM": _HfExamplesInfo("BAAI/Emu3-Chat-hf"),
|
||||
"TransformersMoEForCausalLM": _HfExamplesInfo(
|
||||
"allenai/OLMoE-1B-7B-0924", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEForCausalLM": _HfExamplesInfo("allenai/OLMoE-1B-7B-0924"),
|
||||
"TransformersMultiModalMoEForCausalLM": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEEmbeddingModel": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
|
||||
),
|
||||
"TransformersMoEForSequenceClassification": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-30B-A3B", min_transformers_version="5.0.0"
|
||||
"Qwen/Qwen3-VL-30B-A3B-Instruct"
|
||||
),
|
||||
"TransformersMoEEmbeddingModel": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||
"TransformersMoEForSequenceClassification": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"),
|
||||
"TransformersMultiModalEmbeddingModel": _HfExamplesInfo("google/gemma-3-4b-it"),
|
||||
"TransformersMultiModalForSequenceClassification": _HfExamplesInfo(
|
||||
"google/gemma-3-4b-it"
|
||||
|
||||
@@ -1,116 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the flashinfer moe_ep backend plumbing.
|
||||
|
||||
Everything here runs without a GPU or a flashinfer install: the flashinfer
|
||||
modules the helpers import lazily are replaced with capture fakes.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_BACKENDS,
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
MEGA_MOE_BACKENDS,
|
||||
NATIVE_MEGA_MOE_BACKENDS,
|
||||
PASSTHROUGH_ALL2ALL_BACKEND,
|
||||
bind_passthrough_all2all_backend,
|
||||
validate_flashinfer_moe_ep_model,
|
||||
)
|
||||
from vllm.utils.flashinfer_moe_ep import (
|
||||
_E2M1_LUT,
|
||||
FI_MOE_EP_BACKEND_SPECS,
|
||||
_dequant_fp4_ue8m0_gran32,
|
||||
build_fi_mega_config,
|
||||
fi_moe_ep_backend_spec,
|
||||
make_fi_moe_ep_bootstrap,
|
||||
megakernel_runtime_requirements,
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
epilogue_from_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_quant_config,
|
||||
map_mxfp4_backend,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
backend_to_kernel_cls as mxfp4_backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
map_nvfp4_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
backend_to_kernel_cls as nvfp4_backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
|
||||
PassThroughPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeBootstrapConfig:
|
||||
world_size: int
|
||||
rank: int
|
||||
process_group: Any = None
|
||||
auto_bootstrap: bool = True
|
||||
device: int | None = field(default=None, kw_only=True)
|
||||
def test_flashinfer_backends_are_megakernels_outside_the_native_model_path():
|
||||
assert FLASHINFER_MOE_EP_BACKENDS <= MEGA_MOE_BACKENDS
|
||||
assert FLASHINFER_MOE_EP_BACKENDS.isdisjoint(NATIVE_MEGA_MOE_BACKENDS)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeDeepGemmMegaMoeConfig:
|
||||
intermediate_size: int
|
||||
top_k: int
|
||||
activation_clamp: float | None
|
||||
fast_math: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeNvfp4CutedslMegaMoeConfig:
|
||||
intermediate_size: int
|
||||
top_k: int
|
||||
activation_clamp: float | None
|
||||
fast_math: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeMegaConfig:
|
||||
megakernel: Any
|
||||
preprocess_weights: bool
|
||||
quantize_input: bool
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_flashinfer(monkeypatch):
|
||||
"""Install a minimal fake flashinfer.moe_ep for the lazy imports."""
|
||||
moe_ep = ModuleType("flashinfer.moe_ep")
|
||||
core = ModuleType("flashinfer.moe_ep.core")
|
||||
runtime = ModuleType("flashinfer.moe_ep.core.runtime")
|
||||
flashinfer = ModuleType("flashinfer")
|
||||
fake_attrs: dict[ModuleType, dict[str, Any]] = {
|
||||
moe_ep: {
|
||||
"BootstrapConfig": _FakeBootstrapConfig,
|
||||
"DeepGemmMegaMoeConfig": _FakeDeepGemmMegaMoeConfig,
|
||||
"Nvfp4CutedslMegaMoeConfig": _FakeNvfp4CutedslMegaMoeConfig,
|
||||
"MegaConfig": _FakeMegaConfig,
|
||||
"core": core,
|
||||
},
|
||||
runtime: {"TORCH_DIST": "torch_dist", "NVSHMEM": "nvshmem"},
|
||||
flashinfer: {"moe_ep": moe_ep},
|
||||
core: {"runtime": runtime},
|
||||
}
|
||||
for mod, attrs in fake_attrs.items():
|
||||
for attr, value in attrs.items():
|
||||
setattr(mod, attr, value)
|
||||
|
||||
for name, mod in {
|
||||
"flashinfer": flashinfer,
|
||||
"flashinfer.moe_ep": moe_ep,
|
||||
"flashinfer.moe_ep.core": core,
|
||||
"flashinfer.moe_ep.core.runtime": runtime,
|
||||
}.items():
|
||||
monkeypatch.setitem(sys.modules, name, mod)
|
||||
return moe_ep
|
||||
|
||||
|
||||
def test_fi_backend_strings_are_registered_mega_moe_backends():
|
||||
assert set(FI_MOE_EP_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
|
||||
assert FLASHINFER_MOE_EP_BACKENDS < MEGA_MOE_BACKENDS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
|
||||
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
|
||||
instead of silently falling through to the generic FusedMoE path."""
|
||||
def test_only_deep_gemm_backend_is_dsv4_specific():
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
["MixtralForCausalLM"],
|
||||
)
|
||||
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["DeepseekV4ForCausalLM"])
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
["MixtralForCausalLM"],
|
||||
)
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
["DeepseekV4ForCausalLM"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -118,8 +92,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
|
||||
)
|
||||
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
|
||||
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
|
||||
models validate their own constraints at construction time."""
|
||||
"""Models validate native deep_gemm mega constraints at construction time."""
|
||||
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
|
||||
|
||||
|
||||
@@ -128,11 +101,7 @@ def test_non_fi_backend_ignores_architectures():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(MEGA_MOE_BACKENDS))
|
||||
def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
"""Every mega backend must qualify for sequence-parallel MoE at
|
||||
TP>1/EP: the predicate once matched only the native backend string,
|
||||
which silently ran the fi backends full-batch with an all-reduce on
|
||||
every rank — 0.42-0.65x native e2e at TP8."""
|
||||
def test_token_sharding_backends_enable_dsv4_sequence_parallel(moe_backend: str):
|
||||
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
@@ -147,113 +116,262 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
assert _use_sequence_parallel(vllm_config)
|
||||
|
||||
|
||||
def test_fi_moe_ep_backend_spec_kernel_and_nvshmem_contract():
|
||||
dg = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
|
||||
assert dg.megakernel == "deep_gemm_mega"
|
||||
assert not dg.needs_nvshmem
|
||||
|
||||
cd = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
|
||||
assert cd.megakernel == "nvfp4_cutedsl"
|
||||
assert cd.needs_nvshmem
|
||||
|
||||
with pytest.raises(ValueError, match="not a flashinfer moe_ep backend"):
|
||||
fi_moe_ep_backend_spec("deep_gemm_mega_moe")
|
||||
def test_dsv4_requests_pre_fc2_router_weight_placement():
|
||||
dsv4 = SimpleNamespace(routing_method=RoutingMethodType.DeepseekV4)
|
||||
default = SimpleNamespace(routing_method=RoutingMethodType.Default)
|
||||
assert fi_ep.apply_topk_in_fc1(dsv4)
|
||||
assert not fi_ep.apply_topk_in_fc1(default)
|
||||
|
||||
|
||||
def test_megakernel_runtime_requirements(fake_flashinfer):
|
||||
dg = megakernel_runtime_requirements(
|
||||
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
|
||||
def test_backend_specs_preserve_weight_format_contracts():
|
||||
cutedsl = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert cutedsl.kernel == "cutedsl"
|
||||
assert cutedsl.weight_formats == frozenset({"nvfp4", "mxfp4"})
|
||||
|
||||
deep_gemm = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
assert deep_gemm.kernel == "deep_gemm"
|
||||
assert deep_gemm.weight_formats == frozenset({"mxfp4"})
|
||||
|
||||
|
||||
def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
|
||||
fields = dict(
|
||||
moe_backend=moe_backend,
|
||||
moe_parallel_config=SimpleNamespace(
|
||||
use_ep=True, use_batched_activation_format=False
|
||||
),
|
||||
is_act_and_mul=True,
|
||||
activation=MoEActivation.SILU,
|
||||
has_hash_routing=False,
|
||||
routing_method=RoutingMethodType.DeepseekV4,
|
||||
router_logits_dtype=torch.float32,
|
||||
hidden_dim=256,
|
||||
is_lora_enabled=False,
|
||||
skip_final_all_reduce=False,
|
||||
in_dtype=torch.bfloat16,
|
||||
has_bias=False,
|
||||
swiglu_alpha=None,
|
||||
swiglu_beta=None,
|
||||
)
|
||||
assert dg == frozenset({"torch_dist"})
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
cd = megakernel_runtime_requirements(
|
||||
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
|
||||
|
||||
def _is_supported(moe, weight_key, activation_key) -> tuple[bool, str]:
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
weight_key,
|
||||
activation_key,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert cd == frozenset({"torch_dist", "nvshmem"})
|
||||
return supported, reason or ""
|
||||
|
||||
|
||||
def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
|
||||
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
|
||||
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
|
||||
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
|
||||
device it already bound via BootstrapConfig.device."""
|
||||
import vllm.utils.flashinfer_moe_ep as mod
|
||||
|
||||
pg = object()
|
||||
@pytest.fixture
|
||||
def megakernel_host(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
|
||||
"""A vLLM config without DBO/EPLB/weight transfer, on a supported device."""
|
||||
config = SimpleNamespace(
|
||||
weight_transfer_config=None,
|
||||
parallel_config=SimpleNamespace(enable_dbo=False, enable_eplb=False),
|
||||
)
|
||||
monkeypatch.setattr(fi_ep, "get_current_vllm_config", lambda: config)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(world_size=4, rank_in_group=2, device_group=pg),
|
||||
FlashInferMoeEpExperts, "_supports_current_device", staticmethod(lambda: True)
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 3)
|
||||
|
||||
bootstrap = make_fi_moe_ep_bootstrap()
|
||||
|
||||
assert bootstrap.world_size == 4
|
||||
assert bootstrap.rank == 2
|
||||
assert bootstrap.process_group is pg
|
||||
assert bootstrap.auto_bootstrap is False
|
||||
assert bootstrap.device == 3
|
||||
return config
|
||||
|
||||
|
||||
def test_build_fi_mega_config_selects_kernel_config(fake_flashinfer):
|
||||
dg = build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=7.0,
|
||||
megakernel="deep_gemm_mega",
|
||||
def test_megakernel_requires_expert_parallel(megakernel_host):
|
||||
"""The oracle rejects the megakernel through the experts' support check."""
|
||||
moe = _megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert _is_supported(moe, kNvfp4Static, kNvfp4Dynamic) == (True, "")
|
||||
|
||||
moe.moe_parallel_config.use_ep = False
|
||||
supported, reason = _is_supported(moe, kNvfp4Static, kNvfp4Dynamic)
|
||||
assert not supported and "parallel config" in reason
|
||||
|
||||
|
||||
def test_megakernel_reports_its_own_constraints(megakernel_host):
|
||||
"""Constraints the framework does not know about surface as reasons."""
|
||||
_, reason = _is_supported(
|
||||
_megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL), kNvfp4Static, None
|
||||
)
|
||||
assert isinstance(dg.megakernel, _FakeDeepGemmMegaMoeConfig)
|
||||
assert dg.megakernel.intermediate_size == 2048
|
||||
assert dg.megakernel.top_k == 8
|
||||
assert dg.megakernel.activation_clamp == 7.0
|
||||
assert dg.preprocess_weights and dg.quantize_input
|
||||
assert "A16 activations" in reason
|
||||
|
||||
cd = build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=None,
|
||||
megakernel="nvfp4_cutedsl",
|
||||
deep_gemm = _megakernel_moe(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM, routing_method=RoutingMethodType.Renormalize
|
||||
)
|
||||
assert isinstance(cd.megakernel, _FakeNvfp4CutedslMegaMoeConfig)
|
||||
_, reason = _is_supported(deep_gemm, kMxfp4Static, None)
|
||||
assert "routing method" in reason
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported fi_moe_ep megakernel"):
|
||||
build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=None,
|
||||
megakernel="deep_gemm",
|
||||
megakernel_host.parallel_config.enable_eplb = True
|
||||
_, reason = _is_supported(
|
||||
_megakernel_moe(FLASHINFER_MOE_EP_DEEP_GEMM), kMxfp4Static, None
|
||||
)
|
||||
assert reason.endswith("EPLB")
|
||||
|
||||
|
||||
def test_megakernels_are_oracle_backends():
|
||||
"""Quant methods reach the megakernel through the NVFP4/MXFP4 oracles."""
|
||||
nvfp4 = map_nvfp4_backend(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert nvfp4 is NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
|
||||
assert nvfp4_backend_to_kernel_cls(nvfp4) == [FlashInferMoeEpExperts]
|
||||
|
||||
assert map_mxfp4_backend(FLASHINFER_MOE_EP_DEEP_GEMM) == [
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
]
|
||||
for backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
assert mxfp4_backend_to_kernel_cls(backend) == [FlashInferMoeEpExperts]
|
||||
# The megakernel takes the model's own dimensions.
|
||||
assert mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
backend, 2880, 2880
|
||||
) == (2880, 2880)
|
||||
|
||||
quant_config = make_mxfp4_moe_quant_config(
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL, w1_scale=None, w2_scale=None
|
||||
)
|
||||
assert quant_config is not None and quant_config.use_nvfp4_w4a4
|
||||
|
||||
|
||||
def test_megakernel_is_a_modular_kernel_with_pass_through_stages():
|
||||
"""The megakernel plugs into ``FusedMoEKernel`` like any other experts impl.
|
||||
|
||||
Routing stays outside (top-k ids in), dispatch and combine happen inside,
|
||||
so prepare passes tokens through untouched and finalize has nothing to do.
|
||||
"""
|
||||
moe = make_dummy_moe_config(
|
||||
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
|
||||
kernel = mk.FusedMoEKernel(
|
||||
PassThroughPrepareAndFinalize(),
|
||||
FlashInferMoeEpExperts(moe, quant_config),
|
||||
)
|
||||
|
||||
assert not kernel.is_monolithic
|
||||
assert kernel.prepare_finalize.output_is_reduced()
|
||||
# Generic stages: the int32 top-k ids the megakernel wants are its concern.
|
||||
assert kernel.prepare_finalize.topk_indices_dtype() is None
|
||||
assert kernel.fused_experts.expects_unquantized_inputs
|
||||
assert isinstance(
|
||||
kernel.fused_experts.finalize_weight_and_reduce_impl(), TopKWeightAndReduceNoOP
|
||||
)
|
||||
assert kernel.fused_experts.workspace_shapes(
|
||||
16, 256, 256, 2, 8, 8, None, MoEActivation.SILU
|
||||
) == ((0,), (0,), (16, 256))
|
||||
|
||||
tokens = torch.zeros(4, 256, dtype=torch.bfloat16)
|
||||
prepared = kernel.prepare_finalize.prepare(
|
||||
tokens,
|
||||
torch.ones(4, 2),
|
||||
torch.zeros(4, 2, dtype=torch.int32),
|
||||
8,
|
||||
None,
|
||||
False,
|
||||
quant_config,
|
||||
)
|
||||
assert prepared[0] is tokens and all(item is None for item in prepared[1:])
|
||||
|
||||
|
||||
def test_epilogue_takes_the_weight_global_scales_from_the_quant_config():
|
||||
"""The canonical ``g1/g2_alphas`` become the per-expert fc1/fc2 alphas and
|
||||
keep aliasing the layer's tensors, so EPLB permutations reach the kernel."""
|
||||
g1_alphas = torch.tensor([1.0, 2.0])
|
||||
g2_alphas = torch.tensor([3.0, 4.0])
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=g1_alphas,
|
||||
g2_alphas=g2_alphas,
|
||||
a1_gscale=torch.ones(2),
|
||||
a2_gscale=torch.ones(2),
|
||||
w1_scale=torch.ones(2, 4, 1),
|
||||
w2_scale=torch.ones(2, 2, 1),
|
||||
)
|
||||
|
||||
epilogue = epilogue_from_quant_config(quant_config)
|
||||
|
||||
assert epilogue.fc1_alpha is not None and epilogue.fc2_alpha is not None
|
||||
assert epilogue.fc1_alpha.data_ptr() == g1_alphas.data_ptr()
|
||||
assert epilogue.fc2_alpha.data_ptr() == g2_alphas.data_ptr()
|
||||
# Activations are quantized dynamically: no norm constants.
|
||||
assert epilogue.input_norm_const == 1.0 and epilogue.fc1_norm_const is None
|
||||
|
||||
|
||||
def test_epilogue_rejects_unloaded_weight_global_scales():
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=torch.tensor([1.0, float("nan")]),
|
||||
g2_alphas=torch.ones(2),
|
||||
a1_gscale=torch.ones(2),
|
||||
a2_gscale=torch.ones(2),
|
||||
w1_scale=torch.ones(2, 4, 1),
|
||||
w2_scale=torch.ones(2, 2, 1),
|
||||
)
|
||||
with pytest.raises(ValueError, match="g1_alphas"):
|
||||
epilogue_from_quant_config(quant_config)
|
||||
|
||||
|
||||
def test_mxfp4_checkpoints_keep_the_default_epilogue():
|
||||
"""MXFP4 has no per-tensor global scales; the megakernel needs no alphas."""
|
||||
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
|
||||
assert epilogue_from_quant_config(quant_config) == fi_ep.FlashInferMoeEpEpilogue()
|
||||
|
||||
|
||||
def test_nvfp4_kernel_format_passes_megakernel_weights_through():
|
||||
"""The megakernel takes the canonical tensors as they are; the oracle must
|
||||
not reject or reshape them."""
|
||||
tensors = tuple(torch.arange(4.0).reshape(2, 2) + i for i in range(8))
|
||||
converted = convert_to_nvfp4_moe_kernel_format(
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
SimpleNamespace(),
|
||||
*tensors,
|
||||
is_act_and_mul=True,
|
||||
)
|
||||
assert all(out is src for out, src in zip(converted, tensors))
|
||||
|
||||
|
||||
def test_passthrough_all2all_backend_selects_the_pass_through_stages():
|
||||
"""``all2all_backend="passthrough"`` is how the oracles get the
|
||||
pass-through prepare/finalize."""
|
||||
moe = make_dummy_moe_config(
|
||||
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
moe.moe_parallel_config = replace(
|
||||
moe.moe_parallel_config, all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND
|
||||
)
|
||||
assert not moe.moe_parallel_config.use_all2all_kernels
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
moe, FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
|
||||
)
|
||||
|
||||
assert isinstance(prepare_finalize, PassThroughPrepareAndFinalize)
|
||||
|
||||
|
||||
def test_passthrough_all2all_backend_is_not_a_cli_choice():
|
||||
"""Users cannot pick the passthrough all2all backend explicitly."""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["--all2all-backend", PASSTHROUGH_ALL2ALL_BACKEND])
|
||||
parser.parse_args(["--all2all-backend", "deepep_low_latency"])
|
||||
|
||||
|
||||
def test_megakernel_moe_backend_binds_the_all2all_backend():
|
||||
"""A megakernel deployment gets its all2all backend by default."""
|
||||
parallel = SimpleNamespace(all2all_backend="allgather_reducescatter")
|
||||
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
|
||||
assert parallel.all2all_backend == PASSTHROUGH_ALL2ALL_BACKEND
|
||||
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot be paired"):
|
||||
bind_passthrough_all2all_backend(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
SimpleNamespace(all2all_backend="deepep_low_latency"),
|
||||
)
|
||||
with pytest.raises(ValueError, match="only valid with"):
|
||||
bind_passthrough_all2all_backend(
|
||||
"flashinfer_trtllm",
|
||||
SimpleNamespace(all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND),
|
||||
)
|
||||
|
||||
|
||||
def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
|
||||
from vllm.models.deepseek_v4.nvidia.fi_moe import ckpt_uses_nvfp4_experts
|
||||
|
||||
nvfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo="NVFP4"))
|
||||
assert ckpt_uses_nvfp4_experts(nvfp4)
|
||||
|
||||
mxfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo=None))
|
||||
assert not ckpt_uses_nvfp4_experts(mxfp4)
|
||||
|
||||
no_algo = SimpleNamespace(quant_config=SimpleNamespace())
|
||||
assert not ckpt_uses_nvfp4_experts(no_algo)
|
||||
|
||||
|
||||
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
|
||||
"""One 32-element scale group per row: low nibble is the even element,
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group."""
|
||||
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
|
||||
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
|
||||
|
||||
out = _dequant_fp4_ue8m0_gran32(packed, sf)
|
||||
|
||||
assert out.shape == (2, 32)
|
||||
assert out.dtype == torch.bfloat16
|
||||
expected = torch.empty(2, 32)
|
||||
for row in range(2):
|
||||
for col in range(16):
|
||||
byte = int(packed[row, col])
|
||||
expected[row, 2 * col] = _E2M1_LUT[byte & 0x0F]
|
||||
expected[row, 2 * col + 1] = _E2M1_LUT[byte >> 4]
|
||||
expected[row] *= 2.0**row
|
||||
assert torch.equal(out, expected.to(torch.bfloat16))
|
||||
untouched = SimpleNamespace(all2all_backend="deepep_low_latency")
|
||||
bind_passthrough_all2all_backend("flashinfer_trtllm", untouched)
|
||||
assert untouched.all2all_backend == "deepep_low_latency"
|
||||
|
||||
@@ -657,7 +657,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.use_mega_moe = True
|
||||
moe.use_native_mega_moe = True
|
||||
moe.gate = FakeGate()
|
||||
moe.experts = FakeExperts()
|
||||
moe.shared_experts = FakeSharedExperts()
|
||||
|
||||
@@ -241,7 +241,7 @@ def test_v41_dspark_loads_linear_scales(
|
||||
linear_scale_name=scale_name,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [(runtime_name, param)],
|
||||
|
||||
@@ -185,17 +185,6 @@ def test_sinks(hf_runner: type[HfRunner], vllm_runner: type[VllmRunner]) -> None
|
||||
|
||||
|
||||
def test_mla(vllm_runner: type[VllmRunner], example_prompts: list[str]) -> None:
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
|
||||
installed = Version(transformers.__version__)
|
||||
required = Version("5.15.0.dev0")
|
||||
if installed < required:
|
||||
pytest.skip(
|
||||
"MLA models with the Transformers modeling backend require "
|
||||
f"transformers>={required}, but got {installed}"
|
||||
)
|
||||
|
||||
model = "hmellor/tiny-random-DeepseekV2ForCausalLM"
|
||||
args = (example_prompts, 32, 5)
|
||||
kwargs: dict[str, Any] = {"max_model_len": 2048, "enforce_eager": True}
|
||||
|
||||
@@ -609,17 +609,3 @@ def dummy_hf_overrides(
|
||||
)
|
||||
|
||||
return hf_config
|
||||
|
||||
|
||||
def check_transformers_version(
|
||||
model: str,
|
||||
min_transformers_version: str | None = None,
|
||||
max_transformers_version: str | None = None,
|
||||
):
|
||||
from .registry import _HfExamplesInfo
|
||||
|
||||
return _HfExamplesInfo(
|
||||
model,
|
||||
min_transformers_version=min_transformers_version,
|
||||
max_transformers_version=max_transformers_version,
|
||||
).check_transformers_version(on_fail="skip")
|
||||
|
||||
Reference in New Issue
Block a user