[Mini-VLLM] Repair test suite after SM86 cut

- Restore example connector, test helpers, and dropped patch tests
- Guard MLA metadata import in the example connector
- Point the PLE test at common.ops.ple after upstream #60021 move
- Cut tests for removed features: tools, DRY, epd proxy, STT batch,
  diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 01:06:30 +07:00
co-authored by Hinata
parent 0152af1336
commit 54a396586e
112 changed files with 55726 additions and 10371 deletions
-164
View File
@@ -1,164 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.multimodal.image import rescale_image_size
from ...conftest import IMAGE_ASSETS, ImageTestAssets, VllmRunner
from ..utils import check_logprobs_close
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
{
"stop_sign": "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
"cherry_blossom": "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
}
)
IMAGE_SIZE_FACTOR_GROUPS = (
# Single-scale
(1.0,),
# Single-scale, batched
(1.0, 1.0, 1.0),
# Multi-scale
(0.25, 0.5, 1.0),
)
def run_awq_test(
vllm_runner: type[VllmRunner],
image_assets: ImageTestAssets,
source_model: str,
quant_model: str,
*,
size_factor_groups: tuple[tuple[float, ...], ...],
dtype: str,
max_tokens: int,
num_logprobs: int,
tensor_parallel_size: int,
distributed_executor_backend: str | None = None,
):
images = [asset.pil_image for asset in image_assets]
inputs_per_image_and_size_group = [
(
[prompt for _ in size_factors],
[rescale_image_size(image, factor) for factor in size_factors],
)
for size_factors in size_factor_groups
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
]
# NOTE: take care of the order. run vLLM first, and then run HF.
# vLLM needs a fresh new process without cuda initialization.
# if we run HF first, the cuda initialization will be done and it
# will hurt multiprocessing backend with fork method (the default method).
# max_model_len should be greater than image_feature_size
with vllm_runner(
source_model,
max_model_len=4096,
dtype=dtype,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=True,
default_torch_num_threads=1,
) as vllm_model:
source_outputs_per_image = [
vllm_model.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs=num_logprobs, images=images
)
for prompts, images in inputs_per_image_and_size_group
]
with vllm_runner(
quant_model,
quantization="awq",
max_model_len=4096,
dtype=dtype,
tensor_parallel_size=tensor_parallel_size,
distributed_executor_backend=distributed_executor_backend,
enforce_eager=True,
default_torch_num_threads=1,
) as vllm_model:
quant_outputs_per_image = [
vllm_model.generate_greedy_logprobs(
prompts, max_tokens, num_logprobs=num_logprobs, images=images
)
for prompts, images in inputs_per_image_and_size_group
]
for source_outputs, quant_outputs in zip(
source_outputs_per_image, quant_outputs_per_image
):
# TODO: Check whether using original CLIPVisionModel can improve
# consistency against HF
check_logprobs_close(
outputs_0_lst=source_outputs,
outputs_1_lst=quant_outputs,
name_0="source",
name_1="awq",
)
@pytest.mark.parametrize(
("model", "quantization", "dtype"),
[
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
],
ids=[
"gemma4-moe-standard-awq-dot-suffix",
"gemma4-moe-compressed-tensors-underscore-suffix",
],
)
@torch.inference_mode()
def test_awq_load(
vllm_runner: type[VllmRunner],
example_prompts: list[str],
model: str,
quantization: str,
dtype: str,
) -> None:
"""Regression test: AWQ weight loading must not KeyError."""
with vllm_runner(
model,
quantization=quantization,
dtype=dtype,
max_model_len=128,
enforce_eager=True,
) as vllm_model:
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
assert len(outputs) == 2
@pytest.mark.parametrize(
("source_model", "quant_model"),
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
)
@pytest.mark.parametrize("dtype", ["half"])
@pytest.mark.parametrize("max_tokens", [128])
@pytest.mark.parametrize("num_logprobs", [5])
@torch.inference_mode()
def test_awq_models(
vllm_runner,
image_assets,
source_model,
quant_model,
dtype,
max_tokens,
num_logprobs,
) -> None:
run_awq_test(
vllm_runner,
image_assets,
source_model,
quant_model,
size_factor_groups=IMAGE_SIZE_FACTOR_GROUPS,
dtype=dtype,
max_tokens=max_tokens,
num_logprobs=num_logprobs,
tensor_parallel_size=1,
)
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E tests for online FP8 per-channel quantization.
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
quantization) and compares log-probabilities against the same model served in
BF16 without quantization. This exercises the full pipeline: config parsing,
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
loading, online quantization / shuffling, and inference.
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
diverse prompts from ``tests/prompts/example.txt``.
"""
import pytest
from tests.quantization.utils import is_quant_method_supported
from ..utils import check_logprobs_close
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
# Small dense model (no MoE) to validate the linear-only path.
DENSE_MODEL = "Qwen/Qwen3-0.6B"
MAX_MODEL_LEN = 1024
MAX_TOKENS = 4
NUM_LOG_PROBS = 8
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="fp8 is not supported on this GPU type.",
)
@pytest.mark.quant_model
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
def test_fp8_per_channel_logprobs(
vllm_runner,
quant_baseline_logprobs,
example_prompts,
model: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Compare BF16 baseline logprobs against online per-channel-quantized
model.
Runs the same model twice -- once in BF16 (baseline) and once with online
FP8 per-channel quantization -- then checks that the top log-probabilities
are close. Only 4 tokens are generated to keep the test fast while still
catching numerical divergence beyond expected per-channel error.
"""
with monkeypatch.context() as m:
m.setenv("TOKENIZERS_PARALLELISM", "true")
baseline_outputs = quant_baseline_logprobs(
model,
example_prompts,
max_model_len=MAX_MODEL_LEN,
max_tokens=MAX_TOKENS,
num_logprobs=NUM_LOG_PROBS,
)
with vllm_runner(
model,
max_model_len=MAX_MODEL_LEN,
enforce_eager=True,
quantization="fp8_per_channel",
) as vllm_model:
test_outputs = vllm_model.generate_greedy_logprobs(
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
)
check_logprobs_close(
outputs_0_lst=baseline_outputs,
outputs_1_lst=test_outputs,
name_0="bf16",
name_1="fp8_per_channel",
)
@@ -1,176 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
device gate accepts the card, so the checkpoint could not be served natively at
all.
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
a correction bias), so the backend only has to ask for it. This module covers
both halves: the gate accepts the router family, and the routing the wrapper
requests reproduces the model's own math.
"""
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
get_routing_method_type,
)
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
MIMO_TOP_K = 8
def mimo_routing_method() -> RoutingMethodType:
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
return get_routing_method_type(
scoring_func="sigmoid",
top_k=MIMO_TOP_K,
renormalize=True,
num_expert_group=1,
has_e_score_bias=True,
)
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
return FusedMoEConfig(
num_experts=num_experts,
experts_per_token=MIMO_TOP_K,
hidden_dim=4096,
intermediate_size=2048,
num_local_experts=num_experts,
num_logical_experts=num_experts,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
activation=MoEActivation.SILU,
in_dtype=torch.bfloat16,
device=torch.device("cuda"),
routing_method=mimo_routing_method(),
router_logits_dtype=torch.float32,
)
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
def test_aiter_w4a16_supports_mimo_routing() -> None:
"""The gate accepts MiMo's router family — the reported construction abort.
Only the predicate the fix touches is exercised, so this runs on any host;
the full config path is covered by the test below.
"""
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
)
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="only runs on ROCm with a supported AITER install",
)
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
"""The same gate through the real config path, device check included."""
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
AiterW4A16ExpertsMonolithic,
mimo_moe_config(),
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="only runs on ROCm with a supported AITER install",
)
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
"""The routing the wrapper requests must equal MiMo's own router math.
MiMo's router is ungrouped: sigmoid scores, selection on
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
"""
aiter_routing = pytest.importorskip(
"aiter.ops.triton.moe.moe_routing.routing"
).routing
torch.manual_seed(0)
top_k, num_tokens = MIMO_TOP_K, 16
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
bias = torch.randn(num_experts, device="cuda") * 0.5
# Reference: select on the biased score, return the unbiased score, renorm.
scores = torch.sigmoid(logits.float())
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
ref_weights = scores.gather(1, ref_ids)
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
routing_data, topk_indx, _ = aiter_routing(
logits,
top_k,
score_mode=score_mode,
bias=bias,
renorm=True,
routed_scaling_factor=1.0,
use_grouped_topk=False,
)
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
# i-th entry of the expert-sorted array whose weights are gate_scal.
hist = routing_data.expt_hist.to(torch.long).cpu()
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
slot = topk_indx.reshape(-1).to(torch.long).cpu()
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
out = torch.zeros(num_tokens, top_k)
for i in range(slot.numel()):
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
ids[token, position] = int(expert_of_slot[i])
out[token, position] = float(weights[i])
return ids, out
got_ids, got_weights = routed("sigmoid")
# aiter's triton top-k emits each token's selection sorted by *ascending
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
# rank order — align both sides on the expert id so the weight check
# becomes elementwise and still validates the expert<->weight pairing.
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
order = torch.argsort(ids, dim=-1)
return ids.gather(1, order), w.gather(1, order)
got_ids, got_weights = align(got_ids, got_weights)
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
assert torch.equal(got_ids, ref_ids)
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
# Control: the correction bias must steer selection only. Ignoring it would
# pick from the plain sigmoid scores instead.
plain_ids = scores.topk(top_k, dim=-1).indices
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)