[Mini-VLLM] Repair test suite after SM86 cut
- Restore example connector, test helpers, and dropped patch tests - Guard MLA metadata import in the example connector - Point the PLE test at common.ops.ple after upstream #60021 move - Cut tests for removed features: tools, DRY, epd proxy, STT batch, diffusion scheduler, INC/Quark PLE cases, mamba-2 mixer Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,164 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.multimodal.image import rescale_image_size
|
||||
|
||||
from ...conftest import IMAGE_ASSETS, ImageTestAssets, VllmRunner
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
|
||||
{
|
||||
"stop_sign": "<|im_start|>User\n<image>\nWhat's the content in the center of the image?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
"cherry_blossom": "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
|
||||
}
|
||||
)
|
||||
|
||||
IMAGE_SIZE_FACTOR_GROUPS = (
|
||||
# Single-scale
|
||||
(1.0,),
|
||||
# Single-scale, batched
|
||||
(1.0, 1.0, 1.0),
|
||||
# Multi-scale
|
||||
(0.25, 0.5, 1.0),
|
||||
)
|
||||
|
||||
|
||||
def run_awq_test(
|
||||
vllm_runner: type[VllmRunner],
|
||||
image_assets: ImageTestAssets,
|
||||
source_model: str,
|
||||
quant_model: str,
|
||||
*,
|
||||
size_factor_groups: tuple[tuple[float, ...], ...],
|
||||
dtype: str,
|
||||
max_tokens: int,
|
||||
num_logprobs: int,
|
||||
tensor_parallel_size: int,
|
||||
distributed_executor_backend: str | None = None,
|
||||
):
|
||||
images = [asset.pil_image for asset in image_assets]
|
||||
|
||||
inputs_per_image_and_size_group = [
|
||||
(
|
||||
[prompt for _ in size_factors],
|
||||
[rescale_image_size(image, factor) for factor in size_factors],
|
||||
)
|
||||
for size_factors in size_factor_groups
|
||||
for image, prompt in zip(images, HF_IMAGE_PROMPTS)
|
||||
]
|
||||
|
||||
# NOTE: take care of the order. run vLLM first, and then run HF.
|
||||
# vLLM needs a fresh new process without cuda initialization.
|
||||
# if we run HF first, the cuda initialization will be done and it
|
||||
# will hurt multiprocessing backend with fork method (the default method).
|
||||
|
||||
# max_model_len should be greater than image_feature_size
|
||||
with vllm_runner(
|
||||
source_model,
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
source_outputs_per_image = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs=num_logprobs, images=images
|
||||
)
|
||||
for prompts, images in inputs_per_image_and_size_group
|
||||
]
|
||||
|
||||
with vllm_runner(
|
||||
quant_model,
|
||||
quantization="awq",
|
||||
max_model_len=4096,
|
||||
dtype=dtype,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
distributed_executor_backend=distributed_executor_backend,
|
||||
enforce_eager=True,
|
||||
default_torch_num_threads=1,
|
||||
) as vllm_model:
|
||||
quant_outputs_per_image = [
|
||||
vllm_model.generate_greedy_logprobs(
|
||||
prompts, max_tokens, num_logprobs=num_logprobs, images=images
|
||||
)
|
||||
for prompts, images in inputs_per_image_and_size_group
|
||||
]
|
||||
|
||||
for source_outputs, quant_outputs in zip(
|
||||
source_outputs_per_image, quant_outputs_per_image
|
||||
):
|
||||
# TODO: Check whether using original CLIPVisionModel can improve
|
||||
# consistency against HF
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=source_outputs,
|
||||
outputs_1_lst=quant_outputs,
|
||||
name_0="source",
|
||||
name_1="awq",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "quantization", "dtype"),
|
||||
[
|
||||
("mattbucci/gemma-4-26B-AWQ", "awq", "float16"),
|
||||
("cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit", "compressed-tensors", "bfloat16"),
|
||||
],
|
||||
ids=[
|
||||
"gemma4-moe-standard-awq-dot-suffix",
|
||||
"gemma4-moe-compressed-tensors-underscore-suffix",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_awq_load(
|
||||
vllm_runner: type[VllmRunner],
|
||||
example_prompts: list[str],
|
||||
model: str,
|
||||
quantization: str,
|
||||
dtype: str,
|
||||
) -> None:
|
||||
"""Regression test: AWQ weight loading must not KeyError."""
|
||||
with vllm_runner(
|
||||
model,
|
||||
quantization=quantization,
|
||||
dtype=dtype,
|
||||
max_model_len=128,
|
||||
enforce_eager=True,
|
||||
) as vllm_model:
|
||||
outputs = vllm_model.generate_greedy(example_prompts[:2], max_tokens=32)
|
||||
assert len(outputs) == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("source_model", "quant_model"),
|
||||
[("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("max_tokens", [128])
|
||||
@pytest.mark.parametrize("num_logprobs", [5])
|
||||
@torch.inference_mode()
|
||||
def test_awq_models(
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
source_model,
|
||||
quant_model,
|
||||
dtype,
|
||||
max_tokens,
|
||||
num_logprobs,
|
||||
) -> None:
|
||||
run_awq_test(
|
||||
vllm_runner,
|
||||
image_assets,
|
||||
source_model,
|
||||
quant_model,
|
||||
size_factor_groups=IMAGE_SIZE_FACTOR_GROUPS,
|
||||
dtype=dtype,
|
||||
max_tokens=max_tokens,
|
||||
num_logprobs=num_logprobs,
|
||||
tensor_parallel_size=1,
|
||||
)
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E tests for online FP8 per-channel quantization.
|
||||
|
||||
Loads a BF16 model with ``--quantization fp8_per_channel`` (online
|
||||
quantization) and compares log-probabilities against the same model served in
|
||||
BF16 without quantization. This exercises the full pipeline: config parsing,
|
||||
``Fp8PtpcOnlineLinearMethod``, ``Fp8PtpcOnlineMoEMethod``, weight
|
||||
loading, online quantization / shuffling, and inference.
|
||||
|
||||
``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
|
||||
diverse prompts from ``tests/prompts/example.txt``.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
|
||||
from ..utils import check_logprobs_close
|
||||
|
||||
# Small MoE model that fits on a single GPU and exercises both linear + MoE.
|
||||
MOE_MODEL = "allenai/OLMoE-1B-7B-0125-Instruct"
|
||||
# Small dense model (no MoE) to validate the linear-only path.
|
||||
DENSE_MODEL = "Qwen/Qwen3-0.6B"
|
||||
|
||||
MAX_MODEL_LEN = 1024
|
||||
MAX_TOKENS = 4
|
||||
NUM_LOG_PROBS = 8
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="fp8 is not supported on this GPU type.",
|
||||
)
|
||||
@pytest.mark.quant_model
|
||||
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
|
||||
def test_fp8_per_channel_logprobs(
|
||||
vllm_runner,
|
||||
quant_baseline_logprobs,
|
||||
example_prompts,
|
||||
model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Compare BF16 baseline logprobs against online per-channel-quantized
|
||||
model.
|
||||
|
||||
Runs the same model twice -- once in BF16 (baseline) and once with online
|
||||
FP8 per-channel quantization -- then checks that the top log-probabilities
|
||||
are close. Only 4 tokens are generated to keep the test fast while still
|
||||
catching numerical divergence beyond expected per-channel error.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
|
||||
baseline_outputs = quant_baseline_logprobs(
|
||||
model,
|
||||
example_prompts,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
max_tokens=MAX_TOKENS,
|
||||
num_logprobs=NUM_LOG_PROBS,
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model,
|
||||
max_model_len=MAX_MODEL_LEN,
|
||||
enforce_eager=True,
|
||||
quantization="fp8_per_channel",
|
||||
) as vllm_model:
|
||||
test_outputs = vllm_model.generate_greedy_logprobs(
|
||||
example_prompts, MAX_TOKENS, NUM_LOG_PROBS
|
||||
)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline_outputs,
|
||||
outputs_1_lst=test_outputs,
|
||||
name_0="bf16",
|
||||
name_1="fp8_per_channel",
|
||||
)
|
||||
@@ -1,176 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Routing support for the AITER Triton MXFP4 W4A16 MoE backend.
|
||||
|
||||
MiMo-V2.6 uses an ungrouped sigmoid router with a per-expert correction bias
|
||||
(``scoring_func=sigmoid``, ``topk_method=noaux_tc``, ``n_group == 1``), which
|
||||
``get_routing_method_type`` classifies as ``RoutingMethodType.DeepSeekV3``.
|
||||
``AiterW4A16ExpertsMonolithic`` used to reject that router ("kernel does not
|
||||
support routing method ..."), and on gfx942 it is the only MXFP4 kernel whose
|
||||
device gate accepts the card, so the checkpoint could not be served natively at
|
||||
all.
|
||||
|
||||
aiter's *flat* top-k handles that router natively (``score_mode="sigmoid"`` plus
|
||||
a correction bias), so the backend only has to ask for it. This module covers
|
||||
both halves: the gate accepts the router family, and the routing the wrapper
|
||||
requests reproduces the model's own math.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
get_routing_method_type,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
|
||||
MIMO_EXPERTS = 256 # MiMo-V2.6-Flash
|
||||
MIMO_PRO_EXPERTS = 384 # MiMo-V2.6-Pro
|
||||
MIMO_TOP_K = 8
|
||||
|
||||
|
||||
def mimo_routing_method() -> RoutingMethodType:
|
||||
"""MiMo-V2.6: sigmoid + correction bias, a single expert group."""
|
||||
return get_routing_method_type(
|
||||
scoring_func="sigmoid",
|
||||
top_k=MIMO_TOP_K,
|
||||
renormalize=True,
|
||||
num_expert_group=1,
|
||||
has_e_score_bias=True,
|
||||
)
|
||||
|
||||
|
||||
def mimo_moe_config(num_experts: int = MIMO_EXPERTS) -> FusedMoEConfig:
|
||||
return FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=MIMO_TOP_K,
|
||||
hidden_dim=4096,
|
||||
intermediate_size=2048,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
activation=MoEActivation.SILU,
|
||||
in_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
routing_method=mimo_routing_method(),
|
||||
router_logits_dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
def test_ungrouped_sigmoid_router_is_deepseekv3() -> None:
|
||||
"""Guard the premise of the fix: MiMo's router is DeepSeekV3-classified."""
|
||||
assert mimo_routing_method() == RoutingMethodType.DeepSeekV3
|
||||
|
||||
|
||||
def test_aiter_w4a16_supports_mimo_routing() -> None:
|
||||
"""The gate accepts MiMo's router family — the reported construction abort.
|
||||
|
||||
Only the predicate the fix touches is exercised, so this runs on any host;
|
||||
the full config path is covered by the test below.
|
||||
"""
|
||||
assert AiterW4A16ExpertsMonolithic._supports_routing_method(
|
||||
RoutingMethodType.DeepSeekV3, kMxfp4Static, None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_aiter_w4a16_supports_mimo_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The same gate through the real config path, device check included."""
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
supported, reason = AiterW4A16ExpertsMonolithic.is_supported_config(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
mimo_moe_config(),
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [MIMO_EXPERTS, MIMO_PRO_EXPERTS])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="only runs on ROCm with a supported AITER install",
|
||||
)
|
||||
def test_flat_sigmoid_routing_matches_mimo_reference(num_experts: int) -> None:
|
||||
"""The routing the wrapper requests must equal MiMo's own router math.
|
||||
|
||||
MiMo's router is ungrouped: sigmoid scores, selection on
|
||||
``sigmoid(logits) + bias``, weights = the *unbiased* sigmoid scores, then
|
||||
renormalized. Both checkpoints use it — 256 experts is Flash, 384 is Pro.
|
||||
|
||||
The ``sigmoid`` score mode comes from ROCm/aiter#4688 (aiter >= 0.1.20).
|
||||
"""
|
||||
aiter_routing = pytest.importorskip(
|
||||
"aiter.ops.triton.moe.moe_routing.routing"
|
||||
).routing
|
||||
|
||||
torch.manual_seed(0)
|
||||
top_k, num_tokens = MIMO_TOP_K, 16
|
||||
logits = torch.randn(num_tokens, num_experts, device="cuda") * 2.0
|
||||
bias = torch.randn(num_experts, device="cuda") * 0.5
|
||||
|
||||
# Reference: select on the biased score, return the unbiased score, renorm.
|
||||
scores = torch.sigmoid(logits.float())
|
||||
ref_ids = (scores + bias).topk(top_k, dim=-1).indices
|
||||
ref_weights = scores.gather(1, ref_ids)
|
||||
ref_weights = ref_weights / ref_weights.sum(dim=-1, keepdim=True)
|
||||
|
||||
def routed(score_mode: str) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
routing_data, topk_indx, _ = aiter_routing(
|
||||
logits,
|
||||
top_k,
|
||||
score_mode=score_mode,
|
||||
bias=bias,
|
||||
renorm=True,
|
||||
routed_scaling_factor=1.0,
|
||||
use_grouped_topk=False,
|
||||
)
|
||||
# `topk_indx[i]` is the token-slot index (token * top_k + slot) of the
|
||||
# i-th entry of the expert-sorted array whose weights are gate_scal.
|
||||
hist = routing_data.expt_hist.to(torch.long).cpu()
|
||||
expert_of_slot = torch.repeat_interleave(torch.arange(hist.numel()), hist)
|
||||
slot = topk_indx.reshape(-1).to(torch.long).cpu()
|
||||
weights = routing_data.gate_scal.reshape(-1).to(torch.float32).cpu()
|
||||
ids = torch.full((num_tokens, top_k), -1, dtype=torch.long)
|
||||
out = torch.zeros(num_tokens, top_k)
|
||||
for i in range(slot.numel()):
|
||||
token, position = int(slot[i]) // top_k, int(slot[i]) % top_k
|
||||
ids[token, position] = int(expert_of_slot[i])
|
||||
out[token, position] = float(weights[i])
|
||||
return ids, out
|
||||
|
||||
got_ids, got_weights = routed("sigmoid")
|
||||
|
||||
# aiter's triton top-k emits each token's selection sorted by *ascending
|
||||
# expert id*, while the reference is in descending ``sigmoid(logits)+bias``
|
||||
# rank order — align both sides on the expert id so the weight check
|
||||
# becomes elementwise and still validates the expert<->weight pairing.
|
||||
def align(ids: torch.Tensor, w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
order = torch.argsort(ids, dim=-1)
|
||||
return ids.gather(1, order), w.gather(1, order)
|
||||
|
||||
got_ids, got_weights = align(got_ids, got_weights)
|
||||
ref_ids, ref_weights = align(ref_ids.cpu(), ref_weights.cpu())
|
||||
assert torch.equal(got_ids, ref_ids)
|
||||
assert torch.allclose(got_weights, ref_weights, atol=1e-3)
|
||||
|
||||
# Control: the correction bias must steer selection only. Ignoring it would
|
||||
# pick from the plain sigmoid scores instead.
|
||||
plain_ids = scores.topk(top_k, dim=-1).indices
|
||||
assert not torch.equal(torch.sort(plain_ids.cpu(), dim=-1).values, ref_ids)
|
||||
Reference in New Issue
Block a user