Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
@@ -148,9 +148,7 @@ class Config:
return self.E // self.world_size
def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]:
"""
make env data for vllm launch.
"""
"""Make env data for vllm launch."""
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
@@ -169,7 +167,8 @@ class Config:
def fe_supports_quant_scheme(self) -> bool:
"""Check if the fused experts class supports this quant config.
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.
"""
if self.quant_config is None or self.quant_dtype is None:
return True
if self.quant_dtype != torch.float8_e4m3fn:
@@ -444,9 +443,7 @@ class RankTensors:
def make_hidden_states(
config: Config,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""
Return hidden_states
"""
"""Return hidden_states."""
m, k, dtype = (config.M, config.K, config.dtype)
device = torch.accelerator.current_device_index()
a = torch.randn((m, k), device=device, dtype=dtype) / 15.0
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DeepEP test utilities
"""
"""DeepEP test utilities."""
import dataclasses
import os
+2 -1
View File
@@ -468,7 +468,8 @@ def test_deepseek_v4_flashinfer_cutlass_falls_through_to_w4a8(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Explicit flashinfer_cutlass must try the W4A8 variant when the BF16
variant is unsupported (the BF16 variant is gated to SM90)."""
variant is unsupported (the BF16 variant is gated to SM90).
"""
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
@@ -34,7 +34,6 @@ def test_batched_deepgemm_vs_triton(
E: int, T: int, K: int, N: int, topk: int, monkeypatch, workspace_init
):
"""Compare BatchedDeepGemmExperts to BatchedTritonExperts."""
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1")
device = "cuda"
+6 -3
View File
@@ -129,7 +129,8 @@ def test_batched_mm(
per_act_token_quant: bool,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
and those tests will be skipped on unsupported hardware.
"""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
@@ -268,7 +269,8 @@ def test_fused_moe_batched_experts(
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
and those tests will be skipped on unsupported hardware.
"""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
@@ -649,7 +651,8 @@ def test_batched_experts_masked_activation_end_to_end(
def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
the device-enablement path.
"""
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
+4 -2
View File
@@ -54,7 +54,8 @@ SEEDS = [0]
# For test
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
"""This function performs fused moe with block-wise quantization using
native torch."""
native torch.
"""
B, D = a.shape
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
@@ -97,7 +98,8 @@ def setup_cuda():
@torch.inference_mode()
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
native torch reference."""
native torch reference.
"""
torch.manual_seed(seed)
a = torch.randn((M, K), dtype=dtype) / 10
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests compute_expert_num_tokens kernels
"""
"""Tests compute_expert_num_tokens kernels."""
import dataclasses
+4 -3
View File
@@ -397,7 +397,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
class _StubMoELayer(torch.nn.Module):
"""Minimal stand-in for the real MoE layer module, exposing just what
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
router configuration, and optionally w13_bias/w2_bias)."""
router configuration, and optionally w13_bias/w2_bias).
"""
def __init__(
self,
@@ -500,8 +501,8 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
act: MoEActivation,
):
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
weights before prepacking."""
weights before prepacking.
"""
set_random_seed(0)
batch_size = 64
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
+1
View File
@@ -62,6 +62,7 @@ def _make_int4_moe_weights(
Returns:
(w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias)
where *_ref are the dequantized float reference weights
"""
# Generate INT4 weights as int8 values in [-8, 7]
w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8)
@@ -826,6 +826,7 @@ def _make_int4_moe_weights(E, N, K, group_size, quant_algo):
w1_zeros, w2_zeros,
w1_zeros_packed, w2_zeros_packed,
w1_s, w2_s
"""
w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32)
w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32)
+3 -5
View File
@@ -91,11 +91,9 @@ def break_fp4_bytes(a, dtype):
def generate_balanced_routing(
hidden_states: torch.Tensor, num_experts: int, top_k: int
):
"""
Generate routing weights and topk indices such that every expert is active.
"""Generate routing weights and topk indices such that every expert is active.
Returns routing_weights, topk_idx
"""
num_tokens, hidden_dim = hidden_states.shape
# num_tokens = batch_size * seq_len
@@ -232,13 +230,13 @@ def grouped_gemm_ref(
*,
block_size: int = 16,
) -> torch.Tensor:
"""
Computes the reference grouped GEMM (fp4 quantized per-expert loop),
"""Computes the reference grouped GEMM (fp4 quantized per-expert loop),
computes flashinfer grouped GEMM (for scale consistency),
and returns ONLY the repacked reference output: out_ref.
Returns:
out_ref: Tensor [num_experts, max_m, n_out]
"""
device_hs = hidden_states_expanded.device
device_w = weights.device
+4 -12
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test DeepEP + DeepGEMM integration
"""Test DeepEP + DeepGEMM integration
DeepGEMM are gemm kernels specialized for the
fp8 block-quantized case.
"""
@@ -92,9 +91,7 @@ def make_block_quant_fp8_weights(
k: int,
block_size: list[int],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Return weights w1q, w2q, w1_scale, w2_scale
"""
"""Return weights w1q, w2q, w1_scale, w2_scale."""
(_, w1q, w1_scale, _), (_, w2q, w2_scale, _) = make_test_weights(
e, n, k, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
)
@@ -459,10 +456,7 @@ def test_ht_deepep_deepgemm_moe(
disable_deepgemm_ue8m0,
workspace_init,
):
"""
Tests for High-Throughput DeepEP + DeepGemm integration.
"""
"""Tests for High-Throughput DeepEP + DeepGemm integration."""
m, n, k = mnk
set_random_seed(7)
@@ -533,9 +527,7 @@ def test_ll_deepep_deepgemm_moe(
disable_deepgemm_ue8m0,
workspace_init,
):
"""
Tests for Low-Latency DeepEP + DeepGemm integration.
"""
"""Tests for Low-Latency DeepEP + DeepGemm integration."""
assert not is_deep_gemm_e8m0_used()
m, n, k = mnk
+2 -6
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test deepep dispatch-combine logic
"""
"""Test deepep dispatch-combine logic."""
import dataclasses
@@ -56,9 +54,7 @@ MAX_TOKENS_PER_RANK = 64
def make_weights(
e, n, k, dtype
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Return weights w1, w2, w1_scale, w2_scale
"""
"""Return weights w1, w2, w1_scale, w2_scale."""
if dtype in [torch.float16, torch.bfloat16]:
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
Compares against a pure-PyTorch reference MoE implementation.
"""
+5 -10
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
@@ -79,8 +78,7 @@ def make_block_quant_fp8_weights(
k: int,
block_size: list[int],
):
"""
Generate (w1, w2) expert weights and their per-block scale tensors
"""Generate (w1, w2) expert weights and their per-block scale tensors
in FP8 block-quantized format.
w1 shape: (E, 2N, K)
@@ -121,8 +119,7 @@ def make_block_quant_fp8_weights(
def run_single_case(m, n, k, topk, num_experts, block_size):
"""
Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
Triton baseline within tolerance.
"""
tokens_bf16 = (
@@ -248,8 +245,7 @@ def make_mxfp4_weights(
n: int,
k: int,
):
"""
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
@@ -310,8 +306,7 @@ def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
def run_single_fp4_case(m, n, k, topk, num_experts):
"""
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
+2 -1
View File
@@ -464,7 +464,8 @@ def test_trtllm_fp8_swiglu_clamp_support(
):
"""FlashInfer >= 0.6.18 applies the SwiGLU clamp for both block-scaled
kernels with a SwiGLU activation (DeepSeek-V4 sets swiglu_limit); the
per-tensor kernel has no clamp, and Relu2 rejects the parameters."""
per-tensor kernel has no clamp, and Relu2 rejects the parameters.
"""
class _Experts(TrtLlmFp8ExpertsModular):
@staticmethod
@@ -83,7 +83,7 @@ def _reference_activation(
beta: float | None,
limit: float | None,
):
"""vLLM's own op for this activation, so the reference is not re-derived."""
"""VLLM's own op for this activation, so the reference is not re-derived."""
if activation == MoEActivation.RELU2_NO_MUL:
return lambda x: torch.square(torch.relu(x))
if activation == MoEActivation.SWIGLUOAI:
+1 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoE fused topk kernel
"""Tests for the MoE fused topk kernel.
Run `pytest tests/kernels/moe/test_fused_topk.py`.
"""
@@ -377,7 +377,8 @@ def test_unit_shuffle():
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
"""routing_data_from_sparse_topk must produce routing structures
identical to make_routing_data for the same topk result."""
identical to make_routing_data for the same topk result.
"""
from vllm.model_executor.layers.fused_moe.experts import (
gpt_oss_triton_kernels_moe as gptoss_moe,
)
+1 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoE grouped topk kernel
"""Tests for the MoE grouped topk kernel.
Run `pytest tests/kernels/moe/test_grouped_topk.py`.
"""
@@ -31,6 +31,7 @@ def mxint4_quantize(
Returns:
- uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7]
- scales in BF16: [..., k//sf_vec_size]
"""
x_reshaped = x.reshape(-1, sf_vec_size)
x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32)
@@ -61,6 +62,7 @@ def mxint4_quantize_moe_weights(
Returns:
- weights_mxint4: Quantized weights [e, n, k//2] uint8
- scales_mxint4: Quantization scales [e, n, k//group_size] bf16
"""
e = weights_bf16.shape[0]
weight_list = []
@@ -101,6 +103,7 @@ def marlin_quantize_moe_weights(
Returns:
- weights_marlin: Marlin quantized weights [e, k//8, n] int32
- scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16
"""
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
marlin_quantize,
@@ -286,7 +289,8 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
@torch.inference_mode()
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
trtllm_mxint4_block_scale_moe kernel call."""
trtllm_mxint4_block_scale_moe kernel call.
"""
pytest.importorskip("flashinfer")
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
@@ -352,7 +352,8 @@ def test_modular_kernel_combinations_singlegpu(
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
and those tests will be skipped on unsupported hardware.
"""
config = Config(
Ms=Ms,
K=k,
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test modular OAI Triton MoE
"""
"""Test modular OAI Triton MoE."""
from __future__ import annotations
+4 -7
View File
@@ -119,8 +119,7 @@ def iterative_moe(
expert_map: torch.Tensor = None,
renormalize: bool = False,
) -> torch.Tensor:
"""
Baseline implementation of fused moe.
"""Baseline implementation of fused moe.
Args:
hidden_states: [*, hidden_size]
@@ -128,6 +127,7 @@ def iterative_moe(
w2: [num_experts, hidden_size, intermediate_size]
gating_output: [*, num_experts]
expert_map: [num_experts]
"""
orig_shape = hidden_states.shape
hidden_size = hidden_states.shape[-1]
@@ -1956,8 +1956,7 @@ def test_batched_fused_marlin_moe(
)
def is_valid(self):
"""
Return True only if the input can be represented in a Batched
"""Return True only if the input can be represented in a Batched
format.
"""
return torch.all(self.expert_num_tokens_cpu <= self.max_tokens_per_batch)
@@ -2078,9 +2077,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
dtype: torch.dtype,
workspace_init,
):
"""
Test BF16 unquantized MoE with FlashInfer TRTLLM backend.
"""
"""Test BF16 unquantized MoE with FlashInfer TRTLLM backend."""
set_random_seed(7)
from vllm.model_executor.layers.fused_moe.config import (
@@ -54,8 +54,7 @@ def _verify_expert_level_sorting(
valid_length: int,
total_tokens: int,
):
"""
Verify that actual_sorted_ids follows the correct expert-level sorting.
"""Verify that actual_sorted_ids follows the correct expert-level sorting.
The kerne limplementation may or may not preserve original token order
in topk_ids in the final sorted_ids however this does not impact quality.
"""
@@ -96,8 +95,7 @@ def torch_moe_align_block_size(
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Golden torch implementation of moe_align_block_size.
"""Golden torch implementation of moe_align_block_size.
This function aligns the token distribution across experts to be compatible
with block size for matrix multiplication by sorting tokens by expert and
@@ -189,7 +187,7 @@ def torch_moe_align_block_size(
def test_moe_align_block_size(
m: int, topk: int, num_experts: int, block_size: int, pad_sorted_ids: bool
):
"""Test moe_align_block_size without expert mapping"""
"""Test moe_align_block_size without expert mapping."""
topk_ids = torch.zeros((m, topk), device="cuda", dtype=torch.int32)
for i in range(m):
experts = torch.randperm(num_experts, device="cuda")[:topk]
@@ -254,7 +252,7 @@ def test_moe_align_block_size_with_expert_map(
block_size: int,
mask_inactive_experts: bool,
):
"""Test moe_align_block_size with expert mapping (EP scenario)"""
"""Test moe_align_block_size with expert mapping (EP scenario)."""
expert_map = torch.full((num_experts,), -1, device="cuda", dtype=torch.int32)
local_experts = list(range(0, num_experts, 2))
for i, expert_id in enumerate(local_experts):
+2 -1
View File
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
class TestUnquantizedDelegation:
"""UnquantizedMoEKernelOracle methods must delegate to the existing
module-level functions; behaviour is bit-identical."""
module-level functions; behaviour is bit-identical.
"""
def test_make_kernel_delegates(self) -> None:
quant_config = object()
+6 -4
View File
@@ -240,8 +240,7 @@ def maybe_roundup_layer_hidden_size(
act_dtype: torch.dtype,
backend: str | None,
) -> int:
"""
Given layer hidden size and MoE configurations, round up hidden_size
"""Given layer hidden size and MoE configurations, round up hidden_size
if necessary.
Args:
@@ -253,6 +252,7 @@ def maybe_roundup_layer_hidden_size(
Rounded up hidden_size if rounding up is required based on the configs
and all2all backend.
Original hidden size otherwise.
"""
if backend == "deepep_high_throughput":
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
@@ -315,7 +315,8 @@ def tp_chunk_gate_up(
device: torch.device | int | None = None,
) -> torch.Tensor:
"""TP-chunk a combined [gate; up] weight, splitting each half separately
so every rank gets a portion of both gate and up."""
so every rank gets a portion of both gate and up.
"""
half = w.shape[dim] // 2
gate = chunk_by_rank(
w.narrow(dim, 0, half), tp_rank, tp_size, dim=dim, device=device
@@ -945,6 +946,7 @@ def create_shared_experts_from_config(
Returns:
TestMLP instance or None if config is None
"""
if shared_experts_config is None:
return None
@@ -991,6 +993,7 @@ def setup_moe_test_data(
Returns:
MoETestData containing all test data and transforms
"""
# For latent MoE: latent_size = k // 2
latent_size = k // 2
@@ -1710,7 +1713,6 @@ def test_moe_layer_no_parallel(
monkeypatch,
):
"""Test MoE layer without parallelism (dp_size=1, tp_size=1, use_ep=False)."""
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MOE permute/unpermute kernel
"""Tests for the MOE permute/unpermute kernel.
Run `pytest tests/kernels/test_moe_permute_unpermute.py`.
"""
+4 -9
View File
@@ -67,8 +67,7 @@ def compute_ref_output(
@pytest.mark.parametrize("num_experts", [8, 16, 32])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
"""
Test the MXFP4 grouped GEMM kernel by:
"""Test the MXFP4 grouped GEMM kernel by:
1. Creating random per-expert inputs and weights
2. Quantizing both to MXFP4 using the CUDA kernel
3. Running the CUTLASS grouped GEMM
@@ -205,9 +204,7 @@ def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
reason="mxfp4_experts_quant requires CUDA SM100",
)
def test_mxfp4_experts_quant_basic():
"""
Basic smoke test for the MXFP4 experts quantization kernel.
"""
"""Basic smoke test for the MXFP4 experts quantization kernel."""
device = "cuda"
num_experts = 4
k = 256
@@ -295,8 +292,7 @@ def compute_reference_e8m0_scale(block_max: float) -> int:
@pytest.mark.parametrize("k", [256, 7168])
@pytest.mark.parametrize("m", [16, 64])
def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
"""
Test that mxfp4_experts_quant computes E8M0 block scales correctly
"""Test that mxfp4_experts_quant computes E8M0 block scales correctly
per OCP MX spec (not the NVFP4 formula).
The old buggy kernel used: floor(log2(max/6)) + 127
@@ -416,8 +412,7 @@ def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
reason="mxfp4_experts_quant requires CUDA SM100",
)
def test_mxfp4_experts_quant_no_saturation():
"""
Test that the E8M0 scale is large enough to avoid unexpected saturation.
"""Test that the E8M0 scale is large enough to avoid unexpected saturation.
With the buggy NVFP4 formula, the scale was too small causing most values
to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
@@ -130,7 +130,8 @@ def test_is_supported_config(present, ep_size, supported, reason_substr):
def test_explicit_moe_backend_aiter():
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
ValueError when the flydsl package is missing."""
ValueError when the flydsl package is missing.
"""
with _gfx950(), _flydsl_installed(True):
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
@@ -161,8 +162,9 @@ def test_gfx950_picks_aiter():
def test_gfx942_picks_emulation():
"""flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
dot_scaled backend wins instead."""
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
dot_scaled backend wins instead.
"""
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
backend, experts_cls = select_mxfp8_moe_backend(_config())
assert backend is Fp8MoeBackend.EMULATION
+4 -6
View File
@@ -214,8 +214,7 @@ def reference_moe(
activation: str = "swiglu",
use_interleaved_layout: bool = False,
):
"""
Reference MoE implementation for accuracy testing.
"""Reference MoE implementation for accuracy testing.
Args:
activation: One of "swiglu", "silu", "relu2". Controls the activation
@@ -224,6 +223,7 @@ def reference_moe(
(gate=x[..., ::2], up=x[..., 1::2]) as used by SWIGLUOAI.
If False, uses chunked layout (gate, up = chunk(x, 2)) as used
by standard swiglu/silu.
"""
# renormalize routing
experts = torch.topk(roouting_logits, k=topk, dim=-1, sorted=True)
@@ -1441,8 +1441,7 @@ def test_rocm_mxfp4_moe_oracle(
intermediate_size: int,
monkeypatch: pytest.MonkeyPatch,
):
"""
Test ROCm MXFP4 MoE using oracle functions.
"""Test ROCm MXFP4 MoE using oracle functions.
This test validates that the oracle functions work end-to-end:
- select_mxfp4_moe_backend() selects a valid backend
@@ -1825,8 +1824,7 @@ def test_mxfp4_emulation_rounds_up_to_block_size(
def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
monkeypatch: pytest.MonkeyPatch,
):
"""
select_mxfp4_moe_backend() must raise NotImplementedError, with the
"""select_mxfp4_moe_backend() must raise NotImplementedError, with the
collected per-backend unsupported reasons in the message, when no
backend supports the requested deployment configuration.
"""
+22 -11
View File
@@ -249,7 +249,8 @@ def _make_topk_ids(
device: str = "cuda",
) -> torch.Tensor:
"""Generate distinct expert IDs per token with the same top-k shape as
production routing."""
production routing.
"""
router_logits = torch.randn(num_tokens, num_experts, device=device)
_, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1)
return topk_ids.to(torch.int32)
@@ -443,7 +444,8 @@ def test_aiter_fused_moe_custom_op_registered():
def test_aiter_asm_moe_tkw1_custom_op_registered():
"""The tkw1 custom op should stay registered for FP8 apply-router-weight
paths."""
paths.
"""
_assert_aiter_supported()
import vllm._aiter_ops as aiter_ops # noqa: F401
@@ -453,7 +455,8 @@ def test_aiter_asm_moe_tkw1_custom_op_registered():
def test_aiter_fused_moe_fake_tensor_support():
"""The fused-MoE op should preserve fake-tensor compatibility for
torch.compile-style tracing."""
torch.compile-style tracing.
"""
_assert_aiter_supported()
import vllm._aiter_ops # noqa: F401
@@ -516,7 +519,8 @@ def test_aiter_moe_enablement_follows_env(
monkeypatch: pytest.MonkeyPatch,
):
"""The fused-MoE gate should depend only on the main AITER toggle and the
MoE-specific toggle."""
MoE-specific toggle.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -547,7 +551,8 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
monkeypatch: pytest.MonkeyPatch,
):
"""Shared-expert fusion should only be enabled when the fused-MoE path is
enabled too."""
enabled too.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -688,7 +693,8 @@ def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
"""The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized
MXFP4 reference."""
MXFP4 reference.
"""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
@@ -737,7 +743,8 @@ def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism():
"""The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise
deterministic."""
deterministic.
"""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
@@ -830,7 +837,8 @@ def test_aiter_fused_moe_bf16_accuracy(
intermediate_dim: int,
):
"""The ROCm AITER fused-MoE BF16 path should match the float32 reference
on representative shapes."""
on representative shapes.
"""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -875,7 +883,8 @@ def test_aiter_fused_moe_bf16_accuracy(
def test_aiter_fused_moe_gelu_accuracy():
"""The GELU activation variant should stay aligned with the float32
reference."""
reference.
"""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -920,7 +929,8 @@ def test_aiter_fused_moe_gelu_accuracy():
def test_aiter_fused_moe_determinism():
"""The BF16 fused-MoE kernel should stay bitwise deterministic for the
same inputs."""
same inputs.
"""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -1075,7 +1085,8 @@ def test_aiter_fused_moe_mi3xx_bf16_accuracy():
)
def test_aiter_fused_moe_mi3xx_fp8_accuracy():
"""The MI3xx FP8 per-tensor MoE path should stay within the measured FP8
error budget."""
error budget.
"""
from tests.kernels.moe.utils import make_test_weights
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
@@ -185,7 +185,8 @@ def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts(
) -> None:
"""The capture callback should receive the int16 routed-expert IDs the
kernel actually used, the values should be valid expert indices, and
each token should pick ``top_k`` distinct experts."""
each token should pick ``top_k`` distinct experts.
"""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
@@ -264,7 +265,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
routing_method: RoutingMethodType,
) -> None:
"""Routing replay works for non-DeepSeekV3 routing methods too.
FlashInfer's ``routing_replay_out`` is routing-method-agnostic."""
FlashInfer's ``routing_replay_out`` is routing-method-agnostic.
"""
torch.manual_seed(0)
device = torch.device("cuda:0")
@@ -317,7 +319,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
"""With no callback installed the kernel should not see a
``routing_replay_out`` tensor — verify the helper short-circuits."""
``routing_replay_out`` tensor — verify the helper short-circuits.
"""
torch.manual_seed(0)
device = torch.device("cuda:0")
experts, _, _ = _make_bf16_monolithic_experts(
@@ -339,7 +342,8 @@ def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
"""FlashInfer's ``routing_replay_out`` is supported by all routing
methods, so ``supports_routing_replay_capture`` should be True
regardless of routing method."""
regardless of routing method.
"""
device = torch.device("cuda:0")
for routing_method in (
RoutingMethodType.DeepSeekV3,
@@ -361,7 +365,8 @@ def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None:
"""When capture is installed, the allocated buffer is int16 and shaped
``(num_tokens, experts_per_token)``."""
``(num_tokens, experts_per_token)``.
"""
device = torch.device("cuda:0")
experts, _, _ = _make_bf16_monolithic_experts(
num_experts=_DSV3_NUM_EXPERTS,
@@ -592,7 +597,8 @@ def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts(
top_k: int,
) -> None:
"""End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale
path, DSV3 routing) captures valid expert IDs."""
path, DSV3 routing) captures valid expert IDs.
"""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
@@ -778,7 +784,8 @@ def _run_nvfp4_monolithic(
routing_bias: torch.Tensor,
) -> torch.Tensor:
"""The monolithic NVFP4 apply expects packed fp4 hidden states + the
matching fp8 per-block scale stored in the ``a1q_scale`` slot."""
matching fp8 per-block scale stored in the ``a1q_scale`` slot.
"""
# Stash the weight tensors on the experts in the locations the apply()
# implementation reads from (it pulls them from quant_config / scales
# already; w1/w2 come in as args).
@@ -806,7 +813,8 @@ def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts(
top_k: int,
) -> None:
"""End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs
on the DSV3 routing path."""
on the DSV3 routing path.
"""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
+7 -11
View File
@@ -214,8 +214,7 @@ def assert_routing_results_close(
rtol: float = 1e-3,
atol: float = 1e-3,
):
"""
Compare routing results, sorting by expert ID first to handle non-deterministic
"""Compare routing results, sorting by expert ID first to handle non-deterministic
ordering from sorted=False in topk.
"""
# Sort both results by expert IDs for consistent comparison
@@ -251,7 +250,8 @@ def assert_aiter_routing_valid(
the Python baseline (different group selection, scoring internals),
so numerical comparison is not meaningful. Instead we verify the
outputs satisfy the routing contract: correct shapes, valid expert
IDs, non-negative weights, and proper normalization."""
IDs, non-negative weights, and proper normalization.
"""
n_tokens = topk_weights.shape[0]
# Shape
@@ -294,8 +294,7 @@ def assert_aiter_routing_valid(
def baseline_fused_topk(
router_logits: torch.Tensor, top_k: int, renormalize: bool
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for standard fused top-k routing.
"""Baseline for standard fused top-k routing.
Algorithm:
1. Apply softmax to router logits
@@ -320,8 +319,7 @@ def baseline_fused_topk_bias(
e_score_correction_bias: torch.Tensor,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for fused top-k with bias correction.
"""Baseline for fused top-k with bias correction.
Algorithm:
1. Apply softmax to router logits
@@ -364,8 +362,7 @@ def baseline_grouped_topk(
e_score_correction_bias: torch.Tensor | None,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for grouped top-k routing (e.g., DeepSeek).
"""Baseline for grouped top-k routing (e.g., DeepSeek).
Algorithm:
1. Apply scoring function (softmax or sigmoid)
@@ -436,8 +433,7 @@ def baseline_grouped_topk(
def baseline_custom_llama4(
router_logits: torch.Tensor, top_k: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for Llama4 custom routing.
"""Baseline for Llama4 custom routing.
Algorithm:
1. Select top-k expert indices (without softmax)
+5 -4
View File
@@ -1,8 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test script for the token-to-expert routing simulator.
"""Test script for the token-to-expert routing simulator.
This script demonstrates how to use the routing simulator to test
different routing strategies and analyze their performance, including
@@ -77,7 +76,8 @@ def test_basic_functionality(
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoEFactory."""
FusedMoEFactory.
"""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
@@ -158,7 +158,8 @@ def test_routing_strategy_integration(monkeypatch, device):
def test_distribution_based_routing_with_custom_strategy():
"""Test registering and using DistributionBasedRouting with custom
parameters."""
parameters.
"""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Register custom distribution-based strategy
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for FusedMoEFactory with routed_input_transform.
"""Tests for FusedMoEFactory with routed_input_transform.
Verifies that applying routed_input_transform inside FusedMoEFactory
produces the same results as applying the transform manually outside.
@@ -142,9 +142,7 @@ def silu_mul_quant(
def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor:
"""
pack float32 scales into a int32 tensor
"""
"""Pack float32 scales into a int32 tensor."""
assert x.dtype == torch.float32
E, T, G = x.size()
@@ -170,8 +168,7 @@ def ref_with_scale_fmt(
up: torch.Tensor,
scale_fmt: DeepGemmQuantScaleFMT,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
The precision types of the operations triggered by this function
"""The precision types of the operations triggered by this function
match closely with the kernel implementation so we compare more
accurately.
"""
@@ -204,8 +201,7 @@ def ref_with_scale_fmt(
def token_random(E, T, H2, tokens_per_expert):
"""
Initialize each token in a random range so we test a range of
"""Initialize each token in a random range so we test a range of
scale values.
"""
y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE)
@@ -19,11 +19,9 @@ GROUP_SIZE = 128
def reference_quant(x: torch.Tensor, use_ue8m0: bool):
"""
Reference triton quant kernel from,
"""Reference triton quant kernel from,
vllm.model_executor.layers.quantization.utils.fp8_utils
"""
x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE)
# Allocate the scale tensor in column-major format.
@@ -72,7 +70,8 @@ def reference_with_clamp(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
match the C++ compute() template, then run the standard silu_and_mul +
quant reference."""
quant reference.
"""
N_2 = x.size(1) // 2
dtype = x.dtype
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
@@ -28,7 +28,8 @@ if current_platform.is_fp8_fnuz():
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
"""Matrix multiplication function that supports per-token input
quantization and per-column weight quantization"""
quantization and per-column weight quantization
"""
A = A.to(torch.float32)
B = B.to(torch.float32)
@@ -56,8 +57,8 @@ def fp8_mask(a, mask):
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
"""This function performs fused moe with per-column int8
quantization using native torch."""
quantization using native torch.
"""
B, D = a.shape
# Perform per-token quantization
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for the FlashInfer TRTLLM BF16 MoE backend
"""Tests for the FlashInfer TRTLLM BF16 MoE backend
(`TrtLlmBf16ExpertsModular`).
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for the FlashInfer TRTLLM NvFP4 MoE backend
"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend
(`TrtLlmNvFp4ExpertsModular`).
Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated),
@@ -150,7 +150,8 @@ def test_select_default_backend_by_platform(
expected_backend,
):
"""Test default backend selection per platform with all optional
accelerators (FlashInfer, AITER) disabled."""
accelerators (FlashInfer, AITER) disabled.
"""
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
@@ -444,7 +445,8 @@ def test_select_cuda_flashinfer_cutlass_backend(
mock_has_flashinfer,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available."""
and FlashInfer CUTLASS is available.
"""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
+1 -2
View File
@@ -83,8 +83,7 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
"""
Run a forward pass through FusedMoEFactory with zero experts
"""Run a forward pass through FusedMoEFactory with zero experts
and verify output shape.
"""
layer, vllm_config = zero_expert_moe
+1 -2
View File
@@ -64,8 +64,7 @@ def make_dummy_moe_config(
max_num_tokens: int = 512,
activation: MoEActivation = MoEActivation.SILU,
) -> FusedMoEConfig:
"""
This is a dummy config for the mk constructor interface
"""This is a dummy config for the mk constructor interface
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
do not actually use this config.