Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -148,9 +148,7 @@ class Config:
|
||||
return self.E // self.world_size
|
||||
|
||||
def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]:
|
||||
"""
|
||||
make env data for vllm launch.
|
||||
"""
|
||||
"""Make env data for vllm launch."""
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
enforce_eager=True,
|
||||
@@ -169,7 +167,8 @@ class Config:
|
||||
|
||||
def fe_supports_quant_scheme(self) -> bool:
|
||||
"""Check if the fused experts class supports this quant config.
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.
|
||||
"""
|
||||
if self.quant_config is None or self.quant_dtype is None:
|
||||
return True
|
||||
if self.quant_dtype != torch.float8_e4m3fn:
|
||||
@@ -444,9 +443,7 @@ class RankTensors:
|
||||
def make_hidden_states(
|
||||
config: Config,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
Return hidden_states
|
||||
"""
|
||||
"""Return hidden_states."""
|
||||
m, k, dtype = (config.M, config.K, config.dtype)
|
||||
device = torch.accelerator.current_device_index()
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 15.0
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DeepEP test utilities
|
||||
"""
|
||||
"""DeepEP test utilities."""
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
|
||||
@@ -468,7 +468,8 @@ def test_deepseek_v4_flashinfer_cutlass_falls_through_to_w4a8(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Explicit flashinfer_cutlass must try the W4A8 variant when the BF16
|
||||
variant is unsupported (the BF16 variant is gated to SM90)."""
|
||||
variant is unsupported (the BF16 variant is gated to SM90).
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
@@ -34,7 +34,6 @@ def test_batched_deepgemm_vs_triton(
|
||||
E: int, T: int, K: int, N: int, topk: int, monkeypatch, workspace_init
|
||||
):
|
||||
"""Compare BatchedDeepGemmExperts to BatchedTritonExperts."""
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
device = "cuda"
|
||||
|
||||
@@ -129,7 +129,8 @@ def test_batched_mm(
|
||||
per_act_token_quant: bool,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
@@ -268,7 +269,8 @@ def test_fused_moe_batched_experts(
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
@@ -649,7 +651,8 @@ def test_batched_experts_masked_activation_end_to_end(
|
||||
def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path."""
|
||||
the device-enablement path.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
|
||||
@@ -54,7 +54,8 @@ SEEDS = [0]
|
||||
# For test
|
||||
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
"""This function performs fused moe with block-wise quantization using
|
||||
native torch."""
|
||||
native torch.
|
||||
"""
|
||||
B, D = a.shape
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
@@ -97,7 +98,8 @@ def setup_cuda():
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
|
||||
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
|
||||
native torch reference."""
|
||||
native torch reference.
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests compute_expert_num_tokens kernels
|
||||
"""
|
||||
"""Tests compute_expert_num_tokens kernels."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
|
||||
@@ -397,7 +397,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
|
||||
class _StubMoELayer(torch.nn.Module):
|
||||
"""Minimal stand-in for the real MoE layer module, exposing just what
|
||||
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
|
||||
router configuration, and optionally w13_bias/w2_bias)."""
|
||||
router configuration, and optionally w13_bias/w2_bias).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -500,8 +501,8 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
act: MoEActivation,
|
||||
):
|
||||
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
|
||||
weights before prepacking."""
|
||||
|
||||
weights before prepacking.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
batch_size = 64
|
||||
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
|
||||
|
||||
@@ -62,6 +62,7 @@ def _make_int4_moe_weights(
|
||||
Returns:
|
||||
(w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias)
|
||||
where *_ref are the dequantized float reference weights
|
||||
|
||||
"""
|
||||
# Generate INT4 weights as int8 values in [-8, 7]
|
||||
w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8)
|
||||
|
||||
@@ -826,6 +826,7 @@ def _make_int4_moe_weights(E, N, K, group_size, quant_algo):
|
||||
w1_zeros, w2_zeros,
|
||||
w1_zeros_packed, w2_zeros_packed,
|
||||
w1_s, w2_s
|
||||
|
||||
"""
|
||||
w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32)
|
||||
w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32)
|
||||
|
||||
@@ -91,11 +91,9 @@ def break_fp4_bytes(a, dtype):
|
||||
def generate_balanced_routing(
|
||||
hidden_states: torch.Tensor, num_experts: int, top_k: int
|
||||
):
|
||||
"""
|
||||
Generate routing weights and topk indices such that every expert is active.
|
||||
"""Generate routing weights and topk indices such that every expert is active.
|
||||
Returns routing_weights, topk_idx
|
||||
"""
|
||||
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
# num_tokens = batch_size * seq_len
|
||||
|
||||
@@ -232,13 +230,13 @@ def grouped_gemm_ref(
|
||||
*,
|
||||
block_size: int = 16,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Computes the reference grouped GEMM (fp4 quantized per-expert loop),
|
||||
"""Computes the reference grouped GEMM (fp4 quantized per-expert loop),
|
||||
computes flashinfer grouped GEMM (for scale consistency),
|
||||
and returns ONLY the repacked reference output: out_ref.
|
||||
|
||||
Returns:
|
||||
out_ref: Tensor [num_experts, max_m, n_out]
|
||||
|
||||
"""
|
||||
device_hs = hidden_states_expanded.device
|
||||
device_w = weights.device
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test DeepEP + DeepGEMM integration
|
||||
"""Test DeepEP + DeepGEMM integration
|
||||
DeepGEMM are gemm kernels specialized for the
|
||||
fp8 block-quantized case.
|
||||
"""
|
||||
@@ -92,9 +91,7 @@ def make_block_quant_fp8_weights(
|
||||
k: int,
|
||||
block_size: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Return weights w1q, w2q, w1_scale, w2_scale
|
||||
"""
|
||||
"""Return weights w1q, w2q, w1_scale, w2_scale."""
|
||||
(_, w1q, w1_scale, _), (_, w2q, w2_scale, _) = make_test_weights(
|
||||
e, n, k, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
|
||||
)
|
||||
@@ -459,10 +456,7 @@ def test_ht_deepep_deepgemm_moe(
|
||||
disable_deepgemm_ue8m0,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
Tests for High-Throughput DeepEP + DeepGemm integration.
|
||||
"""
|
||||
|
||||
"""Tests for High-Throughput DeepEP + DeepGemm integration."""
|
||||
m, n, k = mnk
|
||||
set_random_seed(7)
|
||||
|
||||
@@ -533,9 +527,7 @@ def test_ll_deepep_deepgemm_moe(
|
||||
disable_deepgemm_ue8m0,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
Tests for Low-Latency DeepEP + DeepGemm integration.
|
||||
"""
|
||||
"""Tests for Low-Latency DeepEP + DeepGemm integration."""
|
||||
assert not is_deep_gemm_e8m0_used()
|
||||
|
||||
m, n, k = mnk
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test deepep dispatch-combine logic
|
||||
"""
|
||||
"""Test deepep dispatch-combine logic."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
@@ -56,9 +54,7 @@ MAX_TOKENS_PER_RANK = 64
|
||||
def make_weights(
|
||||
e, n, k, dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Return weights w1, w2, w1_scale, w2_scale
|
||||
"""
|
||||
"""Return weights w1, w2, w1_scale, w2_scale."""
|
||||
if dtype in [torch.float16, torch.bfloat16]:
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
|
||||
"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
|
||||
Compares against a pure-PyTorch reference MoE implementation.
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
|
||||
"""
|
||||
|
||||
@@ -79,8 +78,7 @@ def make_block_quant_fp8_weights(
|
||||
k: int,
|
||||
block_size: list[int],
|
||||
):
|
||||
"""
|
||||
Generate (w1, w2) expert weights and their per-block scale tensors
|
||||
"""Generate (w1, w2) expert weights and their per-block scale tensors
|
||||
in FP8 block-quantized format.
|
||||
|
||||
w1 shape: (E, 2N, K)
|
||||
@@ -121,8 +119,7 @@ def make_block_quant_fp8_weights(
|
||||
|
||||
|
||||
def run_single_case(m, n, k, topk, num_experts, block_size):
|
||||
"""
|
||||
Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
Triton baseline within tolerance.
|
||||
"""
|
||||
tokens_bf16 = (
|
||||
@@ -248,8 +245,7 @@ def make_mxfp4_weights(
|
||||
n: int,
|
||||
k: int,
|
||||
):
|
||||
"""
|
||||
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
plus BF16 reference weights for validation.
|
||||
|
||||
w1 shape: (E, 2N, K//2) uint8 — packed FP4
|
||||
@@ -310,8 +306,7 @@ def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
|
||||
|
||||
|
||||
def run_single_fp4_case(m, n, k, topk, num_experts):
|
||||
"""
|
||||
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
DeepGEMM FP4 == BF16 reference within tolerance.
|
||||
"""
|
||||
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
|
||||
|
||||
@@ -464,7 +464,8 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
):
|
||||
"""FlashInfer >= 0.6.18 applies the SwiGLU clamp for both block-scaled
|
||||
kernels with a SwiGLU activation (DeepSeek-V4 sets swiglu_limit); the
|
||||
per-tensor kernel has no clamp, and Relu2 rejects the parameters."""
|
||||
per-tensor kernel has no clamp, and Relu2 rejects the parameters.
|
||||
"""
|
||||
|
||||
class _Experts(TrtLlmFp8ExpertsModular):
|
||||
@staticmethod
|
||||
|
||||
@@ -83,7 +83,7 @@ def _reference_activation(
|
||||
beta: float | None,
|
||||
limit: float | None,
|
||||
):
|
||||
"""vLLM's own op for this activation, so the reference is not re-derived."""
|
||||
"""VLLM's own op for this activation, so the reference is not re-derived."""
|
||||
if activation == MoEActivation.RELU2_NO_MUL:
|
||||
return lambda x: torch.square(torch.relu(x))
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoE fused topk kernel
|
||||
"""Tests for the MoE fused topk kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_fused_topk.py`.
|
||||
"""
|
||||
|
||||
@@ -377,7 +377,8 @@ def test_unit_shuffle():
|
||||
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
|
||||
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
"""routing_data_from_sparse_topk must produce routing structures
|
||||
identical to make_routing_data for the same topk result."""
|
||||
identical to make_routing_data for the same topk result.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoE grouped topk kernel
|
||||
"""Tests for the MoE grouped topk kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_grouped_topk.py`.
|
||||
"""
|
||||
|
||||
@@ -31,6 +31,7 @@ def mxint4_quantize(
|
||||
Returns:
|
||||
- uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7]
|
||||
- scales in BF16: [..., k//sf_vec_size]
|
||||
|
||||
"""
|
||||
x_reshaped = x.reshape(-1, sf_vec_size)
|
||||
x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
@@ -61,6 +62,7 @@ def mxint4_quantize_moe_weights(
|
||||
Returns:
|
||||
- weights_mxint4: Quantized weights [e, n, k//2] uint8
|
||||
- scales_mxint4: Quantization scales [e, n, k//group_size] bf16
|
||||
|
||||
"""
|
||||
e = weights_bf16.shape[0]
|
||||
weight_list = []
|
||||
@@ -101,6 +103,7 @@ def marlin_quantize_moe_weights(
|
||||
Returns:
|
||||
- weights_marlin: Marlin quantized weights [e, k//8, n] int32
|
||||
- scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
|
||||
marlin_quantize,
|
||||
@@ -286,7 +289,8 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
|
||||
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
|
||||
trtllm_mxint4_block_scale_moe kernel call."""
|
||||
trtllm_mxint4_block_scale_moe kernel call.
|
||||
"""
|
||||
pytest.importorskip("flashinfer")
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
@@ -352,7 +352,8 @@ def test_modular_kernel_combinations_singlegpu(
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
config = Config(
|
||||
Ms=Ms,
|
||||
K=k,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test modular OAI Triton MoE
|
||||
"""
|
||||
"""Test modular OAI Triton MoE."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -119,8 +119,7 @@ def iterative_moe(
|
||||
expert_map: torch.Tensor = None,
|
||||
renormalize: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Baseline implementation of fused moe.
|
||||
"""Baseline implementation of fused moe.
|
||||
|
||||
Args:
|
||||
hidden_states: [*, hidden_size]
|
||||
@@ -128,6 +127,7 @@ def iterative_moe(
|
||||
w2: [num_experts, hidden_size, intermediate_size]
|
||||
gating_output: [*, num_experts]
|
||||
expert_map: [num_experts]
|
||||
|
||||
"""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
@@ -1956,8 +1956,7 @@ def test_batched_fused_marlin_moe(
|
||||
)
|
||||
|
||||
def is_valid(self):
|
||||
"""
|
||||
Return True only if the input can be represented in a Batched
|
||||
"""Return True only if the input can be represented in a Batched
|
||||
format.
|
||||
"""
|
||||
return torch.all(self.expert_num_tokens_cpu <= self.max_tokens_per_batch)
|
||||
@@ -2078,9 +2077,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
Test BF16 unquantized MoE with FlashInfer TRTLLM backend.
|
||||
"""
|
||||
"""Test BF16 unquantized MoE with FlashInfer TRTLLM backend."""
|
||||
set_random_seed(7)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
|
||||
@@ -54,8 +54,7 @@ def _verify_expert_level_sorting(
|
||||
valid_length: int,
|
||||
total_tokens: int,
|
||||
):
|
||||
"""
|
||||
Verify that actual_sorted_ids follows the correct expert-level sorting.
|
||||
"""Verify that actual_sorted_ids follows the correct expert-level sorting.
|
||||
The kerne limplementation may or may not preserve original token order
|
||||
in topk_ids in the final sorted_ids however this does not impact quality.
|
||||
"""
|
||||
@@ -96,8 +95,7 @@ def torch_moe_align_block_size(
|
||||
expert_map: torch.Tensor | None = None,
|
||||
pad_sorted_ids: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Golden torch implementation of moe_align_block_size.
|
||||
"""Golden torch implementation of moe_align_block_size.
|
||||
|
||||
This function aligns the token distribution across experts to be compatible
|
||||
with block size for matrix multiplication by sorting tokens by expert and
|
||||
@@ -189,7 +187,7 @@ def torch_moe_align_block_size(
|
||||
def test_moe_align_block_size(
|
||||
m: int, topk: int, num_experts: int, block_size: int, pad_sorted_ids: bool
|
||||
):
|
||||
"""Test moe_align_block_size without expert mapping"""
|
||||
"""Test moe_align_block_size without expert mapping."""
|
||||
topk_ids = torch.zeros((m, topk), device="cuda", dtype=torch.int32)
|
||||
for i in range(m):
|
||||
experts = torch.randperm(num_experts, device="cuda")[:topk]
|
||||
@@ -254,7 +252,7 @@ def test_moe_align_block_size_with_expert_map(
|
||||
block_size: int,
|
||||
mask_inactive_experts: bool,
|
||||
):
|
||||
"""Test moe_align_block_size with expert mapping (EP scenario)"""
|
||||
"""Test moe_align_block_size with expert mapping (EP scenario)."""
|
||||
expert_map = torch.full((num_experts,), -1, device="cuda", dtype=torch.int32)
|
||||
local_experts = list(range(0, num_experts, 2))
|
||||
for i, expert_id in enumerate(local_experts):
|
||||
|
||||
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
|
||||
class TestUnquantizedDelegation:
|
||||
"""UnquantizedMoEKernelOracle methods must delegate to the existing
|
||||
module-level functions; behaviour is bit-identical."""
|
||||
module-level functions; behaviour is bit-identical.
|
||||
"""
|
||||
|
||||
def test_make_kernel_delegates(self) -> None:
|
||||
quant_config = object()
|
||||
|
||||
@@ -240,8 +240,7 @@ def maybe_roundup_layer_hidden_size(
|
||||
act_dtype: torch.dtype,
|
||||
backend: str | None,
|
||||
) -> int:
|
||||
"""
|
||||
Given layer hidden size and MoE configurations, round up hidden_size
|
||||
"""Given layer hidden size and MoE configurations, round up hidden_size
|
||||
if necessary.
|
||||
|
||||
Args:
|
||||
@@ -253,6 +252,7 @@ def maybe_roundup_layer_hidden_size(
|
||||
Rounded up hidden_size if rounding up is required based on the configs
|
||||
and all2all backend.
|
||||
Original hidden size otherwise.
|
||||
|
||||
"""
|
||||
if backend == "deepep_high_throughput":
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
|
||||
@@ -315,7 +315,8 @@ def tp_chunk_gate_up(
|
||||
device: torch.device | int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""TP-chunk a combined [gate; up] weight, splitting each half separately
|
||||
so every rank gets a portion of both gate and up."""
|
||||
so every rank gets a portion of both gate and up.
|
||||
"""
|
||||
half = w.shape[dim] // 2
|
||||
gate = chunk_by_rank(
|
||||
w.narrow(dim, 0, half), tp_rank, tp_size, dim=dim, device=device
|
||||
@@ -945,6 +946,7 @@ def create_shared_experts_from_config(
|
||||
|
||||
Returns:
|
||||
TestMLP instance or None if config is None
|
||||
|
||||
"""
|
||||
if shared_experts_config is None:
|
||||
return None
|
||||
@@ -991,6 +993,7 @@ def setup_moe_test_data(
|
||||
|
||||
Returns:
|
||||
MoETestData containing all test data and transforms
|
||||
|
||||
"""
|
||||
# For latent MoE: latent_size = k // 2
|
||||
latent_size = k // 2
|
||||
@@ -1710,7 +1713,6 @@ def test_moe_layer_no_parallel(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Test MoE layer without parallelism (dp_size=1, tp_size=1, use_ep=False)."""
|
||||
|
||||
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MOE permute/unpermute kernel
|
||||
"""Tests for the MOE permute/unpermute kernel.
|
||||
|
||||
Run `pytest tests/kernels/test_moe_permute_unpermute.py`.
|
||||
"""
|
||||
|
||||
@@ -67,8 +67,7 @@ def compute_ref_output(
|
||||
@pytest.mark.parametrize("num_experts", [8, 16, 32])
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
|
||||
def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
|
||||
"""
|
||||
Test the MXFP4 grouped GEMM kernel by:
|
||||
"""Test the MXFP4 grouped GEMM kernel by:
|
||||
1. Creating random per-expert inputs and weights
|
||||
2. Quantizing both to MXFP4 using the CUDA kernel
|
||||
3. Running the CUTLASS grouped GEMM
|
||||
@@ -205,9 +204,7 @@ def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
def test_mxfp4_experts_quant_basic():
|
||||
"""
|
||||
Basic smoke test for the MXFP4 experts quantization kernel.
|
||||
"""
|
||||
"""Basic smoke test for the MXFP4 experts quantization kernel."""
|
||||
device = "cuda"
|
||||
num_experts = 4
|
||||
k = 256
|
||||
@@ -295,8 +292,7 @@ def compute_reference_e8m0_scale(block_max: float) -> int:
|
||||
@pytest.mark.parametrize("k", [256, 7168])
|
||||
@pytest.mark.parametrize("m", [16, 64])
|
||||
def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
|
||||
"""
|
||||
Test that mxfp4_experts_quant computes E8M0 block scales correctly
|
||||
"""Test that mxfp4_experts_quant computes E8M0 block scales correctly
|
||||
per OCP MX spec (not the NVFP4 formula).
|
||||
|
||||
The old buggy kernel used: floor(log2(max/6)) + 127
|
||||
@@ -416,8 +412,7 @@ def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
def test_mxfp4_experts_quant_no_saturation():
|
||||
"""
|
||||
Test that the E8M0 scale is large enough to avoid unexpected saturation.
|
||||
"""Test that the E8M0 scale is large enough to avoid unexpected saturation.
|
||||
|
||||
With the buggy NVFP4 formula, the scale was too small causing most values
|
||||
to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
|
||||
|
||||
@@ -130,7 +130,8 @@ def test_is_supported_config(present, ep_size, supported, reason_substr):
|
||||
|
||||
def test_explicit_moe_backend_aiter():
|
||||
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
|
||||
ValueError when the flydsl package is missing."""
|
||||
ValueError when the flydsl package is missing.
|
||||
"""
|
||||
with _gfx950(), _flydsl_installed(True):
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
@@ -161,8 +162,9 @@ def test_gfx950_picks_aiter():
|
||||
|
||||
|
||||
def test_gfx942_picks_emulation():
|
||||
"""flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
|
||||
dot_scaled backend wins instead."""
|
||||
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
|
||||
dot_scaled backend wins instead.
|
||||
"""
|
||||
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
|
||||
backend, experts_cls = select_mxfp8_moe_backend(_config())
|
||||
assert backend is Fp8MoeBackend.EMULATION
|
||||
|
||||
@@ -214,8 +214,7 @@ def reference_moe(
|
||||
activation: str = "swiglu",
|
||||
use_interleaved_layout: bool = False,
|
||||
):
|
||||
"""
|
||||
Reference MoE implementation for accuracy testing.
|
||||
"""Reference MoE implementation for accuracy testing.
|
||||
|
||||
Args:
|
||||
activation: One of "swiglu", "silu", "relu2". Controls the activation
|
||||
@@ -224,6 +223,7 @@ def reference_moe(
|
||||
(gate=x[..., ::2], up=x[..., 1::2]) as used by SWIGLUOAI.
|
||||
If False, uses chunked layout (gate, up = chunk(x, 2)) as used
|
||||
by standard swiglu/silu.
|
||||
|
||||
"""
|
||||
# renormalize routing
|
||||
experts = torch.topk(roouting_logits, k=topk, dim=-1, sorted=True)
|
||||
@@ -1441,8 +1441,7 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
intermediate_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
Test ROCm MXFP4 MoE using oracle functions.
|
||||
"""Test ROCm MXFP4 MoE using oracle functions.
|
||||
|
||||
This test validates that the oracle functions work end-to-end:
|
||||
- select_mxfp4_moe_backend() selects a valid backend
|
||||
@@ -1825,8 +1824,7 @@ def test_mxfp4_emulation_rounds_up_to_block_size(
|
||||
def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
select_mxfp4_moe_backend() must raise NotImplementedError, with the
|
||||
"""select_mxfp4_moe_backend() must raise NotImplementedError, with the
|
||||
collected per-backend unsupported reasons in the message, when no
|
||||
backend supports the requested deployment configuration.
|
||||
"""
|
||||
|
||||
@@ -249,7 +249,8 @@ def _make_topk_ids(
|
||||
device: str = "cuda",
|
||||
) -> torch.Tensor:
|
||||
"""Generate distinct expert IDs per token with the same top-k shape as
|
||||
production routing."""
|
||||
production routing.
|
||||
"""
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
_, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1)
|
||||
return topk_ids.to(torch.int32)
|
||||
@@ -443,7 +444,8 @@ def test_aiter_fused_moe_custom_op_registered():
|
||||
|
||||
def test_aiter_asm_moe_tkw1_custom_op_registered():
|
||||
"""The tkw1 custom op should stay registered for FP8 apply-router-weight
|
||||
paths."""
|
||||
paths.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
import vllm._aiter_ops as aiter_ops # noqa: F401
|
||||
|
||||
@@ -453,7 +455,8 @@ def test_aiter_asm_moe_tkw1_custom_op_registered():
|
||||
|
||||
def test_aiter_fused_moe_fake_tensor_support():
|
||||
"""The fused-MoE op should preserve fake-tensor compatibility for
|
||||
torch.compile-style tracing."""
|
||||
torch.compile-style tracing.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
import vllm._aiter_ops # noqa: F401
|
||||
|
||||
@@ -516,7 +519,8 @@ def test_aiter_moe_enablement_follows_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""The fused-MoE gate should depend only on the main AITER toggle and the
|
||||
MoE-specific toggle."""
|
||||
MoE-specific toggle.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -547,7 +551,8 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Shared-expert fusion should only be enabled when the fused-MoE path is
|
||||
enabled too."""
|
||||
enabled too.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -688,7 +693,8 @@ def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
|
||||
"""The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized
|
||||
MXFP4 reference."""
|
||||
MXFP4 reference.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
@@ -737,7 +743,8 @@ def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism():
|
||||
"""The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise
|
||||
deterministic."""
|
||||
deterministic.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
@@ -830,7 +837,8 @@ def test_aiter_fused_moe_bf16_accuracy(
|
||||
intermediate_dim: int,
|
||||
):
|
||||
"""The ROCm AITER fused-MoE BF16 path should match the float32 reference
|
||||
on representative shapes."""
|
||||
on representative shapes.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -875,7 +883,8 @@ def test_aiter_fused_moe_bf16_accuracy(
|
||||
|
||||
def test_aiter_fused_moe_gelu_accuracy():
|
||||
"""The GELU activation variant should stay aligned with the float32
|
||||
reference."""
|
||||
reference.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -920,7 +929,8 @@ def test_aiter_fused_moe_gelu_accuracy():
|
||||
|
||||
def test_aiter_fused_moe_determinism():
|
||||
"""The BF16 fused-MoE kernel should stay bitwise deterministic for the
|
||||
same inputs."""
|
||||
same inputs.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -1075,7 +1085,8 @@ def test_aiter_fused_moe_mi3xx_bf16_accuracy():
|
||||
)
|
||||
def test_aiter_fused_moe_mi3xx_fp8_accuracy():
|
||||
"""The MI3xx FP8 per-tensor MoE path should stay within the measured FP8
|
||||
error budget."""
|
||||
error budget.
|
||||
"""
|
||||
from tests.kernels.moe.utils import make_test_weights
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
|
||||
@@ -185,7 +185,8 @@ def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts(
|
||||
) -> None:
|
||||
"""The capture callback should receive the int16 routed-expert IDs the
|
||||
kernel actually used, the values should be valid expert indices, and
|
||||
each token should pick ``top_k`` distinct experts."""
|
||||
each token should pick ``top_k`` distinct experts.
|
||||
"""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
@@ -264,7 +265,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
|
||||
routing_method: RoutingMethodType,
|
||||
) -> None:
|
||||
"""Routing replay works for non-DeepSeekV3 routing methods too.
|
||||
FlashInfer's ``routing_replay_out`` is routing-method-agnostic."""
|
||||
FlashInfer's ``routing_replay_out`` is routing-method-agnostic.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda:0")
|
||||
|
||||
@@ -317,7 +319,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
|
||||
|
||||
def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
|
||||
"""With no callback installed the kernel should not see a
|
||||
``routing_replay_out`` tensor — verify the helper short-circuits."""
|
||||
``routing_replay_out`` tensor — verify the helper short-circuits.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda:0")
|
||||
experts, _, _ = _make_bf16_monolithic_experts(
|
||||
@@ -339,7 +342,8 @@ def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
|
||||
def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
|
||||
"""FlashInfer's ``routing_replay_out`` is supported by all routing
|
||||
methods, so ``supports_routing_replay_capture`` should be True
|
||||
regardless of routing method."""
|
||||
regardless of routing method.
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
for routing_method in (
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
@@ -361,7 +365,8 @@ def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
|
||||
|
||||
def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None:
|
||||
"""When capture is installed, the allocated buffer is int16 and shaped
|
||||
``(num_tokens, experts_per_token)``."""
|
||||
``(num_tokens, experts_per_token)``.
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
experts, _, _ = _make_bf16_monolithic_experts(
|
||||
num_experts=_DSV3_NUM_EXPERTS,
|
||||
@@ -592,7 +597,8 @@ def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts(
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale
|
||||
path, DSV3 routing) captures valid expert IDs."""
|
||||
path, DSV3 routing) captures valid expert IDs.
|
||||
"""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
@@ -778,7 +784,8 @@ def _run_nvfp4_monolithic(
|
||||
routing_bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""The monolithic NVFP4 apply expects packed fp4 hidden states + the
|
||||
matching fp8 per-block scale stored in the ``a1q_scale`` slot."""
|
||||
matching fp8 per-block scale stored in the ``a1q_scale`` slot.
|
||||
"""
|
||||
# Stash the weight tensors on the experts in the locations the apply()
|
||||
# implementation reads from (it pulls them from quant_config / scales
|
||||
# already; w1/w2 come in as args).
|
||||
@@ -806,7 +813,8 @@ def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts(
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs
|
||||
on the DSV3 routing path."""
|
||||
on the DSV3 routing path.
|
||||
"""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
|
||||
@@ -214,8 +214,7 @@ def assert_routing_results_close(
|
||||
rtol: float = 1e-3,
|
||||
atol: float = 1e-3,
|
||||
):
|
||||
"""
|
||||
Compare routing results, sorting by expert ID first to handle non-deterministic
|
||||
"""Compare routing results, sorting by expert ID first to handle non-deterministic
|
||||
ordering from sorted=False in topk.
|
||||
"""
|
||||
# Sort both results by expert IDs for consistent comparison
|
||||
@@ -251,7 +250,8 @@ def assert_aiter_routing_valid(
|
||||
the Python baseline (different group selection, scoring internals),
|
||||
so numerical comparison is not meaningful. Instead we verify the
|
||||
outputs satisfy the routing contract: correct shapes, valid expert
|
||||
IDs, non-negative weights, and proper normalization."""
|
||||
IDs, non-negative weights, and proper normalization.
|
||||
"""
|
||||
n_tokens = topk_weights.shape[0]
|
||||
|
||||
# Shape
|
||||
@@ -294,8 +294,7 @@ def assert_aiter_routing_valid(
|
||||
def baseline_fused_topk(
|
||||
router_logits: torch.Tensor, top_k: int, renormalize: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for standard fused top-k routing.
|
||||
"""Baseline for standard fused top-k routing.
|
||||
|
||||
Algorithm:
|
||||
1. Apply softmax to router logits
|
||||
@@ -320,8 +319,7 @@ def baseline_fused_topk_bias(
|
||||
e_score_correction_bias: torch.Tensor,
|
||||
routed_scaling_factor: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for fused top-k with bias correction.
|
||||
"""Baseline for fused top-k with bias correction.
|
||||
|
||||
Algorithm:
|
||||
1. Apply softmax to router logits
|
||||
@@ -364,8 +362,7 @@ def baseline_grouped_topk(
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
routed_scaling_factor: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for grouped top-k routing (e.g., DeepSeek).
|
||||
"""Baseline for grouped top-k routing (e.g., DeepSeek).
|
||||
|
||||
Algorithm:
|
||||
1. Apply scoring function (softmax or sigmoid)
|
||||
@@ -436,8 +433,7 @@ def baseline_grouped_topk(
|
||||
def baseline_custom_llama4(
|
||||
router_logits: torch.Tensor, top_k: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for Llama4 custom routing.
|
||||
"""Baseline for Llama4 custom routing.
|
||||
|
||||
Algorithm:
|
||||
1. Select top-k expert indices (without softmax)
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test script for the token-to-expert routing simulator.
|
||||
"""Test script for the token-to-expert routing simulator.
|
||||
|
||||
This script demonstrates how to use the routing simulator to test
|
||||
different routing strategies and analyze their performance, including
|
||||
@@ -77,7 +76,8 @@ def test_basic_functionality(
|
||||
|
||||
def test_routing_strategy_integration(monkeypatch, device):
|
||||
"""Test that the routing strategy environment variable works with
|
||||
FusedMoEFactory."""
|
||||
FusedMoEFactory.
|
||||
"""
|
||||
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -158,7 +158,8 @@ def test_routing_strategy_integration(monkeypatch, device):
|
||||
|
||||
def test_distribution_based_routing_with_custom_strategy():
|
||||
"""Test registering and using DistributionBasedRouting with custom
|
||||
parameters."""
|
||||
parameters.
|
||||
"""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Register custom distribution-based strategy
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for FusedMoEFactory with routed_input_transform.
|
||||
"""Tests for FusedMoEFactory with routed_input_transform.
|
||||
|
||||
Verifies that applying routed_input_transform inside FusedMoEFactory
|
||||
produces the same results as applying the transform manually outside.
|
||||
|
||||
@@ -142,9 +142,7 @@ def silu_mul_quant(
|
||||
|
||||
|
||||
def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
pack float32 scales into a int32 tensor
|
||||
"""
|
||||
"""Pack float32 scales into a int32 tensor."""
|
||||
assert x.dtype == torch.float32
|
||||
E, T, G = x.size()
|
||||
|
||||
@@ -170,8 +168,7 @@ def ref_with_scale_fmt(
|
||||
up: torch.Tensor,
|
||||
scale_fmt: DeepGemmQuantScaleFMT,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
The precision types of the operations triggered by this function
|
||||
"""The precision types of the operations triggered by this function
|
||||
match closely with the kernel implementation so we compare more
|
||||
accurately.
|
||||
"""
|
||||
@@ -204,8 +201,7 @@ def ref_with_scale_fmt(
|
||||
|
||||
|
||||
def token_random(E, T, H2, tokens_per_expert):
|
||||
"""
|
||||
Initialize each token in a random range so we test a range of
|
||||
"""Initialize each token in a random range so we test a range of
|
||||
scale values.
|
||||
"""
|
||||
y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
@@ -19,11 +19,9 @@ GROUP_SIZE = 128
|
||||
|
||||
|
||||
def reference_quant(x: torch.Tensor, use_ue8m0: bool):
|
||||
"""
|
||||
Reference triton quant kernel from,
|
||||
"""Reference triton quant kernel from,
|
||||
vllm.model_executor.layers.quantization.utils.fp8_utils
|
||||
"""
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE)
|
||||
|
||||
# Allocate the scale tensor in column-major format.
|
||||
@@ -72,7 +70,8 @@ def reference_with_clamp(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
|
||||
match the C++ compute() template, then run the standard silu_and_mul +
|
||||
quant reference."""
|
||||
quant reference.
|
||||
"""
|
||||
N_2 = x.size(1) // 2
|
||||
dtype = x.dtype
|
||||
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
|
||||
|
||||
@@ -28,7 +28,8 @@ if current_platform.is_fp8_fnuz():
|
||||
|
||||
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
"""Matrix multiplication function that supports per-token input
|
||||
quantization and per-column weight quantization"""
|
||||
quantization and per-column weight quantization
|
||||
"""
|
||||
A = A.to(torch.float32)
|
||||
B = B.to(torch.float32)
|
||||
|
||||
@@ -56,8 +57,8 @@ def fp8_mask(a, mask):
|
||||
|
||||
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
|
||||
"""This function performs fused moe with per-column int8
|
||||
quantization using native torch."""
|
||||
|
||||
quantization using native torch.
|
||||
"""
|
||||
B, D = a.shape
|
||||
# Perform per-token quantization
|
||||
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for the FlashInfer TRTLLM BF16 MoE backend
|
||||
"""Tests for the FlashInfer TRTLLM BF16 MoE backend
|
||||
(`TrtLlmBf16ExpertsModular`).
|
||||
|
||||
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for the FlashInfer TRTLLM NvFP4 MoE backend
|
||||
"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend
|
||||
(`TrtLlmNvFp4ExpertsModular`).
|
||||
|
||||
Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated),
|
||||
|
||||
@@ -150,7 +150,8 @@ def test_select_default_backend_by_platform(
|
||||
expected_backend,
|
||||
):
|
||||
"""Test default backend selection per platform with all optional
|
||||
accelerators (FlashInfer, AITER) disabled."""
|
||||
accelerators (FlashInfer, AITER) disabled.
|
||||
"""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
@@ -444,7 +445,8 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
||||
mock_has_flashinfer,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||
and FlashInfer CUTLASS is available."""
|
||||
and FlashInfer CUTLASS is available.
|
||||
"""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
|
||||
@@ -83,8 +83,7 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
|
||||
"""
|
||||
Run a forward pass through FusedMoEFactory with zero experts
|
||||
"""Run a forward pass through FusedMoEFactory with zero experts
|
||||
and verify output shape.
|
||||
"""
|
||||
layer, vllm_config = zero_expert_moe
|
||||
|
||||
@@ -64,8 +64,7 @@ def make_dummy_moe_config(
|
||||
max_num_tokens: int = 512,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> FusedMoEConfig:
|
||||
"""
|
||||
This is a dummy config for the mk constructor interface
|
||||
"""This is a dummy config for the mk constructor interface
|
||||
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
|
||||
do not actually use this config.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user