Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+6 -3
View File
@@ -247,7 +247,8 @@ def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
class _FakeLinear:
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
.quant_method are read by get_and_maybe_dequant_weights()."""
.quant_method are read by get_and_maybe_dequant_weights().
"""
def __init__(self, weight: torch.Tensor):
self.weight = weight
@@ -318,7 +319,8 @@ class _FakeAttnMetadata:
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
decode attention through W_UK/W_UV."""
decode attention through W_UK/W_UV.
"""
set_random_seed(1)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
@@ -384,7 +386,8 @@ def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
prefill batch with a mix of fresh and continued (cached-prefix)
sequences."""
sequences.
"""
set_random_seed(2)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
@@ -616,7 +616,8 @@ def test_non_causal_autoselect_backend():
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle."""
that it cannot handle.
"""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
@@ -627,7 +628,8 @@ def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.
"""
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -225,7 +225,8 @@ def test_cutlass_mla_decode_cross_layer_view():
"""The kernel must read the cache's page-dim stride instead of assuming
pages are packed back-to-back. A per-layer view into a cross-layer
(block-major) cache has stride(0) inflated by num_layers; outputs must
match a contiguous cache holding the same data exactly."""
match a contiguous cache holding the same data exactly.
"""
device = torch.device("cuda:0")
torch.set_default_dtype(torch.bfloat16)
torch.set_default_device(device)
+3 -1
View File
@@ -96,6 +96,7 @@ def _make_paged_kv_metadata(
kv_indices – CUDA int32, shape [total_blocks]
kv_last_page_lens – CPU int32, shape [num_seqs]
block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq]
"""
num_seqs = len(kv_lens)
max_blocks = (max(kv_lens) + block_size - 1) // block_size
@@ -161,7 +162,8 @@ def test_fast_decode_plan_importable() -> None:
@torch.inference_mode
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
"""On the first call fast_plan_decode must route through self.plan() and
flip vllm_first_call to False on the wrapper object."""
flip vllm_first_call to False on the wrapper object.
"""
from unittest.mock import patch
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
@@ -286,7 +286,7 @@ def test_flashinfer_sm90_fp8_mla_decode_without_rope():
@requires_sm10x
def test_flashinfer_mla_decode_workspace_supports_autotune():
"""vLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
"""VLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
@@ -134,7 +134,8 @@ def make_quantized_kv_cache(
head_size: int,
) -> tuple:
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
kv_scale, ref_kv_cache, is_nvfp4_kv)."""
kv_scale, ref_kv_cache, is_nvfp4_kv).
"""
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
if is_nvfp4_kv:
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
@@ -73,7 +73,8 @@ def _strided_context_inputs(
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
workspace (left in the fp8 cache layout for a plain fp8 cache).
"""
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
k_nope, v = kv_nope.split((128, 128), dim=-1)
workspace = _randn(num_tokens, 576, dtype=dtype)
@@ -23,7 +23,7 @@ DTYPES = [torch.float32]
def reference_lightning_attention(q, k, v, ed, block_size, kv_history):
"""Reference implementation of lightning attention core algorithm
"""Reference implementation of lightning attention core algorithm.
The difference from the main implementation is that this processes
each step sequentially, instead of using parallelized triton kernels
@@ -72,7 +72,7 @@ def reference_lightning_attention(q, k, v, ed, block_size, kv_history):
def reference_linear_decode(q, k, v, kv_caches, slope_rate, slot_idx):
"""Reference implementation: linear attention decode function"""
"""Reference implementation: linear attention decode function."""
B, H, _, D = q.shape
output = torch.zeros(B, H * D, dtype=q.dtype, device=q.device)
@@ -89,7 +89,8 @@ all_case_info: list[tuple] = []
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
scales must not surface as NaN in the merged output."""
scales must not surface as NaN in the merged output.
"""
num_tokens, num_heads, head_size = 6, 8, 128
prefix_output = torch.zeros(
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
+3 -7
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test:
"""Test:
* Tests for MMEncoderAttention layer
"""
@@ -49,9 +48,7 @@ if current_platform.is_rocm():
@pytest.mark.parametrize("device", devices)
def test_mha_attn_platform(default_vllm_config, device: str):
"""
Test the attention selector between different platform and device.
"""
"""Test the attention selector between different platform and device."""
torch.set_default_dtype(torch.float16)
if device == "cpu":
@@ -113,8 +110,7 @@ def ref_attention(
value: torch.Tensor,
scale: float,
) -> torch.Tensor:
"""
Native implementation of scaled dot product attention without mask:
"""Native implementation of scaled dot product attention without mask:
- query, key, value: [batch_size, seq_len, num_heads, head_size]
- attn_mask: [batch_size, seq_len, seq_len]
"""
+28 -14
View File
@@ -45,7 +45,8 @@ def kv_layout(request) -> KVCacheLayout:
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
"""Per-layer physical stride order for the given layout; the 3-dim
indexer side cache (H=1) is contiguous, so identity."""
indexer side cache (H=1) is contiguous, so identity.
"""
if ndim == 3:
return (0, 1, 2)
stride_order = layout.layer_view_order
@@ -65,7 +66,8 @@ def _main_spec() -> FullAttentionSpec:
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
derived the same way the production allocator does."""
derived the same way the production allocator does.
"""
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
@@ -75,7 +77,8 @@ def _allocate_main_kv_via_contract(
) -> torch.Tensor:
"""Build the main KV cache exactly as the production allocator does for the
given layout: allocate the physical (permuted) tensor, then expose the
inverse-permuted logical [B, H, N, C] view the backend sees."""
inverse-permuted logical [B, H, N, C] view the backend sees.
"""
logical_shape = _main_kv_logical_shape(num_pages)
stride_order = _layer_stride_order(layout, len(logical_shape))
physical_shape = tuple(logical_shape[i] for i in stride_order)
@@ -1376,7 +1379,8 @@ def test_amd_decode_index_topk_end_to_end(
@pytest.mark.parametrize("num_idx_heads", [1, 4])
def test_decode_index_topk_fp8(num_idx_heads: int):
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
matches a reference computed from the dequantized FP8 values."""
matches a reference computed from the dequantized FP8 values.
"""
torch.manual_seed(0)
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
decode_query_len = 1
@@ -1649,7 +1653,8 @@ def test_prefill_sparse_attention_correctness(
def test_main_cache_layout_contract():
"""The standardized per-layer logical shape is [B, H, N, C] with packed
K/V content, and the legacy layout aliases resolve to the expected
per-layer stride orders."""
per-layer stride orders.
"""
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = _main_kv_logical_shape(nb)
assert logical == (nb, h, bs, 2 * d)
@@ -1773,7 +1778,8 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
"""The indexer side cache is standardized 4D with H=1: under both layouts
the allocator's logical view stays contiguous and squeezes (as
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
[num_blocks, block_size, head_dim] cache the kernels consume."""
[num_blocks, block_size, head_dim] cache the kernels consume.
"""
nb = 5
ispec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
@@ -1793,7 +1799,8 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
def test_hnd_allocation_is_packed_head_major():
"""Under HND the backend-visible logical view is the packed head-major
physical allocation."""
physical allocation.
"""
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = _main_kv_logical_shape(nb)
stride_order = KVCacheLayout.LBHNC.layer_view_order
@@ -1821,7 +1828,8 @@ def test_hnd_allocation_is_packed_head_major():
def test_main_cache_is_block_first_and_unpadded():
"""The allocator's contiguous-view branch (not the padded-strided branch)
is used for the main GQA cache: its spec is unpadded and the physical
layout keeps num_blocks as the first dimension under both layouts."""
layout keeps num_blocks as the first dimension under both layouts.
"""
spec = _main_spec()
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
assert spec.page_size_padded is None
@@ -1843,7 +1851,8 @@ def _build_decode_inputs(
):
"""Shared decode setup: uniform query tokens per request, a non-identity
block table, and topk indices selecting the current block plus older causal
blocks for each query token."""
blocks for each query token.
"""
active_batch = len(seq_lens_list)
batch = active_batch + num_padded_reqs
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
@@ -2179,7 +2188,8 @@ def test_decode_sparse_attention_correctness(
):
"""Decode (split-K) parity under both layouts: this is the only coverage of
the decode-site cache feed, and the strided HND case fails if the kernel
ignores the cache strides."""
ignores the cache strides.
"""
torch.manual_seed(0)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
seq_lens_list, decode_query_len, num_padded_reqs
@@ -2229,7 +2239,8 @@ def test_decode_wrong_layout_breaks_parity():
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
reorders the K/V content, so the decode output no longer matches the
reference computed on the correct logical view. The mislabeled tensor keeps
the same shape as the correct view, so the kernel stays in bounds."""
the same shape as the correct view, so the kernel stays in bounds.
"""
torch.manual_seed(0)
seq_lens_list = (130, 257)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
@@ -2264,7 +2275,8 @@ def test_decode_wrong_layout_breaks_parity():
def test_main_cache_byte_identical_through_production_allocator():
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
`FullAttentionSpec` under HND and assert the backend-visible view has the
same shape, stride, and storage offset as the packed-HND allocation."""
same shape, stride, and storage offset as the packed-HND allocation.
"""
nb = 4
spec = _main_spec()
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
@@ -2279,7 +2291,8 @@ def test_main_cache_byte_identical_through_production_allocator():
def test_padded_main_cache_is_flagged():
"""AC-2.1 negative: the M3 main cache relies on the allocator's
contiguous-view branch (`page_size_padded is None`). A spec that sets
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
`page_size_padded` is explicitly flagged rather than silently wrong-strided.
"""
def _require_unpadded_block_first(spec, stride_order):
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
@@ -2316,7 +2329,8 @@ def test_padded_main_cache_is_flagged():
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
packed-content K/V split views. Writing through those views must persist
into the bound storage under both layouts."""
into the bound storage under both layouts.
"""
torch.manual_seed(0)
num_pages = 4
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
@@ -23,7 +23,8 @@ pytestmark = pytest.mark.skipif(
def test_concat_and_cache_mla_into_unified_slot_view():
"""concat_and_cache_mla must write correctly into a per-layer view whose
block stride is the full unified slot (block-major), with zero bleed into
the other layers' segments of the same slot."""
the other layers' segments of the same slot.
"""
from vllm import _custom_ops as ops
torch.manual_seed(0)
@@ -85,7 +86,8 @@ def test_concat_and_cache_mla_into_unified_slot_view():
def test_flashmla_dense_decode_unified_slot_view():
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
on Hopper) must read a unified-slot block-major view bit-identically to a
contiguous per-layer cache."""
contiguous per-layer cache.
"""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_dense_supported()
@@ -150,7 +152,8 @@ def test_flashmla_dense_decode_unified_slot_view():
def test_flashinfer_mla_dense_decode_unified_slot_view():
"""FlashInfer MLA dense decode must read a unified-slot block-major view
(inflated stride(0), non-zero storage offset) bit-identically to a
contiguous per-layer cache."""
contiguous per-layer cache.
"""
try:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
except ImportError:
@@ -221,7 +224,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
cache, with finite nonzero output."""
cache, with finite nonzero output.
"""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
@@ -310,7 +314,8 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
def test_indexer_k_quant_and_cache_into_unified_slot_view():
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
write correctly into a per-layer view whose block stride is the full
unified slot, with zero bleed into the other layers' segments."""
unified slot, with zero bleed into the other layers' segments.
"""
from vllm import _custom_ops as ops
torch.manual_seed(0)
@@ -355,7 +360,8 @@ def test_indexer_k_quant_and_cache_into_unified_slot_view():
def test_flashattn_mla_dense_decode_unified_slot_view():
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
block-major view bit-identically to a contiguous per-layer cache."""
block-major view bit-identically to a contiguous per-layer cache.
"""
try:
from vllm.vllm_flash_attn import flash_attn_varlen_func
except ImportError:
@@ -428,7 +434,8 @@ def test_flashattn_mla_dense_decode_unified_slot_view():
def test_flashmla_dense_fp8_decode_unified_slot_view():
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
must read a unified-slot block-major view bit-identically to a contiguous
per-layer fp8 cache."""
per-layer fp8 cache.
"""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_dense_supported()
@@ -499,7 +506,8 @@ def test_flashmla_dense_fp8_decode_unified_slot_view():
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
unified-slot block-major view bit-identically to a contiguous per-layer
cache."""
cache.
"""
try:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
except ImportError:
+14 -7
View File
@@ -283,7 +283,8 @@ def _run_direct_flash_attn_case(
# Backend contract tests --------------------------------------------------
def test_aiter_mha_backend_contract():
"""The ROCm backend advertises the dtypes, shapes, and attention types it
is designed to handle."""
is designed to handle.
"""
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
@@ -311,7 +312,8 @@ def test_aiter_mha_backend_contract():
def test_aiter_mha_backend_validates_kv_cache_block_size():
"""The backend should reject KV cache specs that cannot be gathered
correctly by the ROCm kernel."""
correctly by the ROCm kernel.
"""
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
from vllm.v1.kv_cache_interface import FullAttentionSpec
@@ -331,7 +333,8 @@ def test_aiter_mha_backend_validates_kv_cache_block_size():
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
"""The backend should trust the ROCm MI3xx probe instead of the raw torch
capability tuple."""
capability tuple.
"""
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
@@ -356,7 +359,8 @@ def test_aiter_mha_env_flags_control_enablement(
use_aiter, use_mha, expected, monkeypatch
):
"""Both the global AITER flag and the MHA-specific flag must be enabled
before the ROCm MHA path is considered active."""
before the ROCm MHA path is considered active.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -374,7 +378,8 @@ def test_aiter_mha_env_flags_control_enablement(
def test_aiter_mha_platform_gate_matches_install_and_arch():
"""The global AITER availability check should only open on ROCm MI3xx when
the aiter package is installed."""
the aiter package is installed.
"""
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found_and_supported
assert is_aiter_found_and_supported() is (
@@ -645,7 +650,8 @@ def test_aiter_mha_multi_batch(num_heads, head_size, dtype):
@pytest.mark.parametrize("num_blocks", DIRECT_NUM_BLOCKS)
def test_aiter_fa_large_block_table_matches_reference(num_blocks):
"""The direct paged-KV path should stay stable for both normal and very
large block tables."""
large block tables.
"""
atol = 2e-2
rtol = 2e-2
_assert_aiter_supported()
@@ -667,7 +673,8 @@ def test_aiter_fa_large_block_table_matches_reference(num_blocks):
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
def test_aiter_fa_sliding_window_matches_reference():
"""The direct kernel should respect the same sliding-window causal mask as
the naive reference implementation."""
the naive reference implementation.
"""
atol = 2e-2
rtol = 2e-2
_assert_aiter_supported()
@@ -35,7 +35,7 @@ def _require_aiter():
@torch.inference_mode()
def test_mla_decode_fwd_op_schema() -> None:
"""opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
A single opcheck call covers that the op is registered/callable, that its
fake implementation matches the real op (torch.compile tracing), and that
@@ -111,7 +111,8 @@ def _make_mixed_common_metadata():
def _patch_build_deps(monkeypatch, events=None):
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
on CPU."""
on CPU.
"""
def fake_generate_sparse_seqlen_triton(
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
@@ -152,7 +153,8 @@ def test_build_populates_decode_only_split_fields(monkeypatch):
def test_build_populates_mixed_split_fields(monkeypatch):
"""Mixed decode+prefill batch: split is reported, prefill fields stay
default because this impl always runs the MQA path."""
default because this impl always runs the MQA path.
"""
builder = _make_builder()
_patch_build_deps(monkeypatch)
@@ -125,7 +125,8 @@ def _quantize_to_fp8(tensor: torch.Tensor):
@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
"""Test FP8 KV cache path: quantize K/V to FP8, run kernel with scales,
and compare against BF16 reference output."""
and compare against BF16 reference output.
"""
assert CACHE_SIZE % PAGE_SIZE == 0
dtype = torch.bfloat16
seq_len = L
@@ -248,7 +249,8 @@ def test_decode_attention_cross_layer_view(H_Q, H_KV, D_QK, D_V, is_mla, PAGE_SI
"""The kernel must honor the cache's page-dim stride, not assume pages are
packed back-to-back. A per-layer view into a cross-layer (block-major)
cache has stride(0) inflated by num_layers; outputs must match a
contiguous cache holding the same data exactly."""
contiguous cache holding the same data exactly.
"""
B = 3
seq_len = 1027
CACHE_SIZE = 16384
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the Triton DiffKV unified-attention kernel.
"""
"""Unit tests for the Triton DiffKV unified-attention kernel."""
import pytest
import torch
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
"""Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
Tests both contiguous and non-contiguous (cross-layer unified) KV cache
layouts against a pure-PyTorch reference implementation.
@@ -43,8 +42,7 @@ def make_random_kv_cache(
layout=KVCacheLayout.LBHNC,
num_layers=None,
):
"""
Create a random fp8 KV cache mimicking the production allocation.
"""Create a random fp8 KV cache mimicking the production allocation.
Physical layout: [L, B, H, N, 2 * hs] permuted by ``layout.stride_order``
Returned view: (num_blocks, 2, num_kv_heads, block_size, head_size)
@@ -382,8 +380,7 @@ def test_large_block_size():
@torch.inference_mode()
def test_cross_layer_many_layers():
"""
Non-contiguous with 36 layers -- matches real gpt-oss-120b.
"""Non-contiguous with 36 layers -- matches real gpt-oss-120b.
Cross-layer (BLHNC) strides are far from contiguous (factor of 36 in the gaps).
"""
from vllm.v1.attention.backends.flashinfer import (
@@ -30,12 +30,12 @@ def reference_mla_sparse_prefill(
topk_length: torch.Tensor | None = None,
attn_sink: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Returns:
"""Returns:
- o: [s_q, h_q, dv]
- o_fp32: [s_q, h_q, dv]
- max_logits: [s_q, h_q]
- lse: [s_q, h_q]
"""
s_q, h_q, d_qk = q.shape
s_kv, _, _ = kv.shape
+2 -4
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for ApplyRotaryEmb CustomOp dispatch behavior.
"""Tests for ApplyRotaryEmb CustomOp dispatch behavior.
This test ensures that RotaryEmbedding classes correctly call the appropriate
ApplyRotaryEmb methods based on the calling context:
@@ -179,8 +178,7 @@ def test_rotary_embedding_dispatch(
test_case: RotaryEmbeddingTestCase,
device: str,
):
"""
Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method.
"""Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method.
- forward_native methods should call ApplyRotaryEmb.forward_native()
- forward_cuda/forward methods should call ApplyRotaryEmb.forward()
+4 -2
View File
@@ -41,7 +41,8 @@ def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
def test_fused_embed_norm_matches_reference():
"""Main-model fusion: the residual is the exact gather and the second output
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel.
"""
set_random_seed(13)
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
@@ -60,7 +61,8 @@ def test_fused_embed_norm_matches_reference():
@torch.inference_mode()
def test_fused_embed_eh_norm_matches_reference():
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
feeding the untouched model-local ``fused_eh_norm``."""
feeding the untouched model-local ``fused_eh_norm``.
"""
set_random_seed(13)
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
@@ -62,7 +62,8 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
qr_out with row stride q_size + kv_size. Downstream dispatchers that
require packed row-major inputs then reject the tensor and silently fall
back to a slower GEMM path on every decode step."""
back to a slower GEMM path on every decode step.
"""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
@@ -88,7 +89,8 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
lifts that bound to 2**31-1."""
lifts that bound to 2**31-1.
"""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
matching the eager triton kernel output."""
matching the eager triton kernel output.
"""
import pytest
import torch
@@ -36,7 +36,6 @@ def _worker_forward_qk(
eps,
):
"""Per-rank worker: compare NCCL allreduce path vs Lamport fused kernel."""
if not hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
cleanup_dist_env_and_memory()
return
+1 -1
View File
@@ -229,7 +229,7 @@ def test_mrope_torch_compile_tracing(
# Create a wrapper that makes the in-place function appear functional
def functional_forward_cuda(pos, q, k):
"""Wrapper that converts in-place operation to functional style
"""Wrapper that converts in-place operation to functional style.
CUDA Graph does not support in-place operations.
This wrapper creates working copies of the
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for miscellaneous utilities
"""
"""Tests for miscellaneous utilities."""
import torch
+2 -1
View File
@@ -579,7 +579,8 @@ def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
contiguous inputs."""
contiguous inputs.
"""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for miscellaneous utilities
"""
"""Tests for miscellaneous utilities."""
import pytest
import torch
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for fused MLA KV-cache write and RoPE fused kernel
"""
"""Tests for fused MLA KV-cache write and RoPE fused kernel."""
import random
@@ -87,7 +87,8 @@ def _shard(full_table, si, rows_per_rank, dtype):
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
def test_matches_masked_gather(tp_size, vocab_size, org_vocab_size, hidden, id_dtype):
"""Every rank's partial output is bit-exact with the eager path, including
the added-vocab (LoRA) rows and the ids owned by another rank."""
the added-vocab (LoRA) rows and the ids owned by another rank.
"""
set_random_seed(7)
dtype = torch.bfloat16
rows = pad_vocab_size(vocab_size, 64) // tp_size
+2 -1
View File
@@ -59,7 +59,8 @@ class TestAutotuneDisabledKernel:
def test_autotune_disabled_kernel_produces_valid_config(self):
"""Register a kernel with no configs (disabled), run autotune,
verify it produces a valid helion.Config."""
verify it produces a valid helion.Config.
"""
with dummy_kernel_registry(configs={}) as register:
wrapper = register(
"autotune_test_kernel",
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for Helion ConfigManager and ConfigSet.
"""Unit tests for Helion ConfigManager and ConfigSet.
Tests the simplified configuration management system for Helion custom kernels.
"""
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel
"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel.
Run `pytest tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py`.
"""
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the fused_qk_norm_rope helion kernel
"""Tests for the fused_qk_norm_rope helion kernel.
Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`.
"""
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for Helion kernel availability and basic functionality.
"""Tests for Helion kernel availability and basic functionality.
This module demonstrates the pattern for testing optional Helion kernels.
Tests in this directory will be skipped if Helion is not installed.
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the per_token_group_fp8_quant helion kernel
"""Tests for the per_token_group_fp8_quant helion kernel.
Run `pytest tests/kernels/helion/test_per_token_group_fp8_quant.py`.
"""
+7 -6
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for Helion kernel registration.
"""Unit tests for Helion kernel registration.
Tests ConfiguredHelionKernel, HelionKernelWrapper, and PresetConfigSearch
including config picker registration and custom autotuner integration.
@@ -599,7 +598,8 @@ class TestHelionKernelWrapper:
)
def test_init_eagerly_initializes_hop_path(self):
"""Test that register_kernel eagerly builds the configured kernel
on the HOP path (no custom op registration needed)."""
on the HOP path (no custom op registration needed).
"""
from vllm.kernels.helion.utils import get_canonical_gpu_name
configs: dict[CaseKey, helion.Config] = {
@@ -634,7 +634,8 @@ class TestHelionKernelWrapper:
)
def test_init_eagerly_initializes(self):
"""Test that register_kernel eagerly loads configs and detects GPU
during construction so __call__ needs no further initialization."""
during construction so __call__ needs no further initialization.
"""
from vllm.kernels.helion.utils import get_canonical_gpu_name
with (
@@ -962,7 +963,8 @@ class TestTorchCompileHOP:
def test_compiled_graph_contains_helion_hop(self):
"""Verify torch.compile on a HelionKernelWrapper emits a
helion_kernel_wrapper_mutation HOP node in the FX graph."""
helion_kernel_wrapper_mutation HOP node in the FX graph.
"""
configs: dict[CaseKey, helion.Config] = {
CaseKey.default(): helion.Config(block_sizes=[4, 4])
}
@@ -1020,7 +1022,6 @@ class TestTorchCompileHOP:
)
def test_inductor_backend_compiles_helion_hop(self):
"""Test torch.compile with inductor backend and Helion fusion enabled."""
configs: dict[CaseKey, helion.Config] = {
CaseKey.default(): helion.Config(block_sizes=[4, 4])
}
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the rms_norm_dynamic_per_token_quant helion kernel
"""Tests for the rms_norm_dynamic_per_token_quant helion kernel.
Run `pytest tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py`.
"""
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the rms_norm_per_block_quant helion kernel
"""Tests for the rms_norm_per_block_quant helion kernel.
Run `pytest tests/kernels/helion/test_rms_norm_per_block_quant.py`.
"""
+1 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Helion Kernel test utils"""
"""Helion Kernel test utils."""
import pytest
import torch
+1 -1
View File
@@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif(
def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim):
"""q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
"""Q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim
qkv = torch.empty(num_tokens, q_size + 2 * kv_size)
q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0]
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Meta-tests for vLLM IR op infrastructure.
"""Meta-tests for vLLM IR op infrastructure.
Ensures all registered ops have input generators defined.
Per-op correctness tests live alongside their op definitions
+2 -1
View File
@@ -368,7 +368,8 @@ class TestFusedAddRMSNorm:
@pytest.mark.parametrize("provider", ["vllm_c"])
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
"""Test that inplace implementations reuse inputs,
for maybe_inplace overload but not for default overload."""
for maybe_inplace overload but not for default overload.
"""
impl = ir.ops.fused_add_rms_norm.impls[provider]
if not impl.supported:
pytest.skip(f"{provider} impl not supported on this platform")
+6 -3
View File
@@ -1055,7 +1055,8 @@ CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
"""High-precision conv reference: everything in float32, no bf16 rounding.
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim]."""
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim].
"""
xf = x.float()
wf = weight.float().unsqueeze(1)
bf = bias.float()
@@ -1164,7 +1165,8 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
conv_state view + is_vnni=True) must match the torch fallback. Validates
the packing + layout handoff on any AVX-512BF16 CPU."""
the packing + layout handoff on any AVX-512BF16 CPU.
"""
x, weight, bias = _conv_inputs(sum(seq_lens))
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
@@ -1178,7 +1180,8 @@ def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
@torch.inference_mode()
def test_conv_update_cpp_matches_torch(batch):
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
including the in-place conv_state update (the next-step handoff)."""
including the in-place conv_state update (the next-step handoff).
"""
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_torch,
)
+3 -6
View File
@@ -36,8 +36,7 @@ def causal_conv1d_ref(
final_states_out: torch.Tensor | None = None,
activation: str | None = "silu",
):
"""
x: (batch, dim, seqlen)
"""x: (batch, dim, seqlen)
weight: (dim, width)
bias: (dim,)
initial_states: (batch, dim, width - 1)
@@ -72,8 +71,7 @@ def causal_conv1d_ref(
def causal_conv1d_update_ref(
x, conv_state, weight, bias=None, activation=None, cache_seqlens=None
):
"""
x: (batch, dim) or (batch, dim, seqlen)
"""x: (batch, dim) or (batch, dim, seqlen)
conv_state: (batch, dim, state_len), where state_len >= width - 1
weight: (dim, width)
bias: (dim,)
@@ -136,8 +134,7 @@ def causal_conv1d_opcheck_fn(
activation: str | None = "silu",
null_block_id: int = NULL_BLOCK_ID,
):
"""
x: (batch, dim, seqlen)
"""x: (batch, dim, seqlen)
weight: (dim, width)
bias: (dim,)
seq_idx: (batch, seqlen)
+2 -3
View File
@@ -51,8 +51,7 @@ def selective_scan_ref(
prev_state=None,
final_state_out=None,
):
"""
u: r(B D L)
"""u: r(B D L)
delta: r(B D L)
A: c(D N) or r(D N)
B: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L)
@@ -135,7 +134,7 @@ def selective_scan_opcheck_fn(
cu_chunk_seqlen=None,
last_chunk_indices=None,
):
"""if return_last_state is True, returns (out, last_state)
"""If return_last_state is True, returns (out, last_state)
last_state has shape (batch, dim, dstate).
"""
if u.stride(-1) != 1:
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for the JSON-based config loader added to selective_state_update.
"""Unit tests for the JSON-based config loader added to selective_state_update.
Tests cover:
- Flat MoE-style filename generation
@@ -123,7 +122,8 @@ def test_fallback_when_no_config(monkeypatch, tmp_path):
def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
"""When effective_batch = batch*nheads is not an exact key, the closest
key should be selected."""
key should be selected.
"""
_write_config(
tmp_path,
dstate=32,
@@ -168,7 +168,8 @@ def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
def test_non_dict_json_returns_none(monkeypatch, tmp_path):
"""A valid JSON file that is not a dict (e.g. a list) must be ignored
and return None rather than raising AttributeError."""
and return None rather than raising AttributeError.
"""
device_name = get_ssm_device_name()
config_path = tmp_path / get_ssm_config_file_name(
_HEADDIM, 16, _CACHE_DTYPE, device_name
@@ -190,7 +191,8 @@ def test_non_dict_json_returns_none(monkeypatch, tmp_path):
def test_empty_config_falls_back_to_heuristic(monkeypatch, tmp_path):
"""An empty JSON object {} must not crash min() — should fall back
to the hard-coded heuristic."""
to the hard-coded heuristic.
"""
_write_config(tmp_path, dstate=64, payload={})
monkeypatch.setenv("VLLM_TUNED_CONFIG_FOLDER", str(tmp_path))
+3 -2
View File
@@ -41,14 +41,15 @@ def segsum(x):
def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None):
"""
Arguments:
"""Arguments:
X: (batch, length, n_heads, d_head)
A: (batch, length, n_heads)
B: (batch, length, n_heads, d_state)
C: (batch, length, n_heads, d_state)
Return:
Y: (batch, length, n_heads, d_head)
"""
assert X.dtype == A.dtype == B.dtype == C.dtype
assert X.shape[1] % block_len == 0
@@ -68,7 +68,8 @@ def _run_prefill_decode_equivalence(
the exact fp32 recurrence (and each other). All paths use the production dt
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
is the recurrent-state precision; ``act_dtype`` the activation/buffer one.
"""
device = current_platform.device_type
rtol, atol = _prefill_tolerances(act_dtype)
set_random_seed(seed)
@@ -95,7 +95,8 @@ def _run_standard_decode(
) -> None:
"""Drive ``num_steps`` decode steps and check every path against the
trusted recurrence. ``state_dtype`` is the recurrent-state precision;
``act_dtype`` is the activation/buffer precision."""
``act_dtype`` is the activation/buffer precision.
"""
device = "cuda"
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
@@ -539,7 +540,8 @@ def test_replayssm_standard_decode_tp_head_shard_equivalence(
separable: one run over all heads must equal concatenating ``tp`` independent
per-rank runs on ``nheads // tp`` heads and ``ngroups // tp`` groups. This
guards the per-rank divisors and group->head mapping the state-shape wiring
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite."""
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite.
"""
state_dtype, act_dtype = precision
nheads, headdim, dstate, ngroups = geometry
assert nheads % tp == 0 and ngroups % tp == 0
+3 -2
View File
@@ -9,8 +9,7 @@ from einops import rearrange, repeat
def selective_state_update_ref(
state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False
):
"""
Argument:
"""Argument:
state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
x: (batch, dim) or (batch, nheads, dim)
dt: (batch, dim) or (batch, nheads, dim)
@@ -20,8 +19,10 @@ def selective_state_update_ref(
D: (dim,) or (nheads, dim)
z: (batch, dim) or (batch, nheads, dim)
dt_bias: (dim,) or (nheads, dim)
Return:
out: (batch, dim) or (batch, nheads, dim)
"""
has_heads = state.dim() > 3
if state.dim() == 3:
@@ -148,9 +148,7 @@ class Config:
return self.E // self.world_size
def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]:
"""
make env data for vllm launch.
"""
"""Make env data for vllm launch."""
vllm_config = VllmConfig()
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
@@ -169,7 +167,8 @@ class Config:
def fe_supports_quant_scheme(self) -> bool:
"""Check if the fused experts class supports this quant config.
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.
"""
if self.quant_config is None or self.quant_dtype is None:
return True
if self.quant_dtype != torch.float8_e4m3fn:
@@ -444,9 +443,7 @@ class RankTensors:
def make_hidden_states(
config: Config,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""
Return hidden_states
"""
"""Return hidden_states."""
m, k, dtype = (config.M, config.K, config.dtype)
device = torch.accelerator.current_device_index()
a = torch.randn((m, k), device=device, dtype=dtype) / 15.0
+1 -3
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
DeepEP test utilities
"""
"""DeepEP test utilities."""
import dataclasses
import os
+2 -1
View File
@@ -468,7 +468,8 @@ def test_deepseek_v4_flashinfer_cutlass_falls_through_to_w4a8(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Explicit flashinfer_cutlass must try the W4A8 variant when the BF16
variant is unsupported (the BF16 variant is gated to SM90)."""
variant is unsupported (the BF16 variant is gated to SM90).
"""
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
@@ -34,7 +34,6 @@ def test_batched_deepgemm_vs_triton(
E: int, T: int, K: int, N: int, topk: int, monkeypatch, workspace_init
):
"""Compare BatchedDeepGemmExperts to BatchedTritonExperts."""
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1")
device = "cuda"
+6 -3
View File
@@ -129,7 +129,8 @@ def test_batched_mm(
per_act_token_quant: bool,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
and those tests will be skipped on unsupported hardware.
"""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
@@ -268,7 +269,8 @@ def test_fused_moe_batched_experts(
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
and those tests will be skipped on unsupported hardware.
"""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
@@ -649,7 +651,8 @@ def test_batched_experts_masked_activation_end_to_end(
def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path."""
the device-enablement path.
"""
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
+4 -2
View File
@@ -54,7 +54,8 @@ SEEDS = [0]
# For test
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
"""This function performs fused moe with block-wise quantization using
native torch."""
native torch.
"""
B, D = a.shape
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
@@ -97,7 +98,8 @@ def setup_cuda():
@torch.inference_mode()
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
native torch reference."""
native torch reference.
"""
torch.manual_seed(seed)
a = torch.randn((M, K), dtype=dtype) / 10
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests compute_expert_num_tokens kernels
"""
"""Tests compute_expert_num_tokens kernels."""
import dataclasses
+4 -3
View File
@@ -397,7 +397,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
class _StubMoELayer(torch.nn.Module):
"""Minimal stand-in for the real MoE layer module, exposing just what
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
router configuration, and optionally w13_bias/w2_bias)."""
router configuration, and optionally w13_bias/w2_bias).
"""
def __init__(
self,
@@ -500,8 +501,8 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
act: MoEActivation,
):
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
weights before prepacking."""
weights before prepacking.
"""
set_random_seed(0)
batch_size = 64
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
+1
View File
@@ -62,6 +62,7 @@ def _make_int4_moe_weights(
Returns:
(w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias)
where *_ref are the dequantized float reference weights
"""
# Generate INT4 weights as int8 values in [-8, 7]
w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8)
@@ -826,6 +826,7 @@ def _make_int4_moe_weights(E, N, K, group_size, quant_algo):
w1_zeros, w2_zeros,
w1_zeros_packed, w2_zeros_packed,
w1_s, w2_s
"""
w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32)
w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32)
+3 -5
View File
@@ -91,11 +91,9 @@ def break_fp4_bytes(a, dtype):
def generate_balanced_routing(
hidden_states: torch.Tensor, num_experts: int, top_k: int
):
"""
Generate routing weights and topk indices such that every expert is active.
"""Generate routing weights and topk indices such that every expert is active.
Returns routing_weights, topk_idx
"""
num_tokens, hidden_dim = hidden_states.shape
# num_tokens = batch_size * seq_len
@@ -232,13 +230,13 @@ def grouped_gemm_ref(
*,
block_size: int = 16,
) -> torch.Tensor:
"""
Computes the reference grouped GEMM (fp4 quantized per-expert loop),
"""Computes the reference grouped GEMM (fp4 quantized per-expert loop),
computes flashinfer grouped GEMM (for scale consistency),
and returns ONLY the repacked reference output: out_ref.
Returns:
out_ref: Tensor [num_experts, max_m, n_out]
"""
device_hs = hidden_states_expanded.device
device_w = weights.device
+4 -12
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test DeepEP + DeepGEMM integration
"""Test DeepEP + DeepGEMM integration
DeepGEMM are gemm kernels specialized for the
fp8 block-quantized case.
"""
@@ -92,9 +91,7 @@ def make_block_quant_fp8_weights(
k: int,
block_size: list[int],
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Return weights w1q, w2q, w1_scale, w2_scale
"""
"""Return weights w1q, w2q, w1_scale, w2_scale."""
(_, w1q, w1_scale, _), (_, w2q, w2_scale, _) = make_test_weights(
e, n, k, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
)
@@ -459,10 +456,7 @@ def test_ht_deepep_deepgemm_moe(
disable_deepgemm_ue8m0,
workspace_init,
):
"""
Tests for High-Throughput DeepEP + DeepGemm integration.
"""
"""Tests for High-Throughput DeepEP + DeepGemm integration."""
m, n, k = mnk
set_random_seed(7)
@@ -533,9 +527,7 @@ def test_ll_deepep_deepgemm_moe(
disable_deepgemm_ue8m0,
workspace_init,
):
"""
Tests for Low-Latency DeepEP + DeepGemm integration.
"""
"""Tests for Low-Latency DeepEP + DeepGemm integration."""
assert not is_deep_gemm_e8m0_used()
m, n, k = mnk
+2 -6
View File
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test deepep dispatch-combine logic
"""
"""Test deepep dispatch-combine logic."""
import dataclasses
@@ -56,9 +54,7 @@ MAX_TOKENS_PER_RANK = 64
def make_weights(
e, n, k, dtype
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Return weights w1, w2, w1_scale, w2_scale
"""
"""Return weights w1, w2, w1_scale, w2_scale."""
if dtype in [torch.float16, torch.bfloat16]:
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
Compares against a pure-PyTorch reference MoE implementation.
"""
+5 -10
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
"""
@@ -79,8 +78,7 @@ def make_block_quant_fp8_weights(
k: int,
block_size: list[int],
):
"""
Generate (w1, w2) expert weights and their per-block scale tensors
"""Generate (w1, w2) expert weights and their per-block scale tensors
in FP8 block-quantized format.
w1 shape: (E, 2N, K)
@@ -121,8 +119,7 @@ def make_block_quant_fp8_weights(
def run_single_case(m, n, k, topk, num_experts, block_size):
"""
Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
Triton baseline within tolerance.
"""
tokens_bf16 = (
@@ -248,8 +245,7 @@ def make_mxfp4_weights(
n: int,
k: int,
):
"""
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
plus BF16 reference weights for validation.
w1 shape: (E, 2N, K//2) uint8 — packed FP4
@@ -310,8 +306,7 @@ def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
def run_single_fp4_case(m, n, k, topk, num_experts):
"""
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
DeepGEMM FP4 == BF16 reference within tolerance.
"""
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
+2 -1
View File
@@ -464,7 +464,8 @@ def test_trtllm_fp8_swiglu_clamp_support(
):
"""FlashInfer >= 0.6.18 applies the SwiGLU clamp for both block-scaled
kernels with a SwiGLU activation (DeepSeek-V4 sets swiglu_limit); the
per-tensor kernel has no clamp, and Relu2 rejects the parameters."""
per-tensor kernel has no clamp, and Relu2 rejects the parameters.
"""
class _Experts(TrtLlmFp8ExpertsModular):
@staticmethod
@@ -83,7 +83,7 @@ def _reference_activation(
beta: float | None,
limit: float | None,
):
"""vLLM's own op for this activation, so the reference is not re-derived."""
"""VLLM's own op for this activation, so the reference is not re-derived."""
if activation == MoEActivation.RELU2_NO_MUL:
return lambda x: torch.square(torch.relu(x))
if activation == MoEActivation.SWIGLUOAI:
+1 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoE fused topk kernel
"""Tests for the MoE fused topk kernel.
Run `pytest tests/kernels/moe/test_fused_topk.py`.
"""
@@ -377,7 +377,8 @@ def test_unit_shuffle():
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
"""routing_data_from_sparse_topk must produce routing structures
identical to make_routing_data for the same topk result."""
identical to make_routing_data for the same topk result.
"""
from vllm.model_executor.layers.fused_moe.experts import (
gpt_oss_triton_kernels_moe as gptoss_moe,
)
+1 -1
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MoE grouped topk kernel
"""Tests for the MoE grouped topk kernel.
Run `pytest tests/kernels/moe/test_grouped_topk.py`.
"""
@@ -31,6 +31,7 @@ def mxint4_quantize(
Returns:
- uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7]
- scales in BF16: [..., k//sf_vec_size]
"""
x_reshaped = x.reshape(-1, sf_vec_size)
x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32)
@@ -61,6 +62,7 @@ def mxint4_quantize_moe_weights(
Returns:
- weights_mxint4: Quantized weights [e, n, k//2] uint8
- scales_mxint4: Quantization scales [e, n, k//group_size] bf16
"""
e = weights_bf16.shape[0]
weight_list = []
@@ -101,6 +103,7 @@ def marlin_quantize_moe_weights(
Returns:
- weights_marlin: Marlin quantized weights [e, k//8, n] int32
- scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16
"""
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
marlin_quantize,
@@ -286,7 +289,8 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
@torch.inference_mode()
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
trtllm_mxint4_block_scale_moe kernel call."""
trtllm_mxint4_block_scale_moe kernel call.
"""
pytest.importorskip("flashinfer")
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
@@ -352,7 +352,8 @@ def test_modular_kernel_combinations_singlegpu(
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
and those tests will be skipped on unsupported hardware.
"""
config = Config(
Ms=Ms,
K=k,
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test modular OAI Triton MoE
"""
"""Test modular OAI Triton MoE."""
from __future__ import annotations
+4 -7
View File
@@ -119,8 +119,7 @@ def iterative_moe(
expert_map: torch.Tensor = None,
renormalize: bool = False,
) -> torch.Tensor:
"""
Baseline implementation of fused moe.
"""Baseline implementation of fused moe.
Args:
hidden_states: [*, hidden_size]
@@ -128,6 +127,7 @@ def iterative_moe(
w2: [num_experts, hidden_size, intermediate_size]
gating_output: [*, num_experts]
expert_map: [num_experts]
"""
orig_shape = hidden_states.shape
hidden_size = hidden_states.shape[-1]
@@ -1956,8 +1956,7 @@ def test_batched_fused_marlin_moe(
)
def is_valid(self):
"""
Return True only if the input can be represented in a Batched
"""Return True only if the input can be represented in a Batched
format.
"""
return torch.all(self.expert_num_tokens_cpu <= self.max_tokens_per_batch)
@@ -2078,9 +2077,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
dtype: torch.dtype,
workspace_init,
):
"""
Test BF16 unquantized MoE with FlashInfer TRTLLM backend.
"""
"""Test BF16 unquantized MoE with FlashInfer TRTLLM backend."""
set_random_seed(7)
from vllm.model_executor.layers.fused_moe.config import (
@@ -54,8 +54,7 @@ def _verify_expert_level_sorting(
valid_length: int,
total_tokens: int,
):
"""
Verify that actual_sorted_ids follows the correct expert-level sorting.
"""Verify that actual_sorted_ids follows the correct expert-level sorting.
The kerne limplementation may or may not preserve original token order
in topk_ids in the final sorted_ids however this does not impact quality.
"""
@@ -96,8 +95,7 @@ def torch_moe_align_block_size(
expert_map: torch.Tensor | None = None,
pad_sorted_ids: bool = False,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""
Golden torch implementation of moe_align_block_size.
"""Golden torch implementation of moe_align_block_size.
This function aligns the token distribution across experts to be compatible
with block size for matrix multiplication by sorting tokens by expert and
@@ -189,7 +187,7 @@ def torch_moe_align_block_size(
def test_moe_align_block_size(
m: int, topk: int, num_experts: int, block_size: int, pad_sorted_ids: bool
):
"""Test moe_align_block_size without expert mapping"""
"""Test moe_align_block_size without expert mapping."""
topk_ids = torch.zeros((m, topk), device="cuda", dtype=torch.int32)
for i in range(m):
experts = torch.randperm(num_experts, device="cuda")[:topk]
@@ -254,7 +252,7 @@ def test_moe_align_block_size_with_expert_map(
block_size: int,
mask_inactive_experts: bool,
):
"""Test moe_align_block_size with expert mapping (EP scenario)"""
"""Test moe_align_block_size with expert mapping (EP scenario)."""
expert_map = torch.full((num_experts,), -1, device="cuda", dtype=torch.int32)
local_experts = list(range(0, num_experts, 2))
for i, expert_id in enumerate(local_experts):
+2 -1
View File
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
class TestUnquantizedDelegation:
"""UnquantizedMoEKernelOracle methods must delegate to the existing
module-level functions; behaviour is bit-identical."""
module-level functions; behaviour is bit-identical.
"""
def test_make_kernel_delegates(self) -> None:
quant_config = object()
+6 -4
View File
@@ -240,8 +240,7 @@ def maybe_roundup_layer_hidden_size(
act_dtype: torch.dtype,
backend: str | None,
) -> int:
"""
Given layer hidden size and MoE configurations, round up hidden_size
"""Given layer hidden size and MoE configurations, round up hidden_size
if necessary.
Args:
@@ -253,6 +252,7 @@ def maybe_roundup_layer_hidden_size(
Rounded up hidden_size if rounding up is required based on the configs
and all2all backend.
Original hidden size otherwise.
"""
if backend == "deepep_high_throughput":
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
@@ -315,7 +315,8 @@ def tp_chunk_gate_up(
device: torch.device | int | None = None,
) -> torch.Tensor:
"""TP-chunk a combined [gate; up] weight, splitting each half separately
so every rank gets a portion of both gate and up."""
so every rank gets a portion of both gate and up.
"""
half = w.shape[dim] // 2
gate = chunk_by_rank(
w.narrow(dim, 0, half), tp_rank, tp_size, dim=dim, device=device
@@ -945,6 +946,7 @@ def create_shared_experts_from_config(
Returns:
TestMLP instance or None if config is None
"""
if shared_experts_config is None:
return None
@@ -991,6 +993,7 @@ def setup_moe_test_data(
Returns:
MoETestData containing all test data and transforms
"""
# For latent MoE: latent_size = k // 2
latent_size = k // 2
@@ -1710,7 +1713,6 @@ def test_moe_layer_no_parallel(
monkeypatch,
):
"""Test MoE layer without parallelism (dp_size=1, tp_size=1, use_ep=False)."""
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the MOE permute/unpermute kernel
"""Tests for the MOE permute/unpermute kernel.
Run `pytest tests/kernels/test_moe_permute_unpermute.py`.
"""
+4 -9
View File
@@ -67,8 +67,7 @@ def compute_ref_output(
@pytest.mark.parametrize("num_experts", [8, 16, 32])
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
"""
Test the MXFP4 grouped GEMM kernel by:
"""Test the MXFP4 grouped GEMM kernel by:
1. Creating random per-expert inputs and weights
2. Quantizing both to MXFP4 using the CUDA kernel
3. Running the CUTLASS grouped GEMM
@@ -205,9 +204,7 @@ def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
reason="mxfp4_experts_quant requires CUDA SM100",
)
def test_mxfp4_experts_quant_basic():
"""
Basic smoke test for the MXFP4 experts quantization kernel.
"""
"""Basic smoke test for the MXFP4 experts quantization kernel."""
device = "cuda"
num_experts = 4
k = 256
@@ -295,8 +292,7 @@ def compute_reference_e8m0_scale(block_max: float) -> int:
@pytest.mark.parametrize("k", [256, 7168])
@pytest.mark.parametrize("m", [16, 64])
def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
"""
Test that mxfp4_experts_quant computes E8M0 block scales correctly
"""Test that mxfp4_experts_quant computes E8M0 block scales correctly
per OCP MX spec (not the NVFP4 formula).
The old buggy kernel used: floor(log2(max/6)) + 127
@@ -416,8 +412,7 @@ def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
reason="mxfp4_experts_quant requires CUDA SM100",
)
def test_mxfp4_experts_quant_no_saturation():
"""
Test that the E8M0 scale is large enough to avoid unexpected saturation.
"""Test that the E8M0 scale is large enough to avoid unexpected saturation.
With the buggy NVFP4 formula, the scale was too small causing most values
to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
@@ -130,7 +130,8 @@ def test_is_supported_config(present, ep_size, supported, reason_substr):
def test_explicit_moe_backend_aiter():
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
ValueError when the flydsl package is missing."""
ValueError when the flydsl package is missing.
"""
with _gfx950(), _flydsl_installed(True):
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
@@ -161,8 +162,9 @@ def test_gfx950_picks_aiter():
def test_gfx942_picks_emulation():
"""flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
dot_scaled backend wins instead."""
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
dot_scaled backend wins instead.
"""
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
backend, experts_cls = select_mxfp8_moe_backend(_config())
assert backend is Fp8MoeBackend.EMULATION
+4 -6
View File
@@ -214,8 +214,7 @@ def reference_moe(
activation: str = "swiglu",
use_interleaved_layout: bool = False,
):
"""
Reference MoE implementation for accuracy testing.
"""Reference MoE implementation for accuracy testing.
Args:
activation: One of "swiglu", "silu", "relu2". Controls the activation
@@ -224,6 +223,7 @@ def reference_moe(
(gate=x[..., ::2], up=x[..., 1::2]) as used by SWIGLUOAI.
If False, uses chunked layout (gate, up = chunk(x, 2)) as used
by standard swiglu/silu.
"""
# renormalize routing
experts = torch.topk(roouting_logits, k=topk, dim=-1, sorted=True)
@@ -1441,8 +1441,7 @@ def test_rocm_mxfp4_moe_oracle(
intermediate_size: int,
monkeypatch: pytest.MonkeyPatch,
):
"""
Test ROCm MXFP4 MoE using oracle functions.
"""Test ROCm MXFP4 MoE using oracle functions.
This test validates that the oracle functions work end-to-end:
- select_mxfp4_moe_backend() selects a valid backend
@@ -1825,8 +1824,7 @@ def test_mxfp4_emulation_rounds_up_to_block_size(
def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
monkeypatch: pytest.MonkeyPatch,
):
"""
select_mxfp4_moe_backend() must raise NotImplementedError, with the
"""select_mxfp4_moe_backend() must raise NotImplementedError, with the
collected per-backend unsupported reasons in the message, when no
backend supports the requested deployment configuration.
"""
+22 -11
View File
@@ -249,7 +249,8 @@ def _make_topk_ids(
device: str = "cuda",
) -> torch.Tensor:
"""Generate distinct expert IDs per token with the same top-k shape as
production routing."""
production routing.
"""
router_logits = torch.randn(num_tokens, num_experts, device=device)
_, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1)
return topk_ids.to(torch.int32)
@@ -443,7 +444,8 @@ def test_aiter_fused_moe_custom_op_registered():
def test_aiter_asm_moe_tkw1_custom_op_registered():
"""The tkw1 custom op should stay registered for FP8 apply-router-weight
paths."""
paths.
"""
_assert_aiter_supported()
import vllm._aiter_ops as aiter_ops # noqa: F401
@@ -453,7 +455,8 @@ def test_aiter_asm_moe_tkw1_custom_op_registered():
def test_aiter_fused_moe_fake_tensor_support():
"""The fused-MoE op should preserve fake-tensor compatibility for
torch.compile-style tracing."""
torch.compile-style tracing.
"""
_assert_aiter_supported()
import vllm._aiter_ops # noqa: F401
@@ -516,7 +519,8 @@ def test_aiter_moe_enablement_follows_env(
monkeypatch: pytest.MonkeyPatch,
):
"""The fused-MoE gate should depend only on the main AITER toggle and the
MoE-specific toggle."""
MoE-specific toggle.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -547,7 +551,8 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
monkeypatch: pytest.MonkeyPatch,
):
"""Shared-expert fusion should only be enabled when the fused-MoE path is
enabled too."""
enabled too.
"""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -688,7 +693,8 @@ def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
"""The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized
MXFP4 reference."""
MXFP4 reference.
"""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
@@ -737,7 +743,8 @@ def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism():
"""The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise
deterministic."""
deterministic.
"""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
@@ -830,7 +837,8 @@ def test_aiter_fused_moe_bf16_accuracy(
intermediate_dim: int,
):
"""The ROCm AITER fused-MoE BF16 path should match the float32 reference
on representative shapes."""
on representative shapes.
"""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -875,7 +883,8 @@ def test_aiter_fused_moe_bf16_accuracy(
def test_aiter_fused_moe_gelu_accuracy():
"""The GELU activation variant should stay aligned with the float32
reference."""
reference.
"""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -920,7 +929,8 @@ def test_aiter_fused_moe_gelu_accuracy():
def test_aiter_fused_moe_determinism():
"""The BF16 fused-MoE kernel should stay bitwise deterministic for the
same inputs."""
same inputs.
"""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -1075,7 +1085,8 @@ def test_aiter_fused_moe_mi3xx_bf16_accuracy():
)
def test_aiter_fused_moe_mi3xx_fp8_accuracy():
"""The MI3xx FP8 per-tensor MoE path should stay within the measured FP8
error budget."""
error budget.
"""
from tests.kernels.moe.utils import make_test_weights
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
@@ -185,7 +185,8 @@ def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts(
) -> None:
"""The capture callback should receive the int16 routed-expert IDs the
kernel actually used, the values should be valid expert indices, and
each token should pick ``top_k`` distinct experts."""
each token should pick ``top_k`` distinct experts.
"""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
@@ -264,7 +265,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
routing_method: RoutingMethodType,
) -> None:
"""Routing replay works for non-DeepSeekV3 routing methods too.
FlashInfer's ``routing_replay_out`` is routing-method-agnostic."""
FlashInfer's ``routing_replay_out`` is routing-method-agnostic.
"""
torch.manual_seed(0)
device = torch.device("cuda:0")
@@ -317,7 +319,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
"""With no callback installed the kernel should not see a
``routing_replay_out`` tensor — verify the helper short-circuits."""
``routing_replay_out`` tensor — verify the helper short-circuits.
"""
torch.manual_seed(0)
device = torch.device("cuda:0")
experts, _, _ = _make_bf16_monolithic_experts(
@@ -339,7 +342,8 @@ def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
"""FlashInfer's ``routing_replay_out`` is supported by all routing
methods, so ``supports_routing_replay_capture`` should be True
regardless of routing method."""
regardless of routing method.
"""
device = torch.device("cuda:0")
for routing_method in (
RoutingMethodType.DeepSeekV3,
@@ -361,7 +365,8 @@ def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None:
"""When capture is installed, the allocated buffer is int16 and shaped
``(num_tokens, experts_per_token)``."""
``(num_tokens, experts_per_token)``.
"""
device = torch.device("cuda:0")
experts, _, _ = _make_bf16_monolithic_experts(
num_experts=_DSV3_NUM_EXPERTS,
@@ -592,7 +597,8 @@ def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts(
top_k: int,
) -> None:
"""End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale
path, DSV3 routing) captures valid expert IDs."""
path, DSV3 routing) captures valid expert IDs.
"""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
@@ -778,7 +784,8 @@ def _run_nvfp4_monolithic(
routing_bias: torch.Tensor,
) -> torch.Tensor:
"""The monolithic NVFP4 apply expects packed fp4 hidden states + the
matching fp8 per-block scale stored in the ``a1q_scale`` slot."""
matching fp8 per-block scale stored in the ``a1q_scale`` slot.
"""
# Stash the weight tensors on the experts in the locations the apply()
# implementation reads from (it pulls them from quant_config / scales
# already; w1/w2 come in as args).
@@ -806,7 +813,8 @@ def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts(
top_k: int,
) -> None:
"""End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs
on the DSV3 routing path."""
on the DSV3 routing path.
"""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
+7 -11
View File
@@ -214,8 +214,7 @@ def assert_routing_results_close(
rtol: float = 1e-3,
atol: float = 1e-3,
):
"""
Compare routing results, sorting by expert ID first to handle non-deterministic
"""Compare routing results, sorting by expert ID first to handle non-deterministic
ordering from sorted=False in topk.
"""
# Sort both results by expert IDs for consistent comparison
@@ -251,7 +250,8 @@ def assert_aiter_routing_valid(
the Python baseline (different group selection, scoring internals),
so numerical comparison is not meaningful. Instead we verify the
outputs satisfy the routing contract: correct shapes, valid expert
IDs, non-negative weights, and proper normalization."""
IDs, non-negative weights, and proper normalization.
"""
n_tokens = topk_weights.shape[0]
# Shape
@@ -294,8 +294,7 @@ def assert_aiter_routing_valid(
def baseline_fused_topk(
router_logits: torch.Tensor, top_k: int, renormalize: bool
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for standard fused top-k routing.
"""Baseline for standard fused top-k routing.
Algorithm:
1. Apply softmax to router logits
@@ -320,8 +319,7 @@ def baseline_fused_topk_bias(
e_score_correction_bias: torch.Tensor,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for fused top-k with bias correction.
"""Baseline for fused top-k with bias correction.
Algorithm:
1. Apply softmax to router logits
@@ -364,8 +362,7 @@ def baseline_grouped_topk(
e_score_correction_bias: torch.Tensor | None,
routed_scaling_factor: float,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for grouped top-k routing (e.g., DeepSeek).
"""Baseline for grouped top-k routing (e.g., DeepSeek).
Algorithm:
1. Apply scoring function (softmax or sigmoid)
@@ -436,8 +433,7 @@ def baseline_grouped_topk(
def baseline_custom_llama4(
router_logits: torch.Tensor, top_k: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""
Baseline for Llama4 custom routing.
"""Baseline for Llama4 custom routing.
Algorithm:
1. Select top-k expert indices (without softmax)
+5 -4
View File
@@ -1,8 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Test script for the token-to-expert routing simulator.
"""Test script for the token-to-expert routing simulator.
This script demonstrates how to use the routing simulator to test
different routing strategies and analyze their performance, including
@@ -77,7 +76,8 @@ def test_basic_functionality(
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoEFactory."""
FusedMoEFactory.
"""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
@@ -158,7 +158,8 @@ def test_routing_strategy_integration(monkeypatch, device):
def test_distribution_based_routing_with_custom_strategy():
"""Test registering and using DistributionBasedRouting with custom
parameters."""
parameters.
"""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Register custom distribution-based strategy
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for FusedMoEFactory with routed_input_transform.
"""Tests for FusedMoEFactory with routed_input_transform.
Verifies that applying routed_input_transform inside FusedMoEFactory
produces the same results as applying the transform manually outside.
@@ -142,9 +142,7 @@ def silu_mul_quant(
def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor:
"""
pack float32 scales into a int32 tensor
"""
"""Pack float32 scales into a int32 tensor."""
assert x.dtype == torch.float32
E, T, G = x.size()
@@ -170,8 +168,7 @@ def ref_with_scale_fmt(
up: torch.Tensor,
scale_fmt: DeepGemmQuantScaleFMT,
) -> tuple[torch.Tensor, torch.Tensor]:
"""
The precision types of the operations triggered by this function
"""The precision types of the operations triggered by this function
match closely with the kernel implementation so we compare more
accurately.
"""
@@ -204,8 +201,7 @@ def ref_with_scale_fmt(
def token_random(E, T, H2, tokens_per_expert):
"""
Initialize each token in a random range so we test a range of
"""Initialize each token in a random range so we test a range of
scale values.
"""
y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE)
@@ -19,11 +19,9 @@ GROUP_SIZE = 128
def reference_quant(x: torch.Tensor, use_ue8m0: bool):
"""
Reference triton quant kernel from,
"""Reference triton quant kernel from,
vllm.model_executor.layers.quantization.utils.fp8_utils
"""
x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE)
# Allocate the scale tensor in column-major format.
@@ -72,7 +70,8 @@ def reference_with_clamp(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
match the C++ compute() template, then run the standard silu_and_mul +
quant reference."""
quant reference.
"""
N_2 = x.size(1) // 2
dtype = x.dtype
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
@@ -28,7 +28,8 @@ if current_platform.is_fp8_fnuz():
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
"""Matrix multiplication function that supports per-token input
quantization and per-column weight quantization"""
quantization and per-column weight quantization
"""
A = A.to(torch.float32)
B = B.to(torch.float32)
@@ -56,8 +57,8 @@ def fp8_mask(a, mask):
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
"""This function performs fused moe with per-column int8
quantization using native torch."""
quantization using native torch.
"""
B, D = a.shape
# Perform per-token quantization
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for the FlashInfer TRTLLM BF16 MoE backend
"""Tests for the FlashInfer TRTLLM BF16 MoE backend
(`TrtLlmBf16ExpertsModular`).
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Tests for the FlashInfer TRTLLM NvFP4 MoE backend
"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend
(`TrtLlmNvFp4ExpertsModular`).
Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated),
@@ -150,7 +150,8 @@ def test_select_default_backend_by_platform(
expected_backend,
):
"""Test default backend selection per platform with all optional
accelerators (FlashInfer, AITER) disabled."""
accelerators (FlashInfer, AITER) disabled.
"""
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
@@ -444,7 +445,8 @@ def test_select_cuda_flashinfer_cutlass_backend(
mock_has_flashinfer,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available."""
and FlashInfer CUTLASS is available.
"""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
+1 -2
View File
@@ -83,8 +83,7 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
@pytest.mark.parametrize("num_tokens", [1, 32])
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
"""
Run a forward pass through FusedMoEFactory with zero experts
"""Run a forward pass through FusedMoEFactory with zero experts
and verify output shape.
"""
layer, vllm_config = zero_expert_moe
+1 -2
View File
@@ -64,8 +64,7 @@ def make_dummy_moe_config(
max_num_tokens: int = 512,
activation: MoEActivation = MoEActivation.SILU,
) -> FusedMoEConfig:
"""
This is a dummy config for the mk constructor interface
"""This is a dummy config for the mk constructor interface
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
do not actually use this config.
+2 -1
View File
@@ -159,7 +159,8 @@ def native_per_token_group_quant_fp8(
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch."""
`x` using native torch.
"""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
)
@@ -118,6 +118,7 @@ def dequant_nvfp4_kv_cache(
Returns:
[..., num_heads, block_size, head_size] float32.
"""
data_dim = head_size // 2
scale_dim = head_size // 16
@@ -304,7 +304,8 @@ def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel)
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
"""Kernel output should match a dequantized fp32 reference within
fp8 per-token / per-channel quantization noise."""
fp8 per-token / per-channel quantization noise.
"""
weight_shape = (512, 4096) # (N, K)
num_tokens = 32
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
@@ -45,6 +45,7 @@ def quantize_weight_block_fp8(
Returns:
fp8_weight: [N, K] float8_e4m3fn
scales: [n_tiles, k_tiles] float32
"""
N, K = weight.shape
block_n, block_k = block_size
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for cutlass kernels
"""Tests for cutlass kernels.
Run `pytest tests/kernels/quantization/test_cutlass_scaled_mm.py`.
"""

Some files were not shown because too many files have changed in this diff Show More