[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
+3 -6
View File
@@ -247,8 +247,7 @@ def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
class _FakeLinear:
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
.quant_method are read by get_and_maybe_dequant_weights().
"""
.quant_method are read by get_and_maybe_dequant_weights()."""
def __init__(self, weight: torch.Tensor):
self.weight = weight
@@ -319,8 +318,7 @@ class _FakeAttnMetadata:
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
decode attention through W_UK/W_UV.
"""
decode attention through W_UK/W_UV."""
set_random_seed(1)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
@@ -386,8 +384,7 @@ def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
prefill batch with a mix of fresh and continued (cached-prefix)
sequences.
"""
sequences."""
set_random_seed(2)
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
impl, kv_b_weight = _make_amx_mla_impl(
@@ -616,8 +616,7 @@ def test_non_causal_autoselect_backend():
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle.
"""
that it cannot handle."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
@@ -628,8 +627,7 @@ def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.
"""
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -225,8 +225,7 @@ def test_cutlass_mla_decode_cross_layer_view():
"""The kernel must read the cache's page-dim stride instead of assuming
pages are packed back-to-back. A per-layer view into a cross-layer
(block-major) cache has stride(0) inflated by num_layers; outputs must
match a contiguous cache holding the same data exactly.
"""
match a contiguous cache holding the same data exactly."""
device = torch.device("cuda:0")
torch.set_default_dtype(torch.bfloat16)
torch.set_default_device(device)
+1 -2
View File
@@ -162,8 +162,7 @@ def test_fast_decode_plan_importable() -> None:
@torch.inference_mode
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
"""On the first call fast_plan_decode must route through self.plan() and
flip vllm_first_call to False on the wrapper object.
"""
flip vllm_first_call to False on the wrapper object."""
from unittest.mock import patch
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
@@ -134,8 +134,7 @@ def make_quantized_kv_cache(
head_size: int,
) -> tuple:
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
kv_scale, ref_kv_cache, is_nvfp4_kv).
"""
kv_scale, ref_kv_cache, is_nvfp4_kv)."""
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
if is_nvfp4_kv:
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
@@ -73,8 +73,7 @@ def _strided_context_inputs(
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
workspace (left in the fp8 cache layout for a plain fp8 cache).
"""
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
k_nope, v = kv_nope.split((128, 128), dim=-1)
workspace = _randn(num_tokens, 576, dtype=dtype)
@@ -89,8 +89,7 @@ all_case_info: list[tuple] = []
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
scales must not surface as NaN in the merged output.
"""
scales must not surface as NaN in the merged output."""
num_tokens, num_heads, head_size = 6, 8, 128
prefix_output = torch.zeros(
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
+14 -28
View File
@@ -45,8 +45,7 @@ def kv_layout(request) -> KVCacheLayout:
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
"""Per-layer physical stride order for the given layout; the 3-dim
indexer side cache (H=1) is contiguous, so identity.
"""
indexer side cache (H=1) is contiguous, so identity."""
if ndim == 3:
return (0, 1, 2)
stride_order = layout.layer_view_order
@@ -66,8 +65,7 @@ def _main_spec() -> FullAttentionSpec:
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
derived the same way the production allocator does.
"""
derived the same way the production allocator does."""
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
@@ -77,8 +75,7 @@ def _allocate_main_kv_via_contract(
) -> torch.Tensor:
"""Build the main KV cache exactly as the production allocator does for the
given layout: allocate the physical (permuted) tensor, then expose the
inverse-permuted logical [B, H, N, C] view the backend sees.
"""
inverse-permuted logical [B, H, N, C] view the backend sees."""
logical_shape = _main_kv_logical_shape(num_pages)
stride_order = _layer_stride_order(layout, len(logical_shape))
physical_shape = tuple(logical_shape[i] for i in stride_order)
@@ -1379,8 +1376,7 @@ def test_amd_decode_index_topk_end_to_end(
@pytest.mark.parametrize("num_idx_heads", [1, 4])
def test_decode_index_topk_fp8(num_idx_heads: int):
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
matches a reference computed from the dequantized FP8 values.
"""
matches a reference computed from the dequantized FP8 values."""
torch.manual_seed(0)
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
decode_query_len = 1
@@ -1653,8 +1649,7 @@ def test_prefill_sparse_attention_correctness(
def test_main_cache_layout_contract():
"""The standardized per-layer logical shape is [B, H, N, C] with packed
K/V content, and the legacy layout aliases resolve to the expected
per-layer stride orders.
"""
per-layer stride orders."""
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = _main_kv_logical_shape(nb)
assert logical == (nb, h, bs, 2 * d)
@@ -1778,8 +1773,7 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
"""The indexer side cache is standardized 4D with H=1: under both layouts
the allocator's logical view stays contiguous and squeezes (as
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
[num_blocks, block_size, head_dim] cache the kernels consume.
"""
[num_blocks, block_size, head_dim] cache the kernels consume."""
nb = 5
ispec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
@@ -1799,8 +1793,7 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
def test_hnd_allocation_is_packed_head_major():
"""Under HND the backend-visible logical view is the packed head-major
physical allocation.
"""
physical allocation."""
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
logical = _main_kv_logical_shape(nb)
stride_order = KVCacheLayout.LBHNC.layer_view_order
@@ -1828,8 +1821,7 @@ def test_hnd_allocation_is_packed_head_major():
def test_main_cache_is_block_first_and_unpadded():
"""The allocator's contiguous-view branch (not the padded-strided branch)
is used for the main GQA cache: its spec is unpadded and the physical
layout keeps num_blocks as the first dimension under both layouts.
"""
layout keeps num_blocks as the first dimension under both layouts."""
spec = _main_spec()
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
assert spec.page_size_padded is None
@@ -1851,8 +1843,7 @@ def _build_decode_inputs(
):
"""Shared decode setup: uniform query tokens per request, a non-identity
block table, and topk indices selecting the current block plus older causal
blocks for each query token.
"""
blocks for each query token."""
active_batch = len(seq_lens_list)
batch = active_batch + num_padded_reqs
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
@@ -2188,8 +2179,7 @@ def test_decode_sparse_attention_correctness(
):
"""Decode (split-K) parity under both layouts: this is the only coverage of
the decode-site cache feed, and the strided HND case fails if the kernel
ignores the cache strides.
"""
ignores the cache strides."""
torch.manual_seed(0)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
seq_lens_list, decode_query_len, num_padded_reqs
@@ -2239,8 +2229,7 @@ def test_decode_wrong_layout_breaks_parity():
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
reorders the K/V content, so the decode output no longer matches the
reference computed on the correct logical view. The mislabeled tensor keeps
the same shape as the correct view, so the kernel stays in bounds.
"""
the same shape as the correct view, so the kernel stays in bounds."""
torch.manual_seed(0)
seq_lens_list = (130, 257)
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
@@ -2275,8 +2264,7 @@ def test_decode_wrong_layout_breaks_parity():
def test_main_cache_byte_identical_through_production_allocator():
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
`FullAttentionSpec` under HND and assert the backend-visible view has the
same shape, stride, and storage offset as the packed-HND allocation.
"""
same shape, stride, and storage offset as the packed-HND allocation."""
nb = 4
spec = _main_spec()
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
@@ -2291,8 +2279,7 @@ def test_main_cache_byte_identical_through_production_allocator():
def test_padded_main_cache_is_flagged():
"""AC-2.1 negative: the M3 main cache relies on the allocator's
contiguous-view branch (`page_size_padded is None`). A spec that sets
`page_size_padded` is explicitly flagged rather than silently wrong-strided.
"""
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
def _require_unpadded_block_first(spec, stride_order):
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
@@ -2329,8 +2316,7 @@ def test_padded_main_cache_is_flagged():
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
packed-content K/V split views. Writing through those views must persist
into the bound storage under both layouts.
"""
into the bound storage under both layouts."""
torch.manual_seed(0)
num_pages = 4
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
@@ -23,8 +23,7 @@ pytestmark = pytest.mark.skipif(
def test_concat_and_cache_mla_into_unified_slot_view():
"""concat_and_cache_mla must write correctly into a per-layer view whose
block stride is the full unified slot (block-major), with zero bleed into
the other layers' segments of the same slot.
"""
the other layers' segments of the same slot."""
from vllm import _custom_ops as ops
torch.manual_seed(0)
@@ -86,8 +85,7 @@ def test_concat_and_cache_mla_into_unified_slot_view():
def test_flashmla_dense_decode_unified_slot_view():
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
on Hopper) must read a unified-slot block-major view bit-identically to a
contiguous per-layer cache.
"""
contiguous per-layer cache."""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_dense_supported()
@@ -152,8 +150,7 @@ def test_flashmla_dense_decode_unified_slot_view():
def test_flashinfer_mla_dense_decode_unified_slot_view():
"""FlashInfer MLA dense decode must read a unified-slot block-major view
(inflated stride(0), non-zero storage offset) bit-identically to a
contiguous per-layer cache.
"""
contiguous per-layer cache."""
try:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
except ImportError:
@@ -224,8 +221,7 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
cache, with finite nonzero output.
"""
cache, with finite nonzero output."""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_sparse_supported()
@@ -314,8 +310,7 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
def test_indexer_k_quant_and_cache_into_unified_slot_view():
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
write correctly into a per-layer view whose block stride is the full
unified slot, with zero bleed into the other layers' segments.
"""
unified slot, with zero bleed into the other layers' segments."""
from vllm import _custom_ops as ops
torch.manual_seed(0)
@@ -360,8 +355,7 @@ def test_indexer_k_quant_and_cache_into_unified_slot_view():
def test_flashattn_mla_dense_decode_unified_slot_view():
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
block-major view bit-identically to a contiguous per-layer cache.
"""
block-major view bit-identically to a contiguous per-layer cache."""
try:
from vllm.vllm_flash_attn import flash_attn_varlen_func
except ImportError:
@@ -434,8 +428,7 @@ def test_flashattn_mla_dense_decode_unified_slot_view():
def test_flashmla_dense_fp8_decode_unified_slot_view():
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
must read a unified-slot block-major view bit-identically to a contiguous
per-layer fp8 cache.
"""
per-layer fp8 cache."""
import vllm.v1.attention.ops.flashmla as fm
ok, reason = fm.is_flashmla_dense_supported()
@@ -506,8 +499,7 @@ def test_flashmla_dense_fp8_decode_unified_slot_view():
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
unified-slot block-major view bit-identically to a contiguous per-layer
cache.
"""
cache."""
try:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
except ImportError:
+7 -14
View File
@@ -283,8 +283,7 @@ def _run_direct_flash_attn_case(
# Backend contract tests --------------------------------------------------
def test_aiter_mha_backend_contract():
"""The ROCm backend advertises the dtypes, shapes, and attention types it
is designed to handle.
"""
is designed to handle."""
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
@@ -312,8 +311,7 @@ def test_aiter_mha_backend_contract():
def test_aiter_mha_backend_validates_kv_cache_block_size():
"""The backend should reject KV cache specs that cannot be gathered
correctly by the ROCm kernel.
"""
correctly by the ROCm kernel."""
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
from vllm.v1.kv_cache_interface import FullAttentionSpec
@@ -333,8 +331,7 @@ def test_aiter_mha_backend_validates_kv_cache_block_size():
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
"""The backend should trust the ROCm MI3xx probe instead of the raw torch
capability tuple.
"""
capability tuple."""
from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
@@ -359,8 +356,7 @@ def test_aiter_mha_env_flags_control_enablement(
use_aiter, use_mha, expected, monkeypatch
):
"""Both the global AITER flag and the MHA-specific flag must be enabled
before the ROCm MHA path is considered active.
"""
before the ROCm MHA path is considered active."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -378,8 +374,7 @@ def test_aiter_mha_env_flags_control_enablement(
def test_aiter_mha_platform_gate_matches_install_and_arch():
"""The global AITER availability check should only open on ROCm MI3xx when
the aiter package is installed.
"""
the aiter package is installed."""
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found_and_supported
assert is_aiter_found_and_supported() is (
@@ -650,8 +645,7 @@ def test_aiter_mha_multi_batch(num_heads, head_size, dtype):
@pytest.mark.parametrize("num_blocks", DIRECT_NUM_BLOCKS)
def test_aiter_fa_large_block_table_matches_reference(num_blocks):
"""The direct paged-KV path should stay stable for both normal and very
large block tables.
"""
large block tables."""
atol = 2e-2
rtol = 2e-2
_assert_aiter_supported()
@@ -673,8 +667,7 @@ def test_aiter_fa_large_block_table_matches_reference(num_blocks):
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
def test_aiter_fa_sliding_window_matches_reference():
"""The direct kernel should respect the same sliding-window causal mask as
the naive reference implementation.
"""
the naive reference implementation."""
atol = 2e-2
rtol = 2e-2
_assert_aiter_supported()
@@ -111,8 +111,7 @@ def _make_mixed_common_metadata():
def _patch_build_deps(monkeypatch, events=None):
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
on CPU.
"""
on CPU."""
def fake_generate_sparse_seqlen_triton(
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
@@ -153,8 +152,7 @@ def test_build_populates_decode_only_split_fields(monkeypatch):
def test_build_populates_mixed_split_fields(monkeypatch):
"""Mixed decode+prefill batch: split is reported, prefill fields stay
default because this impl always runs the MQA path.
"""
default because this impl always runs the MQA path."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
@@ -125,8 +125,7 @@ def _quantize_to_fp8(tensor: torch.Tensor):
@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
"""Test FP8 KV cache path: quantize K/V to FP8, run kernel with scales,
and compare against BF16 reference output.
"""
and compare against BF16 reference output."""
assert CACHE_SIZE % PAGE_SIZE == 0
dtype = torch.bfloat16
seq_len = L
@@ -249,8 +248,7 @@ def test_decode_attention_cross_layer_view(H_Q, H_KV, D_QK, D_V, is_mla, PAGE_SI
"""The kernel must honor the cache's page-dim stride, not assume pages are
packed back-to-back. A per-layer view into a cross-layer (block-major)
cache has stride(0) inflated by num_layers; outputs must match a
contiguous cache holding the same data exactly.
"""
contiguous cache holding the same data exactly."""
B = 3
seq_len = 1027
CACHE_SIZE = 16384
+2 -4
View File
@@ -41,8 +41,7 @@ def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
def test_fused_embed_norm_matches_reference():
"""Main-model fusion: the residual is the exact gather and the second output
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel.
"""
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
set_random_seed(13)
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
@@ -61,8 +60,7 @@ def test_fused_embed_norm_matches_reference():
@torch.inference_mode()
def test_fused_embed_eh_norm_matches_reference():
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
feeding the untouched model-local ``fused_eh_norm``.
"""
feeding the untouched model-local ``fused_eh_norm``."""
set_random_seed(13)
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
@@ -62,8 +62,7 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
qr_out with row stride q_size + kv_size. Downstream dispatchers that
require packed row-major inputs then reject the tensor and silently fall
back to a slower GEMM path on every decode step.
"""
back to a slower GEMM path on every decode step."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
@@ -89,8 +88,7 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
lifts that bound to 2**31-1.
"""
lifts that bound to 2**31-1."""
device = "cuda"
dtype = torch.bfloat16
q_size, kv_size = 192, 576
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
matching the eager triton kernel output.
"""
matching the eager triton kernel output."""
import pytest
import torch
+1 -2
View File
@@ -579,8 +579,7 @@ def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
contiguous inputs.
"""
contiguous inputs."""
require_aiter()
require_fp8()
from vllm._aiter_ops import rocm_aiter_ops
@@ -87,8 +87,7 @@ def _shard(full_table, si, rows_per_rank, dtype):
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
def test_matches_masked_gather(tp_size, vocab_size, org_vocab_size, hidden, id_dtype):
"""Every rank's partial output is bit-exact with the eager path, including
the added-vocab (LoRA) rows and the ids owned by another rank.
"""
the added-vocab (LoRA) rows and the ids owned by another rank."""
set_random_seed(7)
dtype = torch.bfloat16
rows = pad_vocab_size(vocab_size, 64) // tp_size
+1 -2
View File
@@ -59,8 +59,7 @@ class TestAutotuneDisabledKernel:
def test_autotune_disabled_kernel_produces_valid_config(self):
"""Register a kernel with no configs (disabled), run autotune,
verify it produces a valid helion.Config.
"""
verify it produces a valid helion.Config."""
with dummy_kernel_registry(configs={}) as register:
wrapper = register(
"autotune_test_kernel",
+3 -6
View File
@@ -598,8 +598,7 @@ class TestHelionKernelWrapper:
)
def test_init_eagerly_initializes_hop_path(self):
"""Test that register_kernel eagerly builds the configured kernel
on the HOP path (no custom op registration needed).
"""
on the HOP path (no custom op registration needed)."""
from vllm.kernels.helion.utils import get_canonical_gpu_name
configs: dict[CaseKey, helion.Config] = {
@@ -634,8 +633,7 @@ class TestHelionKernelWrapper:
)
def test_init_eagerly_initializes(self):
"""Test that register_kernel eagerly loads configs and detects GPU
during construction so __call__ needs no further initialization.
"""
during construction so __call__ needs no further initialization."""
from vllm.kernels.helion.utils import get_canonical_gpu_name
with (
@@ -963,8 +961,7 @@ class TestTorchCompileHOP:
def test_compiled_graph_contains_helion_hop(self):
"""Verify torch.compile on a HelionKernelWrapper emits a
helion_kernel_wrapper_mutation HOP node in the FX graph.
"""
helion_kernel_wrapper_mutation HOP node in the FX graph."""
configs: dict[CaseKey, helion.Config] = {
CaseKey.default(): helion.Config(block_sizes=[4, 4])
}
+1 -2
View File
@@ -368,8 +368,7 @@ class TestFusedAddRMSNorm:
@pytest.mark.parametrize("provider", ["vllm_c"])
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
"""Test that inplace implementations reuse inputs,
for maybe_inplace overload but not for default overload.
"""
for maybe_inplace overload but not for default overload."""
impl = ir.ops.fused_add_rms_norm.impls[provider]
if not impl.supported:
pytest.skip(f"{provider} impl not supported on this platform")
+3 -6
View File
@@ -1055,8 +1055,7 @@ CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
"""High-precision conv reference: everything in float32, no bf16 rounding.
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim].
"""
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim]."""
xf = x.float()
wf = weight.float().unsqueeze(1)
bf = bias.float()
@@ -1165,8 +1164,7 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
conv_state view + is_vnni=True) must match the torch fallback. Validates
the packing + layout handoff on any AVX-512BF16 CPU.
"""
the packing + layout handoff on any AVX-512BF16 CPU."""
x, weight, bias = _conv_inputs(sum(seq_lens))
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
@@ -1180,8 +1178,7 @@ def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
@torch.inference_mode()
def test_conv_update_cpp_matches_torch(batch):
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
including the in-place conv_state update (the next-step handoff).
"""
including the in-place conv_state update (the next-step handoff)."""
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_torch,
)
@@ -122,8 +122,7 @@ def test_fallback_when_no_config(monkeypatch, tmp_path):
def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
"""When effective_batch = batch*nheads is not an exact key, the closest
key should be selected.
"""
key should be selected."""
_write_config(
tmp_path,
dstate=32,
@@ -168,8 +167,7 @@ def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
def test_non_dict_json_returns_none(monkeypatch, tmp_path):
"""A valid JSON file that is not a dict (e.g. a list) must be ignored
and return None rather than raising AttributeError.
"""
and return None rather than raising AttributeError."""
device_name = get_ssm_device_name()
config_path = tmp_path / get_ssm_config_file_name(
_HEADDIM, 16, _CACHE_DTYPE, device_name
@@ -191,8 +189,7 @@ def test_non_dict_json_returns_none(monkeypatch, tmp_path):
def test_empty_config_falls_back_to_heuristic(monkeypatch, tmp_path):
"""An empty JSON object {} must not crash min() — should fall back
to the hard-coded heuristic.
"""
to the hard-coded heuristic."""
_write_config(tmp_path, dstate=64, payload={})
monkeypatch.setenv("VLLM_TUNED_CONFIG_FOLDER", str(tmp_path))
@@ -68,8 +68,7 @@ def _run_prefill_decode_equivalence(
the exact fp32 recurrence (and each other). All paths use the production dt
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
is the recurrent-state precision; ``act_dtype`` the activation/buffer one.
"""
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
device = current_platform.device_type
rtol, atol = _prefill_tolerances(act_dtype)
set_random_seed(seed)
@@ -95,8 +95,7 @@ def _run_standard_decode(
) -> None:
"""Drive ``num_steps`` decode steps and check every path against the
trusted recurrence. ``state_dtype`` is the recurrent-state precision;
``act_dtype`` is the activation/buffer precision.
"""
``act_dtype`` is the activation/buffer precision."""
device = "cuda"
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
@@ -540,8 +539,7 @@ def test_replayssm_standard_decode_tp_head_shard_equivalence(
separable: one run over all heads must equal concatenating ``tp`` independent
per-rank runs on ``nheads // tp`` heads and ``ngroups // tp`` groups. This
guards the per-rank divisors and group->head mapping the state-shape wiring
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite.
"""
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite."""
state_dtype, act_dtype = precision
nheads, headdim, dstate, ngroups = geometry
assert nheads % tp == 0 and ngroups % tp == 0
@@ -167,8 +167,7 @@ class Config:
def fe_supports_quant_scheme(self) -> bool:
"""Check if the fused experts class supports this quant config.
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.
"""
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
if self.quant_config is None or self.quant_dtype is None:
return True
if self.quant_dtype != torch.float8_e4m3fn:
+1 -2
View File
@@ -468,8 +468,7 @@ def test_deepseek_v4_flashinfer_cutlass_falls_through_to_w4a8(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Explicit flashinfer_cutlass must try the W4A8 variant when the BF16
variant is unsupported (the BF16 variant is gated to SM90).
"""
variant is unsupported (the BF16 variant is gated to SM90)."""
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
+3 -6
View File
@@ -129,8 +129,7 @@ def test_batched_mm(
per_act_token_quant: bool,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware.
"""
and those tests will be skipped on unsupported hardware."""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
@@ -269,8 +268,7 @@ def test_fused_moe_batched_experts(
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware.
"""
and those tests will be skipped on unsupported hardware."""
set_random_seed(7)
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
@@ -651,8 +649,7 @@ def test_batched_experts_masked_activation_end_to_end(
def test_batched_experts_end_to_end(m, n, k, e, topk):
"""End-to-end BatchedTritonExperts via the reference (no-comms)
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
the device-enablement path.
"""
the device-enablement path."""
from vllm.v1.worker.workspace import init_workspace_manager
set_random_seed(7)
+2 -4
View File
@@ -54,8 +54,7 @@ SEEDS = [0]
# For test
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
"""This function performs fused moe with block-wise quantization using
native torch.
"""
native torch."""
B, D = a.shape
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
@@ -98,8 +97,7 @@ def setup_cuda():
@torch.inference_mode()
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
native torch reference.
"""
native torch reference."""
torch.manual_seed(seed)
a = torch.randn((M, K), dtype=dtype) / 10
+2 -4
View File
@@ -397,8 +397,7 @@ UNALIGNED_INTERMEDIATE_DIM = 176
class _StubMoELayer(torch.nn.Module):
"""Minimal stand-in for the real MoE layer module, exposing just what
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
router configuration, and optionally w13_bias/w2_bias).
"""
router configuration, and optionally w13_bias/w2_bias)."""
def __init__(
self,
@@ -501,8 +500,7 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
act: MoEActivation,
):
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
weights before prepacking.
"""
weights before prepacking."""
set_random_seed(0)
batch_size = 64
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
+1 -2
View File
@@ -464,8 +464,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
):
"""FlashInfer >= 0.6.18 applies the SwiGLU clamp for both block-scaled
kernels with a SwiGLU activation (DeepSeek-V4 sets swiglu_limit); the
per-tensor kernel has no clamp, and Relu2 rejects the parameters.
"""
per-tensor kernel has no clamp, and Relu2 rejects the parameters."""
class _Experts(TrtLlmFp8ExpertsModular):
@staticmethod
@@ -377,8 +377,7 @@ def test_unit_shuffle():
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
"""routing_data_from_sparse_topk must produce routing structures
identical to make_routing_data for the same topk result.
"""
identical to make_routing_data for the same topk result."""
from vllm.model_executor.layers.fused_moe.experts import (
gpt_oss_triton_kernels_moe as gptoss_moe,
)
@@ -289,8 +289,7 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
@torch.inference_mode()
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
trtllm_mxint4_block_scale_moe kernel call.
"""
trtllm_mxint4_block_scale_moe kernel call."""
pytest.importorskip("flashinfer")
from flashinfer import RoutingMethodType
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
@@ -352,8 +352,7 @@ def test_modular_kernel_combinations_singlegpu(
workspace_init,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware.
"""
and those tests will be skipped on unsupported hardware."""
config = Config(
Ms=Ms,
K=k,
+1 -2
View File
@@ -21,8 +21,7 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
class TestUnquantizedDelegation:
"""UnquantizedMoEKernelOracle methods must delegate to the existing
module-level functions; behaviour is bit-identical.
"""
module-level functions; behaviour is bit-identical."""
def test_make_kernel_delegates(self) -> None:
quant_config = object()
+1 -2
View File
@@ -315,8 +315,7 @@ def tp_chunk_gate_up(
device: torch.device | int | None = None,
) -> torch.Tensor:
"""TP-chunk a combined [gate; up] weight, splitting each half separately
so every rank gets a portion of both gate and up.
"""
so every rank gets a portion of both gate and up."""
half = w.shape[dim] // 2
gate = chunk_by_rank(
w.narrow(dim, 0, half), tp_rank, tp_size, dim=dim, device=device
@@ -130,8 +130,7 @@ def test_is_supported_config(present, ep_size, supported, reason_substr):
def test_explicit_moe_backend_aiter():
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
ValueError when the flydsl package is missing.
"""
ValueError when the flydsl package is missing."""
with _gfx950(), _flydsl_installed(True):
assert (
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
@@ -163,8 +162,7 @@ def test_gfx950_picks_aiter():
def test_gfx942_picks_emulation():
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
dot_scaled backend wins instead.
"""
dot_scaled backend wins instead."""
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
backend, experts_cls = select_mxfp8_moe_backend(_config())
assert backend is Fp8MoeBackend.EMULATION
+11 -22
View File
@@ -249,8 +249,7 @@ def _make_topk_ids(
device: str = "cuda",
) -> torch.Tensor:
"""Generate distinct expert IDs per token with the same top-k shape as
production routing.
"""
production routing."""
router_logits = torch.randn(num_tokens, num_experts, device=device)
_, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1)
return topk_ids.to(torch.int32)
@@ -444,8 +443,7 @@ def test_aiter_fused_moe_custom_op_registered():
def test_aiter_asm_moe_tkw1_custom_op_registered():
"""The tkw1 custom op should stay registered for FP8 apply-router-weight
paths.
"""
paths."""
_assert_aiter_supported()
import vllm._aiter_ops as aiter_ops # noqa: F401
@@ -455,8 +453,7 @@ def test_aiter_asm_moe_tkw1_custom_op_registered():
def test_aiter_fused_moe_fake_tensor_support():
"""The fused-MoE op should preserve fake-tensor compatibility for
torch.compile-style tracing.
"""
torch.compile-style tracing."""
_assert_aiter_supported()
import vllm._aiter_ops # noqa: F401
@@ -519,8 +516,7 @@ def test_aiter_moe_enablement_follows_env(
monkeypatch: pytest.MonkeyPatch,
):
"""The fused-MoE gate should depend only on the main AITER toggle and the
MoE-specific toggle.
"""
MoE-specific toggle."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -551,8 +547,7 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
monkeypatch: pytest.MonkeyPatch,
):
"""Shared-expert fusion should only be enabled when the fused-MoE path is
enabled too.
"""
enabled too."""
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
@@ -693,8 +688,7 @@ def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
"""The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized
MXFP4 reference.
"""
MXFP4 reference."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
@@ -743,8 +737,7 @@ def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism():
"""The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise
deterministic.
"""
deterministic."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
@@ -837,8 +830,7 @@ def test_aiter_fused_moe_bf16_accuracy(
intermediate_dim: int,
):
"""The ROCm AITER fused-MoE BF16 path should match the float32 reference
on representative shapes.
"""
on representative shapes."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -883,8 +875,7 @@ def test_aiter_fused_moe_bf16_accuracy(
def test_aiter_fused_moe_gelu_accuracy():
"""The GELU activation variant should stay aligned with the float32
reference.
"""
reference."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -929,8 +920,7 @@ def test_aiter_fused_moe_gelu_accuracy():
def test_aiter_fused_moe_determinism():
"""The BF16 fused-MoE kernel should stay bitwise deterministic for the
same inputs.
"""
same inputs."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
@@ -1085,8 +1075,7 @@ def test_aiter_fused_moe_mi3xx_bf16_accuracy():
)
def test_aiter_fused_moe_mi3xx_fp8_accuracy():
"""The MI3xx FP8 per-tensor MoE path should stay within the measured FP8
error budget.
"""
error budget."""
from tests.kernels.moe.utils import make_test_weights
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
@@ -185,8 +185,7 @@ def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts(
) -> None:
"""The capture callback should receive the int16 routed-expert IDs the
kernel actually used, the values should be valid expert indices, and
each token should pick ``top_k`` distinct experts.
"""
each token should pick ``top_k`` distinct experts."""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
@@ -265,8 +264,7 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
routing_method: RoutingMethodType,
) -> None:
"""Routing replay works for non-DeepSeekV3 routing methods too.
FlashInfer's ``routing_replay_out`` is routing-method-agnostic.
"""
FlashInfer's ``routing_replay_out`` is routing-method-agnostic."""
torch.manual_seed(0)
device = torch.device("cuda:0")
@@ -319,8 +317,7 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
"""With no callback installed the kernel should not see a
``routing_replay_out`` tensor — verify the helper short-circuits.
"""
``routing_replay_out`` tensor — verify the helper short-circuits."""
torch.manual_seed(0)
device = torch.device("cuda:0")
experts, _, _ = _make_bf16_monolithic_experts(
@@ -342,8 +339,7 @@ def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
"""FlashInfer's ``routing_replay_out`` is supported by all routing
methods, so ``supports_routing_replay_capture`` should be True
regardless of routing method.
"""
regardless of routing method."""
device = torch.device("cuda:0")
for routing_method in (
RoutingMethodType.DeepSeekV3,
@@ -365,8 +361,7 @@ def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None:
"""When capture is installed, the allocated buffer is int16 and shaped
``(num_tokens, experts_per_token)``.
"""
``(num_tokens, experts_per_token)``."""
device = torch.device("cuda:0")
experts, _, _ = _make_bf16_monolithic_experts(
num_experts=_DSV3_NUM_EXPERTS,
@@ -597,8 +592,7 @@ def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts(
top_k: int,
) -> None:
"""End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale
path, DSV3 routing) captures valid expert IDs.
"""
path, DSV3 routing) captures valid expert IDs."""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
@@ -784,8 +778,7 @@ def _run_nvfp4_monolithic(
routing_bias: torch.Tensor,
) -> torch.Tensor:
"""The monolithic NVFP4 apply expects packed fp4 hidden states + the
matching fp8 per-block scale stored in the ``a1q_scale`` slot.
"""
matching fp8 per-block scale stored in the ``a1q_scale`` slot."""
# Stash the weight tensors on the experts in the locations the apply()
# implementation reads from (it pulls them from quant_config / scales
# already; w1/w2 come in as args).
@@ -813,8 +806,7 @@ def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts(
top_k: int,
) -> None:
"""End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs
on the DSV3 routing path.
"""
on the DSV3 routing path."""
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
pytest.skip(
f"DSV3 requires top_k <= n_group * topk_group "
+1 -2
View File
@@ -250,8 +250,7 @@ def assert_aiter_routing_valid(
the Python baseline (different group selection, scoring internals),
so numerical comparison is not meaningful. Instead we verify the
outputs satisfy the routing contract: correct shapes, valid expert
IDs, non-negative weights, and proper normalization.
"""
IDs, non-negative weights, and proper normalization."""
n_tokens = topk_weights.shape[0]
# Shape
+2 -4
View File
@@ -76,8 +76,7 @@ def test_basic_functionality(
def test_routing_strategy_integration(monkeypatch, device):
"""Test that the routing strategy environment variable works with
FusedMoEFactory.
"""
FusedMoEFactory."""
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
import vllm.envs as envs
@@ -158,8 +157,7 @@ def test_routing_strategy_integration(monkeypatch, device):
def test_distribution_based_routing_with_custom_strategy():
"""Test registering and using DistributionBasedRouting with custom
parameters.
"""
parameters."""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Register custom distribution-based strategy
@@ -70,8 +70,7 @@ def reference_with_clamp(
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
match the C++ compute() template, then run the standard silu_and_mul +
quant reference.
"""
quant reference."""
N_2 = x.size(1) // 2
dtype = x.dtype
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
@@ -28,8 +28,7 @@ if current_platform.is_fp8_fnuz():
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
"""Matrix multiplication function that supports per-token input
quantization and per-column weight quantization
"""
quantization and per-column weight quantization"""
A = A.to(torch.float32)
B = B.to(torch.float32)
@@ -57,8 +56,7 @@ def fp8_mask(a, mask):
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
"""This function performs fused moe with per-column int8
quantization using native torch.
"""
quantization using native torch."""
B, D = a.shape
# Perform per-token quantization
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
@@ -150,8 +150,7 @@ def test_select_default_backend_by_platform(
expected_backend,
):
"""Test default backend selection per platform with all optional
accelerators (FlashInfer, AITER) disabled.
"""
accelerators (FlashInfer, AITER) disabled."""
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
@@ -445,8 +444,7 @@ def test_select_cuda_flashinfer_cutlass_backend(
mock_has_flashinfer,
):
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
and FlashInfer CUTLASS is available.
"""
and FlashInfer CUTLASS is available."""
with (
patch.object(current_platform, "is_cuda", return_value=True),
patch.object(current_platform, "is_rocm", return_value=False),
+1 -2
View File
@@ -159,8 +159,7 @@ def native_per_token_group_quant_fp8(
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
):
"""Function to perform per-token-group quantization on an input tensor
`x` using native torch.
"""
`x` using native torch."""
assert x.shape[-1] % group_size == 0, (
"the last dimension of `x` must be divisible by `group_size`"
)
@@ -304,8 +304,7 @@ def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel)
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
"""Kernel output should match a dequantized fp32 reference within
fp8 per-token / per-channel quantization noise.
"""
fp8 per-token / per-channel quantization noise."""
weight_shape = (512, 4096) # (N, K)
num_tokens = 32
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
@@ -30,8 +30,7 @@ if current_platform.is_cuda_alike() and not current_platform.has_device_capabili
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
"""Matrix multiplication function that supports per-token input
quantization and per-column weight quantization
"""
quantization and per-column weight quantization"""
A = A.to(torch.float32)
B = B.to(torch.float32)
@@ -54,8 +53,7 @@ def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, topk, topk_weight, topk_ids):
"""This function performs fused moe with per-column int8 quantization
using native torch.
"""
using native torch."""
B, D = a.shape
# Perform per-token quantization
a_q, a_s = per_token_quant_int8(a)
@@ -360,8 +360,7 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
def test_fp8_block_marlin_padded_round_trip(shape):
"""Block-quantized FP8 (e.g. Nemotron NVFP4 checkpoints' FP8 layers):
group_size=128 exercises the lcm K-alignment in marlin_padded_nk and the
weight_scale_inv group-wise scale padding.
"""
weight_scale_inv group-wise scale padding."""
size_n, size_k = shape
block = 128
dtype = torch.float16
@@ -411,8 +410,7 @@ def test_fp8_block_marlin_padded_round_trip(shape):
@pytest.mark.parametrize("shape", [(200, 288), (4640, 512)])
def test_mxfp8_marlin_padded_round_trip(shape):
"""MXFP8 exercises the e8m0 scale path, where padded 0.0 scales clamp to
2^-127 instead of zero and must still contribute nothing.
"""
2^-127 instead of zero and must still contribute nothing."""
size_n, size_k = shape
group_size = 32
# The e8m0-scale Marlin kernels are only instantiated for bf16 activations.
@@ -459,8 +457,7 @@ def test_mxfp8_marlin_padded_round_trip(shape):
@pytest.mark.parametrize("shape", [(200, 512), (4640, 512)])
def test_awq_zp_marlin_padded_round_trip(shape):
"""AWQ-style uint4 with runtime zero-points, padded the way
MarlinLinearKernel does: padded columns rely on (q=0 - zp=0) * scale=0.
"""
MarlinLinearKernel does: padded columns rely on (q=0 - zp=0) * scale=0."""
size_n, size_k = shape
group_size = 128
dtype = torch.float16
@@ -695,8 +692,7 @@ def test_check_moe_marlin_supports_layer_padding():
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_fp8_marlin_moe_padded_round_trip(shape, quant):
"""FP8 weight-only MoE: pad a tile-misaligned intermediate and check the
real prepare + fused_marlin_moe against the dequantized reference.
"""
real prepare + fused_marlin_moe against the dequantized reference."""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
@@ -781,8 +777,7 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
def test_mxfp8_marlin_moe_padded_round_trip(shape):
"""MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with
unit e8m0 scales so the reference is the exact fp8 dequant.
"""
unit e8m0 scales so the reference is the exact fp8 dequant."""
from tests.kernels.utils import torch_experts
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import fused_topk
@@ -72,8 +72,7 @@ def test_true_w4a4_kernels_reject_unset_activation(kernel_cls):
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
def test_true_w4a4_kernels_reject_explicit_non_mxfp4_activation(kernel_cls):
"""FlashInfer/XPU/Aiter quantize activations to MXFP4 internally, so an
explicit request for a different activation format must be rejected.
"""
explicit request for a different activation format must be rejected."""
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp6E3M2Dynamic)
can_implement, reason = kernel_cls.can_implement(config)
assert not can_implement
@@ -88,8 +87,7 @@ def test_weight_only_kernels_accept_unquantized_or_mxfp4_activation(
"""Marlin/Humming never quantize activations, so an unset activation key,
or one that already describes MXFP4-shaped data, is tolerated. When an
activation key is explicitly set, a warning must be logged noting that it
is ignored, since these kernels are weight-only (A16).
"""
is ignored, since these kernels are weight-only (A16)."""
config = MxFp4LinearLayerConfig(activation_quant_key=activation_quant_key)
with patch(f"{kernel_cls.__module__}.logger.warning_once") as warning_once:
can_implement, reason = kernel_cls.can_implement(config)
@@ -118,8 +116,7 @@ def test_weight_only_kernels_reject_non_mxfp4_activation(kernel_cls):
)
def test_emulation_kernel_accepts_any_config(activation_quant_key, monkeypatch):
"""EmulationMxfp4LinearKernel is the universal fallback: it must accept
every supported activation format.
"""
every supported activation format."""
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
# which we are not testing here.
monkeypatch.setattr(
@@ -137,8 +134,7 @@ def test_emulation_kernel_accepts_any_config(activation_quant_key, monkeypatch):
def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
"""quant_dequant_func must be derived purely from the config's activation
QuantKey, not set externally.
"""
QuantKey, not set externally."""
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
# which we are not testing here.
monkeypatch.setattr(
@@ -162,8 +158,7 @@ def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
def test_aiter_kernel_is_supported_requires_native_mx_support():
"""AiterMxfp4LinearKernel must not be selected on platforms without
native MX compute, even if AITER itself is importable.
"""
native MX compute, even if AITER itself is importable."""
with patch(
"vllm.model_executor.kernels.linear.mxfp4.aiter.current_platform.supports_mx",
return_value=False,
@@ -199,8 +194,7 @@ class OOTMxFp4LinearKernel(MxFp4LinearKernel):
@patch("vllm.model_executor.kernels.linear.current_platform")
def test_init_mxfp4_linear_kernel_dispatches_to_registered_kernel(platform_mock):
"""init_mxfp4_linear_kernel should select a registered kernel that
reports itself as supported, and construct it with a fresh config.
"""
reports itself as supported, and construct it with a fresh config."""
platform_mock._enum = PlatformEnum.OOT
register_linear_kernel(OOTMxFp4LinearKernel, PlatformEnum.OOT, "mxfp4")
@@ -16,8 +16,7 @@ import torch
class TestTritonMoeForwardExpertMap:
"""Test that triton_kernel_moe_forward applies expert_map remapping
when expert_map is provided (EP active).
"""
when expert_map is provided (EP active)."""
def test_expert_map_remap(self):
device = "cuda" if torch.cuda.is_available() else "cpu"
@@ -41,8 +41,7 @@ def test_can_implement_is_abstract():
def test_emulation_kernel_rejects_non_mxfp6_weights():
"""EmulationMxfp6LinearKernel must not implement a non-MXFP6 weight
format.
"""
format."""
config = MxFp6LinearLayerConfig(weight_quant_key=kMxfp4Static)
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
assert not can_implement
@@ -58,8 +57,7 @@ def test_emulation_kernel_accepts_any_supported_config(
weight_quant_key, activation_quant_key
):
"""EmulationMxfp6LinearKernel is the only backend today: it must accept
every supported weight/activation format combination.
"""
every supported weight/activation format combination."""
config = MxFp6LinearLayerConfig(
weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key
)
@@ -104,8 +102,7 @@ class OOTMxFp6LinearKernel(MxFp6LinearKernel):
def test_init_mxfp6_linear_kernel_dispatches_to_registered_kernel(platform_mock):
"""init_mxfp6_linear_kernel should select a registered kernel that
reports itself as supported/able to implement the given config, and
construct it with that exact config.
"""
construct it with that exact config."""
platform_mock._enum = PlatformEnum.OOT
register_linear_kernel(OOTMxFp6LinearKernel, PlatformEnum.OOT, "mxfp6")
@@ -118,8 +118,7 @@ def test_per_token_group_quant_fp8_packed(
num_tokens, hidden_dim, group_size, poisoned_scales
):
"""Test the packed DeepGEMM quantization kernel against the Triton
reference (row-major, UE8M0 scales).
"""
reference (row-major, UE8M0 scales)."""
device = "cuda"
torch.manual_seed(42)
@@ -257,8 +256,7 @@ def test_per_token_group_quant_fp8_packed_all_zero():
def test_per_token_group_quant_fp8_packed_mantissa_rounds_up():
"""Inputs whose absmax/max_8bit produces a non-power-of-2 force the
mantissa-rounding-up branch (exp_byte += 1). Locks down this behavior
before optimization.
"""
before optimization."""
device = "cuda"
num_tokens, hidden_dim, group_size = 4, 7168, 128
@@ -330,8 +328,7 @@ def test_per_token_group_quant_fp8_packed_zero_fills_padded_output_q(
):
"""When output_q is allocated with shape (tma_aligned_mn, k) instead of
(mn, k), the kernel must overwrite the padded mn rows with zeros so
callers can use ``torch.empty`` instead of ``torch.zeros``.
"""
callers can use ``torch.empty`` instead of ``torch.zeros``."""
device = "cuda"
group_size = 128
torch.manual_seed(42)
@@ -1,8 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the Triton dequant-gather kernel used by
``CompressedTensorsEmbeddingWNA16Int`` (quantized embedding lookup).
"""
``CompressedTensorsEmbeddingWNA16Int`` (quantized embedding lookup)."""
import pytest
import torch
@@ -29,8 +29,7 @@ pytestmark = pytest.mark.skipif(
def test_rocm_aiter_group_fp8_quant_fake_implementation():
"""Test that the fake implementation is correctly
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant.
"""
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant."""
# Create test tensors
M = 128
N = 4096
@@ -50,8 +49,7 @@ def test_rocm_aiter_group_fp8_quant_fake_implementation():
def test_rocm_aiter_group_fp8_quant_torch_compile_with_cudagraph():
"""Test that rocm_aiter_ops.group_fp8_quant
with group size 128 can be used with
torch.compile in cudagraph mode.
"""
torch.compile in cudagraph mode."""
# Create test tensors
M = 128
N = 4096
@@ -209,8 +209,7 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
@pytest.mark.parametrize("shape", LLAMA_SHAPES)
def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype):
"""The public vLLM MXFP4 QDQ helper should stay exact against the torch
reference even on larger Llama-like shapes.
"""
reference even on larger Llama-like shapes."""
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
quant_dequant_mxfp4,
)
+3 -6
View File
@@ -670,8 +670,7 @@ def test_v41_rope_insert_plain_row(compress_ratio: int, store_fp8: bool):
def test_v41_compressor_metadata_maps_tokens_to_their_ring():
"""The ring group's generic slot mapping is disabled (all PAD), so the
builder must map every real token to ``ring_block * capacity + pos %
capacity`` and keep padding tokens at PAD.
"""
capacity`` and keep padding tokens at PAD."""
from unittest.mock import MagicMock
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
@@ -1096,8 +1095,7 @@ def test_get_c128_boundary(starts, query_start_loc, expected):
@pytest.mark.parametrize("block_size", [16, 64])
def test_deepseek_v4_attention_quant_cache_roundtrip(num_tokens: int, block_size: int):
"""compressed_kv → quantize_and_insert_k_cache → dequantize_and_gather_k_cache
→ compare against original.
"""
→ compare against original."""
HEAD_DIM = 512
NOPE_DIM = 448
HEAD_BYTES = 584 # 448 fp8 + 128 bf16 + 8 uint8 scale
@@ -1309,8 +1307,7 @@ def test_dequantize_and_gather_k_cache(
@pytest.mark.parametrize("block_size", [16, 64])
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
"""K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
→ manual dequant → compare against original.
"""
→ manual dequant → compare against original."""
HEAD_DIM = 128
QUANT_BLOCK_SIZE = 128
# cache_stride = head_dim + (head_dim * 4 / quant_block_size) = 128 + 4 = 132
+1 -2
View File
@@ -432,8 +432,7 @@ def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts):
)
def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
"""Correctness test with large sequence lengths matching benchmark
scenarios (8K-128K prefill).
"""
scenarios (8K-128K prefill)."""
(
cache,
block_table,
+19 -38
View File
@@ -85,8 +85,7 @@ def _decode_token_regions(
cache: torch.Tensor, slot_mapping: torch.Tensor, block_size: int
):
"""Slice out, per valid token, the raw NoPE-fp8 / RoPE-bf16 / scale byte
regions this layout defines.
"""
regions this layout defines."""
valid = slot_mapping >= 0
idx = valid.nonzero(as_tuple=True)[0]
slots = slot_mapping[idx]
@@ -111,8 +110,7 @@ def _assert_cache_parity(cache_fused, cache_ref, slot_mapping, block_size):
"""NoPE fp8 + the UE8M0 scale byte are a deterministic quant of an
un-rotated bf16 value, so they match tightly; the RoPE region is rotated
in fp32 before its bf16 store and eager/kernel can round a tie to
opposite sides, so allow <=1 ULP there.
"""
opposite sides, so allow <=1 ULP there."""
nope_ref, rope_ref, scale_ref = _decode_token_regions(
cache_ref, slot_mapping, block_size
)
@@ -192,8 +190,7 @@ def _dup_padded_block_cache(
num_blocks: int, block_size: int, width: int, pad_elems: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Two independently-stored padded caches with identical values, for ops
that mutate the cache in place.
"""
that mutate the cache in place."""
page_stride = block_size * width + pad_elems
template = torch.randn(num_blocks, page_stride, dtype=torch.float32)
@@ -210,8 +207,7 @@ def _run_save_partial_states_eager(
):
"""Reference: state_cache[block_idx, pos_in_block, :W] = kv;
state_cache[..., W:] = score + ape[position % compress_ratio]; skip
slot < 0. Mutates state_cache in place, matching the op's contract.
"""
slot < 0. Mutates state_cache in place, matching the op's contract."""
state_width = kv.shape[-1]
block_size = state_cache.shape[1]
valid = slot_mapping >= 0
@@ -271,8 +267,7 @@ def _run_compress_eager(
):
"""Reference for the per-channel-softmax-pooling + RMSNorm + NoPE-UE8M0
quant + GPT-J RoPE math writing the main (head_dim=512) fp8_ds_mla
layout.
"""
layout."""
num_tokens, window = gather_slots.shape
state_width = state_cache.shape[-1] // 2
state_block_size = state_cache.shape[1]
@@ -449,8 +444,7 @@ def test_compress_norm_rope_store_cpu_matches_eager(
def test_compress_norm_rope_store_cpu_non_contiguous_kv_cache_row_stride():
"""The main (output) kv_cache is a separate physical buffer from the
state cache and can independently be a strided view.
"""
state cache and can independently be a strided view."""
eps = 1e-6
compress_ratio, overlap = 128, False
num_tokens = 5
@@ -526,8 +520,7 @@ def _run_compress_indexer_eager(
"""Same softmax-pool + RMSNorm + GPT-J RoPE prologue as the head=512
path, but the indexer's cache quantizes all 128 post-RoPE dims as one
FP8 block with a single raw fp32 scale per token (matching
_indexer_k_quant_and_cache_cpu's layout), not 7 separate UE8M0 blocks.
"""
_indexer_k_quant_and_cache_cpu's layout), not 7 separate UE8M0 blocks."""
num_tokens, window = gather_slots.shape
state_width = state_cache.shape[-1] // 2
state_block_size = state_cache.shape[1]
@@ -655,8 +648,7 @@ def _pack_flash_mla_cache(kv_f32: torch.Tensor, block_size: int):
"""Quantize kv_f32 ([N, HEAD_DIM]) into the fp8_ds_mla byte layout and
return (cache, dequant_ref) -- the exact post-quantization value the
kernel's dequant should reproduce. Slot id == row index, packed
contiguously from block 0.
"""
contiguously from block 0."""
n = kv_f32.shape[0]
num_blocks = max(1, -(-n // block_size))
cache = _make_cache(num_blocks, block_size)
@@ -714,8 +706,7 @@ def _flash_mla_eager(
"""MQA attention: gather the (already-dequantized) window+compressed KV
rows referenced by *_slots (-1 = invalid/masked), append a learned
per-head sink logit with zero value contribution, softmax, weighted
sum.
"""
sum."""
num_tokens, num_heads, _ = q.shape
q_f32 = q.float()
@@ -814,8 +805,7 @@ def test_flash_mla_cpu_kernel_matches_eager(
def test_flash_mla_cpu_kernel_all_invalid_falls_back_to_sink():
"""When every KV slot is masked out, softmax degenerates to the sink
logit alone and the output must be all-zero (no valid value rows).
"""
logit alone and the output must be all-zero (no valid value rows)."""
torch.manual_seed(2)
dtype = torch.bfloat16
num_tokens, num_heads, num_window = 3, 4, 5
@@ -855,8 +845,7 @@ def _qnorm_kv_insert_eager(
q, kv, positions, cache, slot_mapping, cos_sin_cache, q_head_padded, eps, bs
):
"""Reference: per-head weight-free RMSNorm + GPT-J RoPE on Q; GPT-J RoPE
+ UE8M0 FP8 quant + paged insert on KV.
"""
+ UE8M0 FP8 quant + paged insert on KV."""
num_quant_blocks = NOPE_DIM // QUANT_BLOCK
num_tokens_full, num_heads_q, _ = q.shape
@@ -982,8 +971,7 @@ def test_qnorm_rope_kv_insert_cpu_kernel_matches_eager(
@pytest.mark.parametrize("pad", [1, 5])
def test_qnorm_rope_kv_insert_cpu_kernel_dp_padding(pad: int):
"""slot_mapping shorter than q/kv rows: the KV branch must only touch
the first num_tokens rows while Q-norm+RoPE still runs on all rows.
"""
the first num_tokens rows while Q-norm+RoPE still runs on all rows."""
torch.manual_seed(1)
dtype = torch.bfloat16
eps = 1e-6
@@ -1044,8 +1032,7 @@ def _inverse_gptj_rope_o_proj_eager(
rope_dim: int,
) -> torch.Tensor:
"""NoPE dims pass through as fp32; the RoPE segment is de-rotated by
R(-pos) in GPT-J (interleaved even/odd) convention.
"""
R(-pos) in GPT-J (interleaved even/odd) convention."""
nope_dim = o.shape[-1] - rope_dim
half_dim = rope_dim // 2
@@ -1111,8 +1098,7 @@ def _write_page(
) -> None:
"""Write k_fp8 ([block_size, INDEXER_HEAD_DIM]) and scale ([block_size]
fp32) into one page row: K-region then scale-region, matching
compress_norm_rope_store_indexer_cpu's layout.
"""
compress_norm_rope_store_indexer_cpu's layout."""
k_region = page_row[: block_size * INDEXER_HEAD_DIM]
k_region.copy_(k_fp8.view(torch.uint8).flatten())
scale_region = page_row[block_size * INDEXER_HEAD_DIM :].view(torch.float32)
@@ -1148,8 +1134,7 @@ def _build_paged_mqa_logits_batch(
):
"""Fill pages (a pre-sized [n_pages, buf_width] uint8 tensor, possibly
non-contiguous in dim 0) with deterministic-per-seed random K/scale
data.
"""
data."""
torch.manual_seed(seed)
batch_size = len(seq_lens)
max_seq_len = max(seq_lens)
@@ -1215,8 +1200,7 @@ def test_fp8_paged_mqa_logits_cpu_non_contiguous_kv_cache_row_stride():
"""The paged K-cache in production is a per-layer view into a shared
multi-layer allocation, so its page stride generally exceeds the page
byte width -- check the kernel reads a padded/strided buffer identically
to an equivalent tightly-packed one.
"""
to an equivalent tightly-packed one."""
block_size = 64
num_heads = 4
seq_lens = [1, block_size + 7, 3 * block_size]
@@ -1342,8 +1326,7 @@ def test_fused_indexer_q_rope_quant_dispatches_to_cpu_kernel():
CPU op on a CPU tensor, not silently fall through to the Triton kernel
(which would also "work" under triton-cpu, masking a dispatch
regression). The dispatcher does its CPU import inside the CPU branch,
so the patch target is the source module.
"""
so the patch target is the source module."""
torch.manual_seed(0)
num_tokens = 1
@@ -1410,8 +1393,7 @@ def _hc_head_torch(
) -> torch.Tensor:
"""Eager reference for hc_head_fused_cpu: round-trips the RMSNorm
output through bf16 before the linear, matching the kernel's bf16
arithmetic.
"""
arithmetic."""
hc_mult, hidden_size = hidden_states.shape[-2:]
outer_shape = hidden_states.shape[:-2]
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
@@ -1493,8 +1475,7 @@ def test_mhc_post_cpu(num_tokens: int, hidden_size: int):
@pytest.mark.parametrize("hidden_size", [4096, 7168])
def test_hc_head_cpu(num_tokens: int, hidden_size: int):
"""hc_head_fused_cpu vs. the _hc_head_torch eager reference it
replaces.
"""
replaces."""
torch.set_default_device(DEVICE)
set_random_seed(0)
hc_mult = 4
+3 -6
View File
@@ -383,8 +383,7 @@ def _hash_ids(requests, tables_meta, block_table, cache=None, capture=False):
"""Run the op on a batch of (token_ids, start, window) requests; `window`
is the runner's [depth] lookback for that request (None = all unknown).
Token IDs 14 and 22 stand in for image tokens that break n-grams.
Returns one [len(token_ids), ...] hash tensor per request.
"""
Returns one [len(token_ids), ...] hash tensor per request."""
token_map, multipliers, primes, offsets, block_size = tables_meta
depth = multipliers.shape[1] - 1
@@ -443,8 +442,7 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
V2 supplies every lookback from its device token history and needs no
slot cache. V1 supplies prompt positions only and reads generated
positions from the slot cache it fills itself. Without any window the
first decode token hashes stale slots (negative control).
"""
first decode token hashes stale slots (negative control)."""
block_size = 4
block_table = torch.tensor(
[[3, 1, 5, 0], [2, 4, 6, 7]], dtype=torch.int32, device="cuda"
@@ -710,8 +708,7 @@ def test_engram_lookup_reuses_jit_across_token_shapes():
def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
"""The fused gather must be bit-exact with the torch dequant path it
replaces, from HBM and from pinned host memory alike, and must contribute
zeros for rows another TP rank owns.
"""
zeros for rows another TP rank owns."""
layer = _make_embedding(cpu_offload)
cols, rows = 24, layer.part_num_embeddings
ids = torch.randint(0, rows, (num_tokens, cols), dtype=torch.int32, device="cuda")
+1 -2
View File
@@ -336,8 +336,7 @@ def test_layer_norm_rows_per_block(
@torch.inference_mode()
def test_strided_input(dtype: torch.dtype) -> None:
"""Test that the kernel handles non-contiguous (strided)
inputs correctly.
"""
inputs correctly."""
set_random_seed(42)
device = torch.device(DEVICE)
num_tokens = 128
+1 -2
View File
@@ -97,8 +97,7 @@ def test_topk_routing_consistency(num_tokens: int, hidden_dim: int, num_experts:
"""The gate feeds top-k expert selection: the kernel's top-8 must match
an fp64 reference's top-8 per token (ties tolerated). This is the
business-level correctness of the router — numeric error only matters
if it flips the argsort.
"""
if it flips the argsort."""
_requires_sm90()
top_k = 8
device = torch.device("cuda")
@@ -854,8 +854,7 @@ def test_fused_q_no_indexer(num_tokens: int):
@pytest.mark.parametrize("has_indexer", [True, False])
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
produced (bf16, unquantized); ql_nope is consumed directly by the caller.
"""
produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
torch.manual_seed(6)
dev = "cuda"
max_pos = 8192
@@ -176,8 +176,7 @@ def _call_fused(
def _as_stored_fp8(t: torch.Tensor) -> torch.Tensor:
"""Reinterpret a float8_e4m3fn-typed kernel output under the real (FNUZ on
gfx942) encoding the kernel actually wrote, without touching the bytes.
"""
gfx942) encoding the kernel actually wrote, without touching the bytes."""
return t.contiguous().view(torch.uint8).view(FP8_STORE_DTYPE)
@@ -713,8 +712,7 @@ def test_q_interleaved_is_orthogonal_to_norm_and_rope(
@pytest.mark.parametrize("block_size", [16, 64])
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
"""slot_mapping.size(0) < q.size(0): the kernel must skip padded
tokens in the KV branch while still processing Q on all rows.
"""
tokens in the KV branch while still processing Q on all rows."""
torch.manual_seed(3)
device = "cuda"
dtype = torch.bfloat16
@@ -574,8 +574,7 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
result rounded once, so the scoring kernel needs no cast.
"""
result rounded once, so the scoring kernel needs no cast."""
if use_cutedsl and not has_cutedsl():
pytest.skip("cutedsl (cutlass) not installed")
@@ -446,8 +446,7 @@ def test_output_strides(num_tokens, num_heads, n_groups):
def test_per_group_contiguity(num_tokens):
"""FP8 per-group slices must be contiguous. Scale per-group slices
are column-major (T-stride=1) — not row-major contiguous, which is
correct for TMA loads.
"""
correct for TMA loads."""
num_heads, n_groups = 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
@@ -513,8 +512,7 @@ def test_scales_are_power_of_two():
def test_nope_dims_unchanged():
"""Nope dimensions (first 448 per head) should only be quantized,
not rotated. Verify by dequantizing and comparing against
quantize-only reference (no RoPE).
"""
quantize-only reference (no RoPE)."""
num_tokens, num_heads, n_groups = 16, 64, 8
heads_per_group = num_heads // n_groups
max_pos = 4096
@@ -905,8 +903,7 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
@torch.inference_mode()
def test_with_real_deepseek_v4_rope(num_tokens, default_vllm_config):
"""Test with real DeepseekV4ScalingRotaryEmbedding (GPT-J style,
mscale=0, YaRN scaling) matching the production config.
"""
mscale=0, YaRN scaling) matching the production config."""
num_heads = 64
n_groups = 8
heads_per_group = num_heads // n_groups
+2 -4
View File
@@ -1372,8 +1372,7 @@ def _patch_first_rank_pp_group(monkeypatch):
def test_deepseek_v4_mhc_broadcast_finalize_sums_hc_streams(monkeypatch):
"""First finalize (at the end of load_weights) allocates
hc_attn_fn_broadcast as hc_attn_fn summed over hc streams.
"""
hc_attn_fn_broadcast as hc_attn_fn summed over hc streams."""
_patch_first_rank_pp_group(monkeypatch)
layer = _make_mhc_decoder_layer(hc_mult=2, hidden_size=8)
model = SimpleNamespace(start_layer=0, end_layer=1, layers=[layer])
@@ -1388,8 +1387,7 @@ def test_deepseek_v4_mhc_broadcast_finalize_sums_hc_streams(monkeypatch):
def test_deepseek_v4_mhc_broadcast_refit_refreshes_in_place(monkeypatch):
"""Re-finalizing after a weight refit must copy into the existing
broadcast tensor so its address stays stable for captured CUDA graphs,
while picking up the new hc_attn_fn values.
"""
while picking up the new hc_attn_fn values."""
_patch_first_rank_pp_group(monkeypatch)
layer = _make_mhc_decoder_layer(hc_mult=2, hidden_size=8)
model = SimpleNamespace(start_layer=0, end_layer=1, layers=[layer])
+4 -8
View File
@@ -403,8 +403,7 @@ def test_mxfp8_linear_emulation_bf16_at_load(
):
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
same result; the dtype-match (BF16/FP16 activations) must also hold.
"""
same result; the dtype-match (BF16/FP16 activations) must also hold."""
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
EmulationMxfp8LinearKernel,
)
@@ -518,8 +517,7 @@ def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape):
# produced EP garbage).
def _capture_expert_mask(expert_mask, *, global_num_experts):
"""Drive the real ``AiterMxfp8Experts.apply`` mask branch and capture the
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``.
"""
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``."""
from types import SimpleNamespace
from unittest import mock
@@ -570,8 +568,7 @@ def _capture_expert_mask(expert_mask, *, global_num_experts):
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
"""AiterMxfp8Experts.apply forwards the precomputed 0/1 mask to aiter as-is
(no re-derivation that would collapse an already-0/1 mask to all-ones).
"""
(no re-derivation that would collapse an already-0/1 mask to all-ones)."""
# 0/1 local-expert mask over global ids + trailing sentinel (rank owns 0..3).
ep_mask = torch.tensor(
[1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32, device=DEVICE
@@ -585,8 +582,7 @@ def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
def test_routed_experts_expert_map_delegates_to_kernel():
"""RoutedExperts.expert_map returns the 0/1 mask only for kernels that set
``consumes_expert_mask`` (AITER), and the canonical -1 map otherwise -- keyed
on the resolved kernel, not the global aiter switch.
"""
on the resolved kernel, not the global aiter switch."""
from types import SimpleNamespace
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
+2 -4
View File
@@ -17,8 +17,7 @@ from vllm.platforms import current_platform
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
def test_shuffle_rows_basic(num_tokens: int, hidden_size: int, dtype: torch.dtype):
"""Test basic functionality of shuffle_rows with various tensor sizes and
dtypes.
"""
dtypes."""
if not current_platform.is_cuda():
pytest.skip("shuffle_rows requires CUDA")
@@ -73,8 +72,7 @@ def test_shuffle_rows_permutation(
@pytest.mark.parametrize("hidden_size", [256, 512])
def test_shuffle_rows_expansion(num_tokens: int, hidden_size: int):
"""Test shuffle_rows with expansion (more output tokens than input
tokens).
"""
tokens)."""
if not current_platform.is_cuda():
pytest.skip("shuffle_rows requires CUDA")
+2 -4
View File
@@ -1740,8 +1740,7 @@ def test_sparse_indexer_decode_topk_explicit_backends(
) -> None:
"""Every explicit sparse-indexer decode top-k backend must restrict each
row to its seq_len (dirty data past the end must never be selected) and
match torch.topk on the valid region.
"""
match torch.topk on the valid region."""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
@@ -1812,8 +1811,7 @@ def test_sparse_indexer_decode_topk_short_seq_lens(
backend: str, workspace_init
) -> None:
"""seq_len < next_n: derived per-row ends must clamp at 0 (the reference
kernels do max(0, ...)); negative ends are OOB for the ragged kernels.
"""
kernels do max(0, ...)); negative ends are OOB for the ragged kernels."""
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk