Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -247,7 +247,8 @@ def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
|
||||
|
||||
class _FakeLinear:
|
||||
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
|
||||
.quant_method are read by get_and_maybe_dequant_weights()."""
|
||||
.quant_method are read by get_and_maybe_dequant_weights().
|
||||
"""
|
||||
|
||||
def __init__(self, weight: torch.Tensor):
|
||||
self.weight = weight
|
||||
@@ -318,7 +319,8 @@ class _FakeAttnMetadata:
|
||||
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
|
||||
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
|
||||
decode attention through W_UK/W_UV."""
|
||||
decode attention through W_UK/W_UV.
|
||||
"""
|
||||
set_random_seed(1)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
@@ -384,7 +386,8 @@ def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
|
||||
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
|
||||
prefill batch with a mix of fresh and continued (cached-prefix)
|
||||
sequences."""
|
||||
sequences.
|
||||
"""
|
||||
set_random_seed(2)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
|
||||
@@ -616,7 +616,8 @@ def test_non_causal_autoselect_backend():
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle."""
|
||||
that it cannot handle.
|
||||
"""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
@@ -627,7 +628,8 @@ def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.
|
||||
"""
|
||||
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
|
||||
@@ -225,7 +225,8 @@ def test_cutlass_mla_decode_cross_layer_view():
|
||||
"""The kernel must read the cache's page-dim stride instead of assuming
|
||||
pages are packed back-to-back. A per-layer view into a cross-layer
|
||||
(block-major) cache has stride(0) inflated by num_layers; outputs must
|
||||
match a contiguous cache holding the same data exactly."""
|
||||
match a contiguous cache holding the same data exactly.
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
torch.set_default_device(device)
|
||||
|
||||
@@ -96,6 +96,7 @@ def _make_paged_kv_metadata(
|
||||
kv_indices – CUDA int32, shape [total_blocks]
|
||||
kv_last_page_lens – CPU int32, shape [num_seqs]
|
||||
block_tables – CUDA int32, shape [num_seqs, max_blocks_per_seq]
|
||||
|
||||
"""
|
||||
num_seqs = len(kv_lens)
|
||||
max_blocks = (max(kv_lens) + block_size - 1) // block_size
|
||||
@@ -161,7 +162,8 @@ def test_fast_decode_plan_importable() -> None:
|
||||
@torch.inference_mode
|
||||
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
||||
"""On the first call fast_plan_decode must route through self.plan() and
|
||||
flip vllm_first_call to False on the wrapper object."""
|
||||
flip vllm_first_call to False on the wrapper object.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||
|
||||
@@ -286,7 +286,7 @@ def test_flashinfer_sm90_fp8_mla_decode_without_rope():
|
||||
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode_workspace_supports_autotune():
|
||||
"""vLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
|
||||
"""VLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
|
||||
|
||||
Model Runner V2's warmup autotunes ``trtllm_batch_decode_mla``, which makes
|
||||
the FlashInfer autotuner enumerate the CuteDSL tactic. That tactic asserts
|
||||
|
||||
@@ -134,7 +134,8 @@ def make_quantized_kv_cache(
|
||||
head_size: int,
|
||||
) -> tuple:
|
||||
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
|
||||
kv_scale, ref_kv_cache, is_nvfp4_kv)."""
|
||||
kv_scale, ref_kv_cache, is_nvfp4_kv).
|
||||
"""
|
||||
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
|
||||
if is_nvfp4_kv:
|
||||
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
|
||||
|
||||
@@ -73,7 +73,8 @@ def _strided_context_inputs(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
||||
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache).
|
||||
"""
|
||||
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
||||
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
||||
workspace = _randn(num_tokens, 576, dtype=dtype)
|
||||
|
||||
@@ -23,7 +23,7 @@ DTYPES = [torch.float32]
|
||||
|
||||
|
||||
def reference_lightning_attention(q, k, v, ed, block_size, kv_history):
|
||||
"""Reference implementation of lightning attention core algorithm
|
||||
"""Reference implementation of lightning attention core algorithm.
|
||||
|
||||
The difference from the main implementation is that this processes
|
||||
each step sequentially, instead of using parallelized triton kernels
|
||||
@@ -72,7 +72,7 @@ def reference_lightning_attention(q, k, v, ed, block_size, kv_history):
|
||||
|
||||
|
||||
def reference_linear_decode(q, k, v, kv_caches, slope_rate, slot_idx):
|
||||
"""Reference implementation: linear attention decode function"""
|
||||
"""Reference implementation: linear attention decode function."""
|
||||
B, H, _, D = q.shape
|
||||
output = torch.zeros(B, H * D, dtype=q.dtype, device=q.device)
|
||||
|
||||
|
||||
@@ -89,7 +89,8 @@ all_case_info: list[tuple] = []
|
||||
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
|
||||
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
||||
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
|
||||
scales must not surface as NaN in the merged output."""
|
||||
scales must not surface as NaN in the merged output.
|
||||
"""
|
||||
num_tokens, num_heads, head_size = 6, 8, 128
|
||||
prefix_output = torch.zeros(
|
||||
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test:
|
||||
"""Test:
|
||||
|
||||
* Tests for MMEncoderAttention layer
|
||||
"""
|
||||
@@ -49,9 +48,7 @@ if current_platform.is_rocm():
|
||||
|
||||
@pytest.mark.parametrize("device", devices)
|
||||
def test_mha_attn_platform(default_vllm_config, device: str):
|
||||
"""
|
||||
Test the attention selector between different platform and device.
|
||||
"""
|
||||
"""Test the attention selector between different platform and device."""
|
||||
torch.set_default_dtype(torch.float16)
|
||||
|
||||
if device == "cpu":
|
||||
@@ -113,8 +110,7 @@ def ref_attention(
|
||||
value: torch.Tensor,
|
||||
scale: float,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Native implementation of scaled dot product attention without mask:
|
||||
"""Native implementation of scaled dot product attention without mask:
|
||||
- query, key, value: [batch_size, seq_len, num_heads, head_size]
|
||||
- attn_mask: [batch_size, seq_len, seq_len]
|
||||
"""
|
||||
|
||||
@@ -45,7 +45,8 @@ def kv_layout(request) -> KVCacheLayout:
|
||||
|
||||
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
|
||||
"""Per-layer physical stride order for the given layout; the 3-dim
|
||||
indexer side cache (H=1) is contiguous, so identity."""
|
||||
indexer side cache (H=1) is contiguous, so identity.
|
||||
"""
|
||||
if ndim == 3:
|
||||
return (0, 1, 2)
|
||||
stride_order = layout.layer_view_order
|
||||
@@ -65,7 +66,8 @@ def _main_spec() -> FullAttentionSpec:
|
||||
|
||||
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
|
||||
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
|
||||
derived the same way the production allocator does."""
|
||||
derived the same way the production allocator does.
|
||||
"""
|
||||
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
|
||||
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
|
||||
|
||||
@@ -75,7 +77,8 @@ def _allocate_main_kv_via_contract(
|
||||
) -> torch.Tensor:
|
||||
"""Build the main KV cache exactly as the production allocator does for the
|
||||
given layout: allocate the physical (permuted) tensor, then expose the
|
||||
inverse-permuted logical [B, H, N, C] view the backend sees."""
|
||||
inverse-permuted logical [B, H, N, C] view the backend sees.
|
||||
"""
|
||||
logical_shape = _main_kv_logical_shape(num_pages)
|
||||
stride_order = _layer_stride_order(layout, len(logical_shape))
|
||||
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
||||
@@ -1376,7 +1379,8 @@ def test_amd_decode_index_topk_end_to_end(
|
||||
@pytest.mark.parametrize("num_idx_heads", [1, 4])
|
||||
def test_decode_index_topk_fp8(num_idx_heads: int):
|
||||
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
|
||||
matches a reference computed from the dequantized FP8 values."""
|
||||
matches a reference computed from the dequantized FP8 values.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
|
||||
decode_query_len = 1
|
||||
@@ -1649,7 +1653,8 @@ def test_prefill_sparse_attention_correctness(
|
||||
def test_main_cache_layout_contract():
|
||||
"""The standardized per-layer logical shape is [B, H, N, C] with packed
|
||||
K/V content, and the legacy layout aliases resolve to the expected
|
||||
per-layer stride orders."""
|
||||
per-layer stride orders.
|
||||
"""
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = _main_kv_logical_shape(nb)
|
||||
assert logical == (nb, h, bs, 2 * d)
|
||||
@@ -1773,7 +1778,8 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
|
||||
"""The indexer side cache is standardized 4D with H=1: under both layouts
|
||||
the allocator's logical view stays contiguous and squeezes (as
|
||||
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
|
||||
[num_blocks, block_size, head_dim] cache the kernels consume."""
|
||||
[num_blocks, block_size, head_dim] cache the kernels consume.
|
||||
"""
|
||||
nb = 5
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
@@ -1793,7 +1799,8 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
|
||||
|
||||
def test_hnd_allocation_is_packed_head_major():
|
||||
"""Under HND the backend-visible logical view is the packed head-major
|
||||
physical allocation."""
|
||||
physical allocation.
|
||||
"""
|
||||
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = _main_kv_logical_shape(nb)
|
||||
stride_order = KVCacheLayout.LBHNC.layer_view_order
|
||||
@@ -1821,7 +1828,8 @@ def test_hnd_allocation_is_packed_head_major():
|
||||
def test_main_cache_is_block_first_and_unpadded():
|
||||
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
||||
is used for the main GQA cache: its spec is unpadded and the physical
|
||||
layout keeps num_blocks as the first dimension under both layouts."""
|
||||
layout keeps num_blocks as the first dimension under both layouts.
|
||||
"""
|
||||
spec = _main_spec()
|
||||
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
||||
assert spec.page_size_padded is None
|
||||
@@ -1843,7 +1851,8 @@ def _build_decode_inputs(
|
||||
):
|
||||
"""Shared decode setup: uniform query tokens per request, a non-identity
|
||||
block table, and topk indices selecting the current block plus older causal
|
||||
blocks for each query token."""
|
||||
blocks for each query token.
|
||||
"""
|
||||
active_batch = len(seq_lens_list)
|
||||
batch = active_batch + num_padded_reqs
|
||||
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
|
||||
@@ -2179,7 +2188,8 @@ def test_decode_sparse_attention_correctness(
|
||||
):
|
||||
"""Decode (split-K) parity under both layouts: this is the only coverage of
|
||||
the decode-site cache feed, and the strided HND case fails if the kernel
|
||||
ignores the cache strides."""
|
||||
ignores the cache strides.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
||||
seq_lens_list, decode_query_len, num_padded_reqs
|
||||
@@ -2229,7 +2239,8 @@ def test_decode_wrong_layout_breaks_parity():
|
||||
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
||||
reorders the K/V content, so the decode output no longer matches the
|
||||
reference computed on the correct logical view. The mislabeled tensor keeps
|
||||
the same shape as the correct view, so the kernel stays in bounds."""
|
||||
the same shape as the correct view, so the kernel stays in bounds.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
seq_lens_list = (130, 257)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
||||
@@ -2264,7 +2275,8 @@ def test_decode_wrong_layout_breaks_parity():
|
||||
def test_main_cache_byte_identical_through_production_allocator():
|
||||
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
|
||||
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
||||
same shape, stride, and storage offset as the packed-HND allocation."""
|
||||
same shape, stride, and storage offset as the packed-HND allocation.
|
||||
"""
|
||||
nb = 4
|
||||
spec = _main_spec()
|
||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||
@@ -2279,7 +2291,8 @@ def test_main_cache_byte_identical_through_production_allocator():
|
||||
def test_padded_main_cache_is_flagged():
|
||||
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
||||
contiguous-view branch (`page_size_padded is None`). A spec that sets
|
||||
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
|
||||
`page_size_padded` is explicitly flagged rather than silently wrong-strided.
|
||||
"""
|
||||
|
||||
def _require_unpadded_block_first(spec, stride_order):
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
@@ -2316,7 +2329,8 @@ def test_padded_main_cache_is_flagged():
|
||||
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
|
||||
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
||||
packed-content K/V split views. Writing through those views must persist
|
||||
into the bound storage under both layouts."""
|
||||
into the bound storage under both layouts.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_pages = 4
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
||||
|
||||
@@ -23,7 +23,8 @@ pytestmark = pytest.mark.skipif(
|
||||
def test_concat_and_cache_mla_into_unified_slot_view():
|
||||
"""concat_and_cache_mla must write correctly into a per-layer view whose
|
||||
block stride is the full unified slot (block-major), with zero bleed into
|
||||
the other layers' segments of the same slot."""
|
||||
the other layers' segments of the same slot.
|
||||
"""
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
torch.manual_seed(0)
|
||||
@@ -85,7 +86,8 @@ def test_concat_and_cache_mla_into_unified_slot_view():
|
||||
def test_flashmla_dense_decode_unified_slot_view():
|
||||
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
|
||||
on Hopper) must read a unified-slot block-major view bit-identically to a
|
||||
contiguous per-layer cache."""
|
||||
contiguous per-layer cache.
|
||||
"""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_dense_supported()
|
||||
@@ -150,7 +152,8 @@ def test_flashmla_dense_decode_unified_slot_view():
|
||||
def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
"""FlashInfer MLA dense decode must read a unified-slot block-major view
|
||||
(inflated stride(0), non-zero storage offset) bit-identically to a
|
||||
contiguous per-layer cache."""
|
||||
contiguous per-layer cache.
|
||||
"""
|
||||
try:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
except ImportError:
|
||||
@@ -221,7 +224,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
||||
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
||||
cache, with finite nonzero output."""
|
||||
cache, with finite nonzero output.
|
||||
"""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
@@ -310,7 +314,8 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
||||
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
|
||||
write correctly into a per-layer view whose block stride is the full
|
||||
unified slot, with zero bleed into the other layers' segments."""
|
||||
unified slot, with zero bleed into the other layers' segments.
|
||||
"""
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
torch.manual_seed(0)
|
||||
@@ -355,7 +360,8 @@ def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
||||
|
||||
def test_flashattn_mla_dense_decode_unified_slot_view():
|
||||
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
|
||||
block-major view bit-identically to a contiguous per-layer cache."""
|
||||
block-major view bit-identically to a contiguous per-layer cache.
|
||||
"""
|
||||
try:
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
except ImportError:
|
||||
@@ -428,7 +434,8 @@ def test_flashattn_mla_dense_decode_unified_slot_view():
|
||||
def test_flashmla_dense_fp8_decode_unified_slot_view():
|
||||
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
|
||||
must read a unified-slot block-major view bit-identically to a contiguous
|
||||
per-layer fp8 cache."""
|
||||
per-layer fp8 cache.
|
||||
"""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_dense_supported()
|
||||
@@ -499,7 +506,8 @@ def test_flashmla_dense_fp8_decode_unified_slot_view():
|
||||
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
|
||||
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
|
||||
unified-slot block-major view bit-identically to a contiguous per-layer
|
||||
cache."""
|
||||
cache.
|
||||
"""
|
||||
try:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
except ImportError:
|
||||
|
||||
@@ -283,7 +283,8 @@ def _run_direct_flash_attn_case(
|
||||
# Backend contract tests --------------------------------------------------
|
||||
def test_aiter_mha_backend_contract():
|
||||
"""The ROCm backend advertises the dtypes, shapes, and attention types it
|
||||
is designed to handle."""
|
||||
is designed to handle.
|
||||
"""
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
|
||||
@@ -311,7 +312,8 @@ def test_aiter_mha_backend_contract():
|
||||
|
||||
def test_aiter_mha_backend_validates_kv_cache_block_size():
|
||||
"""The backend should reject KV cache specs that cannot be gathered
|
||||
correctly by the ROCm kernel."""
|
||||
correctly by the ROCm kernel.
|
||||
"""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
@@ -331,7 +333,8 @@ def test_aiter_mha_backend_validates_kv_cache_block_size():
|
||||
|
||||
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
|
||||
"""The backend should trust the ROCm MI3xx probe instead of the raw torch
|
||||
capability tuple."""
|
||||
capability tuple.
|
||||
"""
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
|
||||
@@ -356,7 +359,8 @@ def test_aiter_mha_env_flags_control_enablement(
|
||||
use_aiter, use_mha, expected, monkeypatch
|
||||
):
|
||||
"""Both the global AITER flag and the MHA-specific flag must be enabled
|
||||
before the ROCm MHA path is considered active."""
|
||||
before the ROCm MHA path is considered active.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -374,7 +378,8 @@ def test_aiter_mha_env_flags_control_enablement(
|
||||
|
||||
def test_aiter_mha_platform_gate_matches_install_and_arch():
|
||||
"""The global AITER availability check should only open on ROCm MI3xx when
|
||||
the aiter package is installed."""
|
||||
the aiter package is installed.
|
||||
"""
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found_and_supported
|
||||
|
||||
assert is_aiter_found_and_supported() is (
|
||||
@@ -645,7 +650,8 @@ def test_aiter_mha_multi_batch(num_heads, head_size, dtype):
|
||||
@pytest.mark.parametrize("num_blocks", DIRECT_NUM_BLOCKS)
|
||||
def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
||||
"""The direct paged-KV path should stay stable for both normal and very
|
||||
large block tables."""
|
||||
large block tables.
|
||||
"""
|
||||
atol = 2e-2
|
||||
rtol = 2e-2
|
||||
_assert_aiter_supported()
|
||||
@@ -667,7 +673,8 @@ def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
||||
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
||||
def test_aiter_fa_sliding_window_matches_reference():
|
||||
"""The direct kernel should respect the same sliding-window causal mask as
|
||||
the naive reference implementation."""
|
||||
the naive reference implementation.
|
||||
"""
|
||||
atol = 2e-2
|
||||
rtol = 2e-2
|
||||
_assert_aiter_supported()
|
||||
|
||||
@@ -35,7 +35,7 @@ def _require_aiter():
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_fwd_op_schema() -> None:
|
||||
"""opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
|
||||
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
|
||||
|
||||
A single opcheck call covers that the op is registered/callable, that its
|
||||
fake implementation matches the real op (torch.compile tracing), and that
|
||||
|
||||
@@ -111,7 +111,8 @@ def _make_mixed_common_metadata():
|
||||
|
||||
def _patch_build_deps(monkeypatch, events=None):
|
||||
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
|
||||
on CPU."""
|
||||
on CPU.
|
||||
"""
|
||||
|
||||
def fake_generate_sparse_seqlen_triton(
|
||||
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
||||
@@ -152,7 +153,8 @@ def test_build_populates_decode_only_split_fields(monkeypatch):
|
||||
|
||||
def test_build_populates_mixed_split_fields(monkeypatch):
|
||||
"""Mixed decode+prefill batch: split is reported, prefill fields stay
|
||||
default because this impl always runs the MQA path."""
|
||||
default because this impl always runs the MQA path.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
_patch_build_deps(monkeypatch)
|
||||
|
||||
|
||||
@@ -125,7 +125,8 @@ def _quantize_to_fp8(tensor: torch.Tensor):
|
||||
@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
|
||||
def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
|
||||
"""Test FP8 KV cache path: quantize K/V to FP8, run kernel with scales,
|
||||
and compare against BF16 reference output."""
|
||||
and compare against BF16 reference output.
|
||||
"""
|
||||
assert CACHE_SIZE % PAGE_SIZE == 0
|
||||
dtype = torch.bfloat16
|
||||
seq_len = L
|
||||
@@ -248,7 +249,8 @@ def test_decode_attention_cross_layer_view(H_Q, H_KV, D_QK, D_V, is_mla, PAGE_SI
|
||||
"""The kernel must honor the cache's page-dim stride, not assume pages are
|
||||
packed back-to-back. A per-layer view into a cross-layer (block-major)
|
||||
cache has stride(0) inflated by num_layers; outputs must match a
|
||||
contiguous cache holding the same data exactly."""
|
||||
contiguous cache holding the same data exactly.
|
||||
"""
|
||||
B = 3
|
||||
seq_len = 1027
|
||||
CACHE_SIZE = 16384
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the Triton DiffKV unified-attention kernel.
|
||||
"""
|
||||
"""Unit tests for the Triton DiffKV unified-attention kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
|
||||
"""Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
|
||||
|
||||
Tests both contiguous and non-contiguous (cross-layer unified) KV cache
|
||||
layouts against a pure-PyTorch reference implementation.
|
||||
@@ -43,8 +42,7 @@ def make_random_kv_cache(
|
||||
layout=KVCacheLayout.LBHNC,
|
||||
num_layers=None,
|
||||
):
|
||||
"""
|
||||
Create a random fp8 KV cache mimicking the production allocation.
|
||||
"""Create a random fp8 KV cache mimicking the production allocation.
|
||||
|
||||
Physical layout: [L, B, H, N, 2 * hs] permuted by ``layout.stride_order``
|
||||
Returned view: (num_blocks, 2, num_kv_heads, block_size, head_size)
|
||||
@@ -382,8 +380,7 @@ def test_large_block_size():
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_cross_layer_many_layers():
|
||||
"""
|
||||
Non-contiguous with 36 layers -- matches real gpt-oss-120b.
|
||||
"""Non-contiguous with 36 layers -- matches real gpt-oss-120b.
|
||||
Cross-layer (BLHNC) strides are far from contiguous (factor of 36 in the gaps).
|
||||
"""
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
|
||||
@@ -30,12 +30,12 @@ def reference_mla_sparse_prefill(
|
||||
topk_length: torch.Tensor | None = None,
|
||||
attn_sink: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Returns:
|
||||
"""Returns:
|
||||
- o: [s_q, h_q, dv]
|
||||
- o_fp32: [s_q, h_q, dv]
|
||||
- max_logits: [s_q, h_q]
|
||||
- lse: [s_q, h_q]
|
||||
|
||||
"""
|
||||
s_q, h_q, d_qk = q.shape
|
||||
s_kv, _, _ = kv.shape
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for ApplyRotaryEmb CustomOp dispatch behavior.
|
||||
"""Tests for ApplyRotaryEmb CustomOp dispatch behavior.
|
||||
|
||||
This test ensures that RotaryEmbedding classes correctly call the appropriate
|
||||
ApplyRotaryEmb methods based on the calling context:
|
||||
@@ -179,8 +178,7 @@ def test_rotary_embedding_dispatch(
|
||||
test_case: RotaryEmbeddingTestCase,
|
||||
device: str,
|
||||
):
|
||||
"""
|
||||
Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method.
|
||||
"""Test that RotaryEmbedding classes dispatch to the correct ApplyRotaryEmb method.
|
||||
|
||||
- forward_native methods should call ApplyRotaryEmb.forward_native()
|
||||
- forward_cuda/forward methods should call ApplyRotaryEmb.forward()
|
||||
|
||||
@@ -41,7 +41,8 @@ def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
def test_fused_embed_norm_matches_reference():
|
||||
"""Main-model fusion: the residual is the exact gather and the second output
|
||||
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
|
||||
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
|
||||
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel.
|
||||
"""
|
||||
set_random_seed(13)
|
||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||
@@ -60,7 +61,8 @@ def test_fused_embed_norm_matches_reference():
|
||||
@torch.inference_mode()
|
||||
def test_fused_embed_eh_norm_matches_reference():
|
||||
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
|
||||
feeding the untouched model-local ``fused_eh_norm``."""
|
||||
feeding the untouched model-local ``fused_eh_norm``.
|
||||
"""
|
||||
set_random_seed(13)
|
||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||
|
||||
@@ -62,7 +62,8 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
||||
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
|
||||
qr_out with row stride q_size + kv_size. Downstream dispatchers that
|
||||
require packed row-major inputs then reject the tensor and silently fall
|
||||
back to a slower GEMM path on every decode step."""
|
||||
back to a slower GEMM path on every decode step.
|
||||
"""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
@@ -88,7 +89,8 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
||||
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
|
||||
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
|
||||
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
|
||||
lifts that bound to 2**31-1."""
|
||||
lifts that bound to 2**31-1.
|
||||
"""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
|
||||
matching the eager triton kernel output."""
|
||||
matching the eager triton kernel output.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -36,7 +36,6 @@ def _worker_forward_qk(
|
||||
eps,
|
||||
):
|
||||
"""Per-rank worker: compare NCCL allreduce path vs Lamport fused kernel."""
|
||||
|
||||
if not hasattr(torch.ops._C, "minimax_allreduce_rms_qk"):
|
||||
cleanup_dist_env_and_memory()
|
||||
return
|
||||
|
||||
@@ -229,7 +229,7 @@ def test_mrope_torch_compile_tracing(
|
||||
|
||||
# Create a wrapper that makes the in-place function appear functional
|
||||
def functional_forward_cuda(pos, q, k):
|
||||
"""Wrapper that converts in-place operation to functional style
|
||||
"""Wrapper that converts in-place operation to functional style.
|
||||
|
||||
CUDA Graph does not support in-place operations.
|
||||
This wrapper creates working copies of the
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for miscellaneous utilities
|
||||
"""
|
||||
"""Tests for miscellaneous utilities."""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
@@ -579,7 +579,8 @@ def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
|
||||
|
||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
|
||||
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
|
||||
contiguous inputs."""
|
||||
contiguous inputs.
|
||||
"""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for miscellaneous utilities
|
||||
"""
|
||||
"""Tests for miscellaneous utilities."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for fused MLA KV-cache write and RoPE fused kernel
|
||||
"""
|
||||
"""Tests for fused MLA KV-cache write and RoPE fused kernel."""
|
||||
|
||||
import random
|
||||
|
||||
|
||||
@@ -87,7 +87,8 @@ def _shard(full_table, si, rows_per_rank, dtype):
|
||||
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
|
||||
def test_matches_masked_gather(tp_size, vocab_size, org_vocab_size, hidden, id_dtype):
|
||||
"""Every rank's partial output is bit-exact with the eager path, including
|
||||
the added-vocab (LoRA) rows and the ids owned by another rank."""
|
||||
the added-vocab (LoRA) rows and the ids owned by another rank.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
dtype = torch.bfloat16
|
||||
rows = pad_vocab_size(vocab_size, 64) // tp_size
|
||||
|
||||
@@ -59,7 +59,8 @@ class TestAutotuneDisabledKernel:
|
||||
|
||||
def test_autotune_disabled_kernel_produces_valid_config(self):
|
||||
"""Register a kernel with no configs (disabled), run autotune,
|
||||
verify it produces a valid helion.Config."""
|
||||
verify it produces a valid helion.Config.
|
||||
"""
|
||||
with dummy_kernel_registry(configs={}) as register:
|
||||
wrapper = register(
|
||||
"autotune_test_kernel",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for Helion ConfigManager and ConfigSet.
|
||||
"""Unit tests for Helion ConfigManager and ConfigSet.
|
||||
|
||||
Tests the simplified configuration management system for Helion custom kernels.
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel
|
||||
"""Tests for the dynamic_per_token_scaled_fp8_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_dynamic_per_token_scaled_fp8_quant.py`.
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the fused_qk_norm_rope helion kernel
|
||||
"""Tests for the fused_qk_norm_rope helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`.
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for Helion kernel availability and basic functionality.
|
||||
"""Tests for Helion kernel availability and basic functionality.
|
||||
|
||||
This module demonstrates the pattern for testing optional Helion kernels.
|
||||
Tests in this directory will be skipped if Helion is not installed.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the per_token_group_fp8_quant helion kernel
|
||||
"""Tests for the per_token_group_fp8_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_per_token_group_fp8_quant.py`.
|
||||
"""
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for Helion kernel registration.
|
||||
"""Unit tests for Helion kernel registration.
|
||||
|
||||
Tests ConfiguredHelionKernel, HelionKernelWrapper, and PresetConfigSearch
|
||||
including config picker registration and custom autotuner integration.
|
||||
@@ -599,7 +598,8 @@ class TestHelionKernelWrapper:
|
||||
)
|
||||
def test_init_eagerly_initializes_hop_path(self):
|
||||
"""Test that register_kernel eagerly builds the configured kernel
|
||||
on the HOP path (no custom op registration needed)."""
|
||||
on the HOP path (no custom op registration needed).
|
||||
"""
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
configs: dict[CaseKey, helion.Config] = {
|
||||
@@ -634,7 +634,8 @@ class TestHelionKernelWrapper:
|
||||
)
|
||||
def test_init_eagerly_initializes(self):
|
||||
"""Test that register_kernel eagerly loads configs and detects GPU
|
||||
during construction so __call__ needs no further initialization."""
|
||||
during construction so __call__ needs no further initialization.
|
||||
"""
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
with (
|
||||
@@ -962,7 +963,8 @@ class TestTorchCompileHOP:
|
||||
|
||||
def test_compiled_graph_contains_helion_hop(self):
|
||||
"""Verify torch.compile on a HelionKernelWrapper emits a
|
||||
helion_kernel_wrapper_mutation HOP node in the FX graph."""
|
||||
helion_kernel_wrapper_mutation HOP node in the FX graph.
|
||||
"""
|
||||
configs: dict[CaseKey, helion.Config] = {
|
||||
CaseKey.default(): helion.Config(block_sizes=[4, 4])
|
||||
}
|
||||
@@ -1020,7 +1022,6 @@ class TestTorchCompileHOP:
|
||||
)
|
||||
def test_inductor_backend_compiles_helion_hop(self):
|
||||
"""Test torch.compile with inductor backend and Helion fusion enabled."""
|
||||
|
||||
configs: dict[CaseKey, helion.Config] = {
|
||||
CaseKey.default(): helion.Config(block_sizes=[4, 4])
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the rms_norm_dynamic_per_token_quant helion kernel
|
||||
"""Tests for the rms_norm_dynamic_per_token_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_rms_norm_dynamic_per_token_quant.py`.
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the rms_norm_per_block_quant helion kernel
|
||||
"""Tests for the rms_norm_per_block_quant helion kernel.
|
||||
|
||||
Run `pytest tests/kernels/helion/test_rms_norm_per_block_quant.py`.
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Helion Kernel test utils"""
|
||||
"""Helion Kernel test utils."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -21,7 +21,7 @@ pytestmark = pytest.mark.skipif(
|
||||
|
||||
|
||||
def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim):
|
||||
"""q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
|
||||
"""Q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
|
||||
q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim
|
||||
qkv = torch.empty(num_tokens, q_size + 2 * kv_size)
|
||||
q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Meta-tests for vLLM IR op infrastructure.
|
||||
"""Meta-tests for vLLM IR op infrastructure.
|
||||
|
||||
Ensures all registered ops have input generators defined.
|
||||
Per-op correctness tests live alongside their op definitions
|
||||
|
||||
@@ -368,7 +368,8 @@ class TestFusedAddRMSNorm:
|
||||
@pytest.mark.parametrize("provider", ["vllm_c"])
|
||||
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
"""Test that inplace implementations reuse inputs,
|
||||
for maybe_inplace overload but not for default overload."""
|
||||
for maybe_inplace overload but not for default overload.
|
||||
"""
|
||||
impl = ir.ops.fused_add_rms_norm.impls[provider]
|
||||
if not impl.supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
@@ -1055,7 +1055,8 @@ CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]
|
||||
|
||||
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
|
||||
"""High-precision conv reference: everything in float32, no bf16 rounding.
|
||||
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim]."""
|
||||
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim].
|
||||
"""
|
||||
xf = x.float()
|
||||
wf = weight.float().unsqueeze(1)
|
||||
bf = bias.float()
|
||||
@@ -1164,7 +1165,8 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
|
||||
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
|
||||
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
|
||||
conv_state view + is_vnni=True) must match the torch fallback. Validates
|
||||
the packing + layout handoff on any AVX-512BF16 CPU."""
|
||||
the packing + layout handoff on any AVX-512BF16 CPU.
|
||||
"""
|
||||
x, weight, bias = _conv_inputs(sum(seq_lens))
|
||||
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
|
||||
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
|
||||
@@ -1178,7 +1180,8 @@ def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
|
||||
@torch.inference_mode()
|
||||
def test_conv_update_cpp_matches_torch(batch):
|
||||
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
|
||||
including the in-place conv_state update (the next-step handoff)."""
|
||||
including the in-place conv_state update (the next-step handoff).
|
||||
"""
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
|
||||
@@ -36,8 +36,7 @@ def causal_conv1d_ref(
|
||||
final_states_out: torch.Tensor | None = None,
|
||||
activation: str | None = "silu",
|
||||
):
|
||||
"""
|
||||
x: (batch, dim, seqlen)
|
||||
"""x: (batch, dim, seqlen)
|
||||
weight: (dim, width)
|
||||
bias: (dim,)
|
||||
initial_states: (batch, dim, width - 1)
|
||||
@@ -72,8 +71,7 @@ def causal_conv1d_ref(
|
||||
def causal_conv1d_update_ref(
|
||||
x, conv_state, weight, bias=None, activation=None, cache_seqlens=None
|
||||
):
|
||||
"""
|
||||
x: (batch, dim) or (batch, dim, seqlen)
|
||||
"""x: (batch, dim) or (batch, dim, seqlen)
|
||||
conv_state: (batch, dim, state_len), where state_len >= width - 1
|
||||
weight: (dim, width)
|
||||
bias: (dim,)
|
||||
@@ -136,8 +134,7 @@ def causal_conv1d_opcheck_fn(
|
||||
activation: str | None = "silu",
|
||||
null_block_id: int = NULL_BLOCK_ID,
|
||||
):
|
||||
"""
|
||||
x: (batch, dim, seqlen)
|
||||
"""x: (batch, dim, seqlen)
|
||||
weight: (dim, width)
|
||||
bias: (dim,)
|
||||
seq_idx: (batch, seqlen)
|
||||
|
||||
@@ -51,8 +51,7 @@ def selective_scan_ref(
|
||||
prev_state=None,
|
||||
final_state_out=None,
|
||||
):
|
||||
"""
|
||||
u: r(B D L)
|
||||
"""u: r(B D L)
|
||||
delta: r(B D L)
|
||||
A: c(D N) or r(D N)
|
||||
B: c(D N) or r(B N L) or r(B N 2L) or r(B G N L) or (B G N L)
|
||||
@@ -135,7 +134,7 @@ def selective_scan_opcheck_fn(
|
||||
cu_chunk_seqlen=None,
|
||||
last_chunk_indices=None,
|
||||
):
|
||||
"""if return_last_state is True, returns (out, last_state)
|
||||
"""If return_last_state is True, returns (out, last_state)
|
||||
last_state has shape (batch, dim, dstate).
|
||||
"""
|
||||
if u.stride(-1) != 1:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for the JSON-based config loader added to selective_state_update.
|
||||
"""Unit tests for the JSON-based config loader added to selective_state_update.
|
||||
|
||||
Tests cover:
|
||||
- Flat MoE-style filename generation
|
||||
@@ -123,7 +122,8 @@ def test_fallback_when_no_config(monkeypatch, tmp_path):
|
||||
|
||||
def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
|
||||
"""When effective_batch = batch*nheads is not an exact key, the closest
|
||||
key should be selected."""
|
||||
key should be selected.
|
||||
"""
|
||||
_write_config(
|
||||
tmp_path,
|
||||
dstate=32,
|
||||
@@ -168,7 +168,8 @@ def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
|
||||
|
||||
def test_non_dict_json_returns_none(monkeypatch, tmp_path):
|
||||
"""A valid JSON file that is not a dict (e.g. a list) must be ignored
|
||||
and return None rather than raising AttributeError."""
|
||||
and return None rather than raising AttributeError.
|
||||
"""
|
||||
device_name = get_ssm_device_name()
|
||||
config_path = tmp_path / get_ssm_config_file_name(
|
||||
_HEADDIM, 16, _CACHE_DTYPE, device_name
|
||||
@@ -190,7 +191,8 @@ def test_non_dict_json_returns_none(monkeypatch, tmp_path):
|
||||
|
||||
def test_empty_config_falls_back_to_heuristic(monkeypatch, tmp_path):
|
||||
"""An empty JSON object {} must not crash min() — should fall back
|
||||
to the hard-coded heuristic."""
|
||||
to the hard-coded heuristic.
|
||||
"""
|
||||
_write_config(tmp_path, dstate=64, payload={})
|
||||
|
||||
monkeypatch.setenv("VLLM_TUNED_CONFIG_FOLDER", str(tmp_path))
|
||||
|
||||
@@ -41,14 +41,15 @@ def segsum(x):
|
||||
|
||||
|
||||
def ssd_minimal_discrete(X, A, B, C, block_len, initial_states=None):
|
||||
"""
|
||||
Arguments:
|
||||
"""Arguments:
|
||||
X: (batch, length, n_heads, d_head)
|
||||
A: (batch, length, n_heads)
|
||||
B: (batch, length, n_heads, d_state)
|
||||
C: (batch, length, n_heads, d_state)
|
||||
|
||||
Return:
|
||||
Y: (batch, length, n_heads, d_head)
|
||||
|
||||
"""
|
||||
assert X.dtype == A.dtype == B.dtype == C.dtype
|
||||
assert X.shape[1] % block_len == 0
|
||||
|
||||
@@ -68,7 +68,8 @@ def _run_prefill_decode_equivalence(
|
||||
the exact fp32 recurrence (and each other). All paths use the production dt
|
||||
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
|
||||
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one.
|
||||
"""
|
||||
device = current_platform.device_type
|
||||
rtol, atol = _prefill_tolerances(act_dtype)
|
||||
set_random_seed(seed)
|
||||
|
||||
@@ -95,7 +95,8 @@ def _run_standard_decode(
|
||||
) -> None:
|
||||
"""Drive ``num_steps`` decode steps and check every path against the
|
||||
trusted recurrence. ``state_dtype`` is the recurrent-state precision;
|
||||
``act_dtype`` is the activation/buffer precision."""
|
||||
``act_dtype`` is the activation/buffer precision.
|
||||
"""
|
||||
device = "cuda"
|
||||
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
|
||||
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
|
||||
@@ -539,7 +540,8 @@ def test_replayssm_standard_decode_tp_head_shard_equivalence(
|
||||
separable: one run over all heads must equal concatenating ``tp`` independent
|
||||
per-rank runs on ``nheads // tp`` heads and ``ngroups // tp`` groups. This
|
||||
guards the per-rank divisors and group->head mapping the state-shape wiring
|
||||
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite."""
|
||||
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite.
|
||||
"""
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
assert nheads % tp == 0 and ngroups % tp == 0
|
||||
|
||||
@@ -9,8 +9,7 @@ from einops import rearrange, repeat
|
||||
def selective_state_update_ref(
|
||||
state, x, dt, A, B, C, D=None, z=None, dt_bias=None, dt_softplus=False
|
||||
):
|
||||
"""
|
||||
Argument:
|
||||
"""Argument:
|
||||
state: (batch, dim, dstate) or (batch, nheads, dim, dstate)
|
||||
x: (batch, dim) or (batch, nheads, dim)
|
||||
dt: (batch, dim) or (batch, nheads, dim)
|
||||
@@ -20,8 +19,10 @@ def selective_state_update_ref(
|
||||
D: (dim,) or (nheads, dim)
|
||||
z: (batch, dim) or (batch, nheads, dim)
|
||||
dt_bias: (dim,) or (nheads, dim)
|
||||
|
||||
Return:
|
||||
out: (batch, dim) or (batch, nheads, dim)
|
||||
|
||||
"""
|
||||
has_heads = state.dim() > 3
|
||||
if state.dim() == 3:
|
||||
|
||||
@@ -148,9 +148,7 @@ class Config:
|
||||
return self.E // self.world_size
|
||||
|
||||
def make_env_data(self) -> tuple[VllmConfig, dict[Any, Any]]:
|
||||
"""
|
||||
make env data for vllm launch.
|
||||
"""
|
||||
"""Make env data for vllm launch."""
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
enforce_eager=True,
|
||||
@@ -169,7 +167,8 @@ class Config:
|
||||
|
||||
def fe_supports_quant_scheme(self) -> bool:
|
||||
"""Check if the fused experts class supports this quant config.
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.
|
||||
"""
|
||||
if self.quant_config is None or self.quant_dtype is None:
|
||||
return True
|
||||
if self.quant_dtype != torch.float8_e4m3fn:
|
||||
@@ -444,9 +443,7 @@ class RankTensors:
|
||||
def make_hidden_states(
|
||||
config: Config,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""
|
||||
Return hidden_states
|
||||
"""
|
||||
"""Return hidden_states."""
|
||||
m, k, dtype = (config.M, config.K, config.dtype)
|
||||
device = torch.accelerator.current_device_index()
|
||||
a = torch.randn((m, k), device=device, dtype=dtype) / 15.0
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
DeepEP test utilities
|
||||
"""
|
||||
"""DeepEP test utilities."""
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
|
||||
@@ -468,7 +468,8 @@ def test_deepseek_v4_flashinfer_cutlass_falls_through_to_w4a8(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Explicit flashinfer_cutlass must try the W4A8 variant when the BF16
|
||||
variant is unsupported (the BF16 variant is gated to SM90)."""
|
||||
variant is unsupported (the BF16 variant is gated to SM90).
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
@@ -34,7 +34,6 @@ def test_batched_deepgemm_vs_triton(
|
||||
E: int, T: int, K: int, N: int, topk: int, monkeypatch, workspace_init
|
||||
):
|
||||
"""Compare BatchedDeepGemmExperts to BatchedTritonExperts."""
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1")
|
||||
|
||||
device = "cuda"
|
||||
|
||||
@@ -129,7 +129,8 @@ def test_batched_mm(
|
||||
per_act_token_quant: bool,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
@@ -268,7 +269,8 @@ def test_fused_moe_batched_experts(
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
@@ -649,7 +651,8 @@ def test_batched_experts_masked_activation_end_to_end(
|
||||
def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path."""
|
||||
the device-enablement path.
|
||||
"""
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
|
||||
@@ -54,7 +54,8 @@ SEEDS = [0]
|
||||
# For test
|
||||
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
"""This function performs fused moe with block-wise quantization using
|
||||
native torch."""
|
||||
native torch.
|
||||
"""
|
||||
B, D = a.shape
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
@@ -97,7 +98,8 @@ def setup_cuda():
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
|
||||
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
|
||||
native torch reference."""
|
||||
native torch reference.
|
||||
"""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests compute_expert_num_tokens kernels
|
||||
"""
|
||||
"""Tests compute_expert_num_tokens kernels."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
|
||||
@@ -397,7 +397,8 @@ UNALIGNED_INTERMEDIATE_DIM = 176
|
||||
class _StubMoELayer(torch.nn.Module):
|
||||
"""Minimal stand-in for the real MoE layer module, exposing just what
|
||||
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
|
||||
router configuration, and optionally w13_bias/w2_bias)."""
|
||||
router configuration, and optionally w13_bias/w2_bias).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -500,8 +501,8 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
act: MoEActivation,
|
||||
):
|
||||
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
|
||||
weights before prepacking."""
|
||||
|
||||
weights before prepacking.
|
||||
"""
|
||||
set_random_seed(0)
|
||||
batch_size = 64
|
||||
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
|
||||
|
||||
@@ -62,6 +62,7 @@ def _make_int4_moe_weights(
|
||||
Returns:
|
||||
(w13_packed, w2_packed, w13_ref, w2_ref, w13_bias, w2_bias)
|
||||
where *_ref are the dequantized float reference weights
|
||||
|
||||
"""
|
||||
# Generate INT4 weights as int8 values in [-8, 7]
|
||||
w13_int4 = torch.randint(-8, 8, (E, 2 * N, K), dtype=torch.int8)
|
||||
|
||||
@@ -826,6 +826,7 @@ def _make_int4_moe_weights(E, N, K, group_size, quant_algo):
|
||||
w1_zeros, w2_zeros,
|
||||
w1_zeros_packed, w2_zeros_packed,
|
||||
w1_s, w2_s
|
||||
|
||||
"""
|
||||
w1_int4 = torch.randint(0, 16, (E, K, 2 * N), dtype=torch.int32)
|
||||
w2_int4 = torch.randint(0, 16, (E, N, K), dtype=torch.int32)
|
||||
|
||||
@@ -91,11 +91,9 @@ def break_fp4_bytes(a, dtype):
|
||||
def generate_balanced_routing(
|
||||
hidden_states: torch.Tensor, num_experts: int, top_k: int
|
||||
):
|
||||
"""
|
||||
Generate routing weights and topk indices such that every expert is active.
|
||||
"""Generate routing weights and topk indices such that every expert is active.
|
||||
Returns routing_weights, topk_idx
|
||||
"""
|
||||
|
||||
num_tokens, hidden_dim = hidden_states.shape
|
||||
# num_tokens = batch_size * seq_len
|
||||
|
||||
@@ -232,13 +230,13 @@ def grouped_gemm_ref(
|
||||
*,
|
||||
block_size: int = 16,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Computes the reference grouped GEMM (fp4 quantized per-expert loop),
|
||||
"""Computes the reference grouped GEMM (fp4 quantized per-expert loop),
|
||||
computes flashinfer grouped GEMM (for scale consistency),
|
||||
and returns ONLY the repacked reference output: out_ref.
|
||||
|
||||
Returns:
|
||||
out_ref: Tensor [num_experts, max_m, n_out]
|
||||
|
||||
"""
|
||||
device_hs = hidden_states_expanded.device
|
||||
device_w = weights.device
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test DeepEP + DeepGEMM integration
|
||||
"""Test DeepEP + DeepGEMM integration
|
||||
DeepGEMM are gemm kernels specialized for the
|
||||
fp8 block-quantized case.
|
||||
"""
|
||||
@@ -92,9 +91,7 @@ def make_block_quant_fp8_weights(
|
||||
k: int,
|
||||
block_size: list[int],
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Return weights w1q, w2q, w1_scale, w2_scale
|
||||
"""
|
||||
"""Return weights w1q, w2q, w1_scale, w2_scale."""
|
||||
(_, w1q, w1_scale, _), (_, w2q, w2_scale, _) = make_test_weights(
|
||||
e, n, k, torch.bfloat16, torch.float8_e4m3fn, block_shape=block_size
|
||||
)
|
||||
@@ -459,10 +456,7 @@ def test_ht_deepep_deepgemm_moe(
|
||||
disable_deepgemm_ue8m0,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
Tests for High-Throughput DeepEP + DeepGemm integration.
|
||||
"""
|
||||
|
||||
"""Tests for High-Throughput DeepEP + DeepGemm integration."""
|
||||
m, n, k = mnk
|
||||
set_random_seed(7)
|
||||
|
||||
@@ -533,9 +527,7 @@ def test_ll_deepep_deepgemm_moe(
|
||||
disable_deepgemm_ue8m0,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
Tests for Low-Latency DeepEP + DeepGemm integration.
|
||||
"""
|
||||
"""Tests for Low-Latency DeepEP + DeepGemm integration."""
|
||||
assert not is_deep_gemm_e8m0_used()
|
||||
|
||||
m, n, k = mnk
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test deepep dispatch-combine logic
|
||||
"""
|
||||
"""Test deepep dispatch-combine logic."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
@@ -56,9 +54,7 @@ MAX_TOKENS_PER_RANK = 64
|
||||
def make_weights(
|
||||
e, n, k, dtype
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Return weights w1, w2, w1_scale, w2_scale
|
||||
"""
|
||||
"""Return weights w1, w2, w1_scale, w2_scale."""
|
||||
if dtype in [torch.float16, torch.bfloat16]:
|
||||
w1 = torch.randn((e, 2 * n, k), device="cuda", dtype=dtype) / 10
|
||||
w2 = torch.randn((e, k, n), device="cuda", dtype=dtype) / 10
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
|
||||
"""Test DeepEP v2 (ElasticBuffer) dispatch-combine logic.
|
||||
Compares against a pure-PyTorch reference MoE implementation.
|
||||
"""
|
||||
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
"""Unit-test DeepGEMM FP8 and FP4 kernels (no DeepEP).
|
||||
Compare DeepGEMM path against the Triton fallback inside vLLM's fused_experts.
|
||||
"""
|
||||
|
||||
@@ -79,8 +78,7 @@ def make_block_quant_fp8_weights(
|
||||
k: int,
|
||||
block_size: list[int],
|
||||
):
|
||||
"""
|
||||
Generate (w1, w2) expert weights and their per-block scale tensors
|
||||
"""Generate (w1, w2) expert weights and their per-block scale tensors
|
||||
in FP8 block-quantized format.
|
||||
|
||||
w1 shape: (E, 2N, K)
|
||||
@@ -121,8 +119,7 @@ def make_block_quant_fp8_weights(
|
||||
|
||||
|
||||
def run_single_case(m, n, k, topk, num_experts, block_size):
|
||||
"""
|
||||
Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
Triton baseline within tolerance.
|
||||
"""
|
||||
tokens_bf16 = (
|
||||
@@ -248,8 +245,7 @@ def make_mxfp4_weights(
|
||||
n: int,
|
||||
k: int,
|
||||
):
|
||||
"""
|
||||
Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
"""Generate (w1, w2) expert weights in MXFP4 packed format with float32 scales,
|
||||
plus BF16 reference weights for validation.
|
||||
|
||||
w1 shape: (E, 2N, K//2) uint8 — packed FP4
|
||||
@@ -310,8 +306,7 @@ def _bf16_moe_reference(x, w1, w2, topk_weights, topk_ids):
|
||||
|
||||
|
||||
def run_single_fp4_case(m, n, k, topk, num_experts):
|
||||
"""
|
||||
Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
"""Run one (M,N,K) configuration with FP4 weights on DeepGEMM and assert
|
||||
DeepGEMM FP4 == BF16 reference within tolerance.
|
||||
"""
|
||||
tokens_bf16 = torch.randn(m, k, device="cuda", dtype=torch.bfloat16) * (k**-0.5)
|
||||
|
||||
@@ -464,7 +464,8 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
):
|
||||
"""FlashInfer >= 0.6.18 applies the SwiGLU clamp for both block-scaled
|
||||
kernels with a SwiGLU activation (DeepSeek-V4 sets swiglu_limit); the
|
||||
per-tensor kernel has no clamp, and Relu2 rejects the parameters."""
|
||||
per-tensor kernel has no clamp, and Relu2 rejects the parameters.
|
||||
"""
|
||||
|
||||
class _Experts(TrtLlmFp8ExpertsModular):
|
||||
@staticmethod
|
||||
|
||||
@@ -83,7 +83,7 @@ def _reference_activation(
|
||||
beta: float | None,
|
||||
limit: float | None,
|
||||
):
|
||||
"""vLLM's own op for this activation, so the reference is not re-derived."""
|
||||
"""VLLM's own op for this activation, so the reference is not re-derived."""
|
||||
if activation == MoEActivation.RELU2_NO_MUL:
|
||||
return lambda x: torch.square(torch.relu(x))
|
||||
if activation == MoEActivation.SWIGLUOAI:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoE fused topk kernel
|
||||
"""Tests for the MoE fused topk kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_fused_topk.py`.
|
||||
"""
|
||||
|
||||
@@ -377,7 +377,8 @@ def test_unit_shuffle():
|
||||
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
|
||||
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
"""routing_data_from_sparse_topk must produce routing structures
|
||||
identical to make_routing_data for the same topk result."""
|
||||
identical to make_routing_data for the same topk result.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MoE grouped topk kernel
|
||||
"""Tests for the MoE grouped topk kernel.
|
||||
|
||||
Run `pytest tests/kernels/moe/test_grouped_topk.py`.
|
||||
"""
|
||||
|
||||
@@ -31,6 +31,7 @@ def mxint4_quantize(
|
||||
Returns:
|
||||
- uint8 packed (2 INT4/byte): [..., k//2] - stores SIGNED INT4 [-8, 7]
|
||||
- scales in BF16: [..., k//sf_vec_size]
|
||||
|
||||
"""
|
||||
x_reshaped = x.reshape(-1, sf_vec_size)
|
||||
x_max = x_reshaped.max(dim=-1, keepdim=True)[0].to(torch.float32)
|
||||
@@ -61,6 +62,7 @@ def mxint4_quantize_moe_weights(
|
||||
Returns:
|
||||
- weights_mxint4: Quantized weights [e, n, k//2] uint8
|
||||
- scales_mxint4: Quantization scales [e, n, k//group_size] bf16
|
||||
|
||||
"""
|
||||
e = weights_bf16.shape[0]
|
||||
weight_list = []
|
||||
@@ -101,6 +103,7 @@ def marlin_quantize_moe_weights(
|
||||
Returns:
|
||||
- weights_marlin: Marlin quantized weights [e, k//8, n] int32
|
||||
- scales_marlin: Marlin quantization scales [e, k//group_size, n] bf16
|
||||
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_test import (
|
||||
marlin_quantize,
|
||||
@@ -286,7 +289,8 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
|
||||
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
|
||||
trtllm_mxint4_block_scale_moe kernel call."""
|
||||
trtllm_mxint4_block_scale_moe kernel call.
|
||||
"""
|
||||
pytest.importorskip("flashinfer")
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
@@ -352,7 +352,8 @@ def test_modular_kernel_combinations_singlegpu(
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
config = Config(
|
||||
Ms=Ms,
|
||||
K=k,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test modular OAI Triton MoE
|
||||
"""
|
||||
"""Test modular OAI Triton MoE."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -119,8 +119,7 @@ def iterative_moe(
|
||||
expert_map: torch.Tensor = None,
|
||||
renormalize: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Baseline implementation of fused moe.
|
||||
"""Baseline implementation of fused moe.
|
||||
|
||||
Args:
|
||||
hidden_states: [*, hidden_size]
|
||||
@@ -128,6 +127,7 @@ def iterative_moe(
|
||||
w2: [num_experts, hidden_size, intermediate_size]
|
||||
gating_output: [*, num_experts]
|
||||
expert_map: [num_experts]
|
||||
|
||||
"""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_size = hidden_states.shape[-1]
|
||||
@@ -1956,8 +1956,7 @@ def test_batched_fused_marlin_moe(
|
||||
)
|
||||
|
||||
def is_valid(self):
|
||||
"""
|
||||
Return True only if the input can be represented in a Batched
|
||||
"""Return True only if the input can be represented in a Batched
|
||||
format.
|
||||
"""
|
||||
return torch.all(self.expert_num_tokens_cpu <= self.max_tokens_per_batch)
|
||||
@@ -2078,9 +2077,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
"""
|
||||
Test BF16 unquantized MoE with FlashInfer TRTLLM backend.
|
||||
"""
|
||||
"""Test BF16 unquantized MoE with FlashInfer TRTLLM backend."""
|
||||
set_random_seed(7)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
|
||||
@@ -54,8 +54,7 @@ def _verify_expert_level_sorting(
|
||||
valid_length: int,
|
||||
total_tokens: int,
|
||||
):
|
||||
"""
|
||||
Verify that actual_sorted_ids follows the correct expert-level sorting.
|
||||
"""Verify that actual_sorted_ids follows the correct expert-level sorting.
|
||||
The kerne limplementation may or may not preserve original token order
|
||||
in topk_ids in the final sorted_ids however this does not impact quality.
|
||||
"""
|
||||
@@ -96,8 +95,7 @@ def torch_moe_align_block_size(
|
||||
expert_map: torch.Tensor | None = None,
|
||||
pad_sorted_ids: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Golden torch implementation of moe_align_block_size.
|
||||
"""Golden torch implementation of moe_align_block_size.
|
||||
|
||||
This function aligns the token distribution across experts to be compatible
|
||||
with block size for matrix multiplication by sorting tokens by expert and
|
||||
@@ -189,7 +187,7 @@ def torch_moe_align_block_size(
|
||||
def test_moe_align_block_size(
|
||||
m: int, topk: int, num_experts: int, block_size: int, pad_sorted_ids: bool
|
||||
):
|
||||
"""Test moe_align_block_size without expert mapping"""
|
||||
"""Test moe_align_block_size without expert mapping."""
|
||||
topk_ids = torch.zeros((m, topk), device="cuda", dtype=torch.int32)
|
||||
for i in range(m):
|
||||
experts = torch.randperm(num_experts, device="cuda")[:topk]
|
||||
@@ -254,7 +252,7 @@ def test_moe_align_block_size_with_expert_map(
|
||||
block_size: int,
|
||||
mask_inactive_experts: bool,
|
||||
):
|
||||
"""Test moe_align_block_size with expert mapping (EP scenario)"""
|
||||
"""Test moe_align_block_size with expert mapping (EP scenario)."""
|
||||
expert_map = torch.full((num_experts,), -1, device="cuda", dtype=torch.int32)
|
||||
local_experts = list(range(0, num_experts, 2))
|
||||
for i, expert_id in enumerate(local_experts):
|
||||
|
||||
@@ -21,7 +21,8 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
|
||||
class TestUnquantizedDelegation:
|
||||
"""UnquantizedMoEKernelOracle methods must delegate to the existing
|
||||
module-level functions; behaviour is bit-identical."""
|
||||
module-level functions; behaviour is bit-identical.
|
||||
"""
|
||||
|
||||
def test_make_kernel_delegates(self) -> None:
|
||||
quant_config = object()
|
||||
|
||||
@@ -240,8 +240,7 @@ def maybe_roundup_layer_hidden_size(
|
||||
act_dtype: torch.dtype,
|
||||
backend: str | None,
|
||||
) -> int:
|
||||
"""
|
||||
Given layer hidden size and MoE configurations, round up hidden_size
|
||||
"""Given layer hidden size and MoE configurations, round up hidden_size
|
||||
if necessary.
|
||||
|
||||
Args:
|
||||
@@ -253,6 +252,7 @@ def maybe_roundup_layer_hidden_size(
|
||||
Rounded up hidden_size if rounding up is required based on the configs
|
||||
and all2all backend.
|
||||
Original hidden size otherwise.
|
||||
|
||||
"""
|
||||
if backend == "deepep_high_throughput":
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
|
||||
@@ -315,7 +315,8 @@ def tp_chunk_gate_up(
|
||||
device: torch.device | int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""TP-chunk a combined [gate; up] weight, splitting each half separately
|
||||
so every rank gets a portion of both gate and up."""
|
||||
so every rank gets a portion of both gate and up.
|
||||
"""
|
||||
half = w.shape[dim] // 2
|
||||
gate = chunk_by_rank(
|
||||
w.narrow(dim, 0, half), tp_rank, tp_size, dim=dim, device=device
|
||||
@@ -945,6 +946,7 @@ def create_shared_experts_from_config(
|
||||
|
||||
Returns:
|
||||
TestMLP instance or None if config is None
|
||||
|
||||
"""
|
||||
if shared_experts_config is None:
|
||||
return None
|
||||
@@ -991,6 +993,7 @@ def setup_moe_test_data(
|
||||
|
||||
Returns:
|
||||
MoETestData containing all test data and transforms
|
||||
|
||||
"""
|
||||
# For latent MoE: latent_size = k // 2
|
||||
latent_size = k // 2
|
||||
@@ -1710,7 +1713,6 @@ def test_moe_layer_no_parallel(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Test MoE layer without parallelism (dp_size=1, tp_size=1, use_ep=False)."""
|
||||
|
||||
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the MOE permute/unpermute kernel
|
||||
"""Tests for the MOE permute/unpermute kernel.
|
||||
|
||||
Run `pytest tests/kernels/test_moe_permute_unpermute.py`.
|
||||
"""
|
||||
|
||||
@@ -67,8 +67,7 @@ def compute_ref_output(
|
||||
@pytest.mark.parametrize("num_experts", [8, 16, 32])
|
||||
@pytest.mark.parametrize("out_dtype", [torch.bfloat16])
|
||||
def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
|
||||
"""
|
||||
Test the MXFP4 grouped GEMM kernel by:
|
||||
"""Test the MXFP4 grouped GEMM kernel by:
|
||||
1. Creating random per-expert inputs and weights
|
||||
2. Quantizing both to MXFP4 using the CUDA kernel
|
||||
3. Running the CUTLASS grouped GEMM
|
||||
@@ -205,9 +204,7 @@ def test_cutlass_mxfp4_grouped_mm(num_experts, out_dtype):
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
def test_mxfp4_experts_quant_basic():
|
||||
"""
|
||||
Basic smoke test for the MXFP4 experts quantization kernel.
|
||||
"""
|
||||
"""Basic smoke test for the MXFP4 experts quantization kernel."""
|
||||
device = "cuda"
|
||||
num_experts = 4
|
||||
k = 256
|
||||
@@ -295,8 +292,7 @@ def compute_reference_e8m0_scale(block_max: float) -> int:
|
||||
@pytest.mark.parametrize("k", [256, 7168])
|
||||
@pytest.mark.parametrize("m", [16, 64])
|
||||
def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
|
||||
"""
|
||||
Test that mxfp4_experts_quant computes E8M0 block scales correctly
|
||||
"""Test that mxfp4_experts_quant computes E8M0 block scales correctly
|
||||
per OCP MX spec (not the NVFP4 formula).
|
||||
|
||||
The old buggy kernel used: floor(log2(max/6)) + 127
|
||||
@@ -416,8 +412,7 @@ def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
|
||||
reason="mxfp4_experts_quant requires CUDA SM100",
|
||||
)
|
||||
def test_mxfp4_experts_quant_no_saturation():
|
||||
"""
|
||||
Test that the E8M0 scale is large enough to avoid unexpected saturation.
|
||||
"""Test that the E8M0 scale is large enough to avoid unexpected saturation.
|
||||
|
||||
With the buggy NVFP4 formula, the scale was too small causing most values
|
||||
to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
|
||||
|
||||
@@ -130,7 +130,8 @@ def test_is_supported_config(present, ep_size, supported, reason_substr):
|
||||
|
||||
def test_explicit_moe_backend_aiter():
|
||||
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
|
||||
ValueError when the flydsl package is missing."""
|
||||
ValueError when the flydsl package is missing.
|
||||
"""
|
||||
with _gfx950(), _flydsl_installed(True):
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
@@ -161,8 +162,9 @@ def test_gfx950_picks_aiter():
|
||||
|
||||
|
||||
def test_gfx942_picks_emulation():
|
||||
"""flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
|
||||
dot_scaled backend wins instead."""
|
||||
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
|
||||
dot_scaled backend wins instead.
|
||||
"""
|
||||
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
|
||||
backend, experts_cls = select_mxfp8_moe_backend(_config())
|
||||
assert backend is Fp8MoeBackend.EMULATION
|
||||
|
||||
@@ -214,8 +214,7 @@ def reference_moe(
|
||||
activation: str = "swiglu",
|
||||
use_interleaved_layout: bool = False,
|
||||
):
|
||||
"""
|
||||
Reference MoE implementation for accuracy testing.
|
||||
"""Reference MoE implementation for accuracy testing.
|
||||
|
||||
Args:
|
||||
activation: One of "swiglu", "silu", "relu2". Controls the activation
|
||||
@@ -224,6 +223,7 @@ def reference_moe(
|
||||
(gate=x[..., ::2], up=x[..., 1::2]) as used by SWIGLUOAI.
|
||||
If False, uses chunked layout (gate, up = chunk(x, 2)) as used
|
||||
by standard swiglu/silu.
|
||||
|
||||
"""
|
||||
# renormalize routing
|
||||
experts = torch.topk(roouting_logits, k=topk, dim=-1, sorted=True)
|
||||
@@ -1441,8 +1441,7 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
intermediate_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
Test ROCm MXFP4 MoE using oracle functions.
|
||||
"""Test ROCm MXFP4 MoE using oracle functions.
|
||||
|
||||
This test validates that the oracle functions work end-to-end:
|
||||
- select_mxfp4_moe_backend() selects a valid backend
|
||||
@@ -1825,8 +1824,7 @@ def test_mxfp4_emulation_rounds_up_to_block_size(
|
||||
def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""
|
||||
select_mxfp4_moe_backend() must raise NotImplementedError, with the
|
||||
"""select_mxfp4_moe_backend() must raise NotImplementedError, with the
|
||||
collected per-backend unsupported reasons in the message, when no
|
||||
backend supports the requested deployment configuration.
|
||||
"""
|
||||
|
||||
@@ -249,7 +249,8 @@ def _make_topk_ids(
|
||||
device: str = "cuda",
|
||||
) -> torch.Tensor:
|
||||
"""Generate distinct expert IDs per token with the same top-k shape as
|
||||
production routing."""
|
||||
production routing.
|
||||
"""
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
_, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1)
|
||||
return topk_ids.to(torch.int32)
|
||||
@@ -443,7 +444,8 @@ def test_aiter_fused_moe_custom_op_registered():
|
||||
|
||||
def test_aiter_asm_moe_tkw1_custom_op_registered():
|
||||
"""The tkw1 custom op should stay registered for FP8 apply-router-weight
|
||||
paths."""
|
||||
paths.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
import vllm._aiter_ops as aiter_ops # noqa: F401
|
||||
|
||||
@@ -453,7 +455,8 @@ def test_aiter_asm_moe_tkw1_custom_op_registered():
|
||||
|
||||
def test_aiter_fused_moe_fake_tensor_support():
|
||||
"""The fused-MoE op should preserve fake-tensor compatibility for
|
||||
torch.compile-style tracing."""
|
||||
torch.compile-style tracing.
|
||||
"""
|
||||
_assert_aiter_supported()
|
||||
import vllm._aiter_ops # noqa: F401
|
||||
|
||||
@@ -516,7 +519,8 @@ def test_aiter_moe_enablement_follows_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""The fused-MoE gate should depend only on the main AITER toggle and the
|
||||
MoE-specific toggle."""
|
||||
MoE-specific toggle.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -547,7 +551,8 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Shared-expert fusion should only be enabled when the fused-MoE path is
|
||||
enabled too."""
|
||||
enabled too.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -688,7 +693,8 @@ def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
|
||||
"""The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized
|
||||
MXFP4 reference."""
|
||||
MXFP4 reference.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
@@ -737,7 +743,8 @@ def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism():
|
||||
"""The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise
|
||||
deterministic."""
|
||||
deterministic.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
@@ -830,7 +837,8 @@ def test_aiter_fused_moe_bf16_accuracy(
|
||||
intermediate_dim: int,
|
||||
):
|
||||
"""The ROCm AITER fused-MoE BF16 path should match the float32 reference
|
||||
on representative shapes."""
|
||||
on representative shapes.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -875,7 +883,8 @@ def test_aiter_fused_moe_bf16_accuracy(
|
||||
|
||||
def test_aiter_fused_moe_gelu_accuracy():
|
||||
"""The GELU activation variant should stay aligned with the float32
|
||||
reference."""
|
||||
reference.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -920,7 +929,8 @@ def test_aiter_fused_moe_gelu_accuracy():
|
||||
|
||||
def test_aiter_fused_moe_determinism():
|
||||
"""The BF16 fused-MoE kernel should stay bitwise deterministic for the
|
||||
same inputs."""
|
||||
same inputs.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -1075,7 +1085,8 @@ def test_aiter_fused_moe_mi3xx_bf16_accuracy():
|
||||
)
|
||||
def test_aiter_fused_moe_mi3xx_fp8_accuracy():
|
||||
"""The MI3xx FP8 per-tensor MoE path should stay within the measured FP8
|
||||
error budget."""
|
||||
error budget.
|
||||
"""
|
||||
from tests.kernels.moe.utils import make_test_weights
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
|
||||
@@ -185,7 +185,8 @@ def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts(
|
||||
) -> None:
|
||||
"""The capture callback should receive the int16 routed-expert IDs the
|
||||
kernel actually used, the values should be valid expert indices, and
|
||||
each token should pick ``top_k`` distinct experts."""
|
||||
each token should pick ``top_k`` distinct experts.
|
||||
"""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
@@ -264,7 +265,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
|
||||
routing_method: RoutingMethodType,
|
||||
) -> None:
|
||||
"""Routing replay works for non-DeepSeekV3 routing methods too.
|
||||
FlashInfer's ``routing_replay_out`` is routing-method-agnostic."""
|
||||
FlashInfer's ``routing_replay_out`` is routing-method-agnostic.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda:0")
|
||||
|
||||
@@ -317,7 +319,8 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
|
||||
|
||||
def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
|
||||
"""With no callback installed the kernel should not see a
|
||||
``routing_replay_out`` tensor — verify the helper short-circuits."""
|
||||
``routing_replay_out`` tensor — verify the helper short-circuits.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda:0")
|
||||
experts, _, _ = _make_bf16_monolithic_experts(
|
||||
@@ -339,7 +342,8 @@ def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
|
||||
def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
|
||||
"""FlashInfer's ``routing_replay_out`` is supported by all routing
|
||||
methods, so ``supports_routing_replay_capture`` should be True
|
||||
regardless of routing method."""
|
||||
regardless of routing method.
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
for routing_method in (
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
@@ -361,7 +365,8 @@ def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
|
||||
|
||||
def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None:
|
||||
"""When capture is installed, the allocated buffer is int16 and shaped
|
||||
``(num_tokens, experts_per_token)``."""
|
||||
``(num_tokens, experts_per_token)``.
|
||||
"""
|
||||
device = torch.device("cuda:0")
|
||||
experts, _, _ = _make_bf16_monolithic_experts(
|
||||
num_experts=_DSV3_NUM_EXPERTS,
|
||||
@@ -592,7 +597,8 @@ def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts(
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale
|
||||
path, DSV3 routing) captures valid expert IDs."""
|
||||
path, DSV3 routing) captures valid expert IDs.
|
||||
"""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
@@ -778,7 +784,8 @@ def _run_nvfp4_monolithic(
|
||||
routing_bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""The monolithic NVFP4 apply expects packed fp4 hidden states + the
|
||||
matching fp8 per-block scale stored in the ``a1q_scale`` slot."""
|
||||
matching fp8 per-block scale stored in the ``a1q_scale`` slot.
|
||||
"""
|
||||
# Stash the weight tensors on the experts in the locations the apply()
|
||||
# implementation reads from (it pulls them from quant_config / scales
|
||||
# already; w1/w2 come in as args).
|
||||
@@ -806,7 +813,8 @@ def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts(
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs
|
||||
on the DSV3 routing path."""
|
||||
on the DSV3 routing path.
|
||||
"""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
|
||||
@@ -214,8 +214,7 @@ def assert_routing_results_close(
|
||||
rtol: float = 1e-3,
|
||||
atol: float = 1e-3,
|
||||
):
|
||||
"""
|
||||
Compare routing results, sorting by expert ID first to handle non-deterministic
|
||||
"""Compare routing results, sorting by expert ID first to handle non-deterministic
|
||||
ordering from sorted=False in topk.
|
||||
"""
|
||||
# Sort both results by expert IDs for consistent comparison
|
||||
@@ -251,7 +250,8 @@ def assert_aiter_routing_valid(
|
||||
the Python baseline (different group selection, scoring internals),
|
||||
so numerical comparison is not meaningful. Instead we verify the
|
||||
outputs satisfy the routing contract: correct shapes, valid expert
|
||||
IDs, non-negative weights, and proper normalization."""
|
||||
IDs, non-negative weights, and proper normalization.
|
||||
"""
|
||||
n_tokens = topk_weights.shape[0]
|
||||
|
||||
# Shape
|
||||
@@ -294,8 +294,7 @@ def assert_aiter_routing_valid(
|
||||
def baseline_fused_topk(
|
||||
router_logits: torch.Tensor, top_k: int, renormalize: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for standard fused top-k routing.
|
||||
"""Baseline for standard fused top-k routing.
|
||||
|
||||
Algorithm:
|
||||
1. Apply softmax to router logits
|
||||
@@ -320,8 +319,7 @@ def baseline_fused_topk_bias(
|
||||
e_score_correction_bias: torch.Tensor,
|
||||
routed_scaling_factor: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for fused top-k with bias correction.
|
||||
"""Baseline for fused top-k with bias correction.
|
||||
|
||||
Algorithm:
|
||||
1. Apply softmax to router logits
|
||||
@@ -364,8 +362,7 @@ def baseline_grouped_topk(
|
||||
e_score_correction_bias: torch.Tensor | None,
|
||||
routed_scaling_factor: float,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for grouped top-k routing (e.g., DeepSeek).
|
||||
"""Baseline for grouped top-k routing (e.g., DeepSeek).
|
||||
|
||||
Algorithm:
|
||||
1. Apply scoring function (softmax or sigmoid)
|
||||
@@ -436,8 +433,7 @@ def baseline_grouped_topk(
|
||||
def baseline_custom_llama4(
|
||||
router_logits: torch.Tensor, top_k: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Baseline for Llama4 custom routing.
|
||||
"""Baseline for Llama4 custom routing.
|
||||
|
||||
Algorithm:
|
||||
1. Select top-k expert indices (without softmax)
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test script for the token-to-expert routing simulator.
|
||||
"""Test script for the token-to-expert routing simulator.
|
||||
|
||||
This script demonstrates how to use the routing simulator to test
|
||||
different routing strategies and analyze their performance, including
|
||||
@@ -77,7 +76,8 @@ def test_basic_functionality(
|
||||
|
||||
def test_routing_strategy_integration(monkeypatch, device):
|
||||
"""Test that the routing strategy environment variable works with
|
||||
FusedMoEFactory."""
|
||||
FusedMoEFactory.
|
||||
"""
|
||||
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -158,7 +158,8 @@ def test_routing_strategy_integration(monkeypatch, device):
|
||||
|
||||
def test_distribution_based_routing_with_custom_strategy():
|
||||
"""Test registering and using DistributionBasedRouting with custom
|
||||
parameters."""
|
||||
parameters.
|
||||
"""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Register custom distribution-based strategy
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for FusedMoEFactory with routed_input_transform.
|
||||
"""Tests for FusedMoEFactory with routed_input_transform.
|
||||
|
||||
Verifies that applying routed_input_transform inside FusedMoEFactory
|
||||
produces the same results as applying the transform manually outside.
|
||||
|
||||
@@ -142,9 +142,7 @@ def silu_mul_quant(
|
||||
|
||||
|
||||
def pack_scales(x: torch.Tensor, tokens_per_expert: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
pack float32 scales into a int32 tensor
|
||||
"""
|
||||
"""Pack float32 scales into a int32 tensor."""
|
||||
assert x.dtype == torch.float32
|
||||
E, T, G = x.size()
|
||||
|
||||
@@ -170,8 +168,7 @@ def ref_with_scale_fmt(
|
||||
up: torch.Tensor,
|
||||
scale_fmt: DeepGemmQuantScaleFMT,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
The precision types of the operations triggered by this function
|
||||
"""The precision types of the operations triggered by this function
|
||||
match closely with the kernel implementation so we compare more
|
||||
accurately.
|
||||
"""
|
||||
@@ -204,8 +201,7 @@ def ref_with_scale_fmt(
|
||||
|
||||
|
||||
def token_random(E, T, H2, tokens_per_expert):
|
||||
"""
|
||||
Initialize each token in a random range so we test a range of
|
||||
"""Initialize each token in a random range so we test a range of
|
||||
scale values.
|
||||
"""
|
||||
y = torch.empty((E, T, H2), dtype=torch.bfloat16, device=DEVICE)
|
||||
|
||||
@@ -19,11 +19,9 @@ GROUP_SIZE = 128
|
||||
|
||||
|
||||
def reference_quant(x: torch.Tensor, use_ue8m0: bool):
|
||||
"""
|
||||
Reference triton quant kernel from,
|
||||
"""Reference triton quant kernel from,
|
||||
vllm.model_executor.layers.quantization.utils.fp8_utils
|
||||
"""
|
||||
|
||||
x_q = torch.empty_like(x, device=x.device, dtype=FLOAT8_DTYPE)
|
||||
|
||||
# Allocate the scale tensor in column-major format.
|
||||
@@ -72,7 +70,8 @@ def reference_with_clamp(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
|
||||
match the C++ compute() template, then run the standard silu_and_mul +
|
||||
quant reference."""
|
||||
quant reference.
|
||||
"""
|
||||
N_2 = x.size(1) // 2
|
||||
dtype = x.dtype
|
||||
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
|
||||
|
||||
@@ -28,7 +28,8 @@ if current_platform.is_fp8_fnuz():
|
||||
|
||||
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
"""Matrix multiplication function that supports per-token input
|
||||
quantization and per-column weight quantization"""
|
||||
quantization and per-column weight quantization
|
||||
"""
|
||||
A = A.to(torch.float32)
|
||||
B = B.to(torch.float32)
|
||||
|
||||
@@ -56,8 +57,8 @@ def fp8_mask(a, mask):
|
||||
|
||||
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
|
||||
"""This function performs fused moe with per-column int8
|
||||
quantization using native torch."""
|
||||
|
||||
quantization using native torch.
|
||||
"""
|
||||
B, D = a.shape
|
||||
# Perform per-token quantization
|
||||
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for the FlashInfer TRTLLM BF16 MoE backend
|
||||
"""Tests for the FlashInfer TRTLLM BF16 MoE backend
|
||||
(`TrtLlmBf16ExpertsModular`).
|
||||
|
||||
This mirrors the TRTLLM NvFP4 modular test shape: construct the modular
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for the FlashInfer TRTLLM NvFP4 MoE backend
|
||||
"""Tests for the FlashInfer TRTLLM NvFP4 MoE backend
|
||||
(`TrtLlmNvFp4ExpertsModular`).
|
||||
|
||||
Covers the activations the wrapper claims to support — SiLU, RELU^2 (non-gated),
|
||||
|
||||
@@ -150,7 +150,8 @@ def test_select_default_backend_by_platform(
|
||||
expected_backend,
|
||||
):
|
||||
"""Test default backend selection per platform with all optional
|
||||
accelerators (FlashInfer, AITER) disabled."""
|
||||
accelerators (FlashInfer, AITER) disabled.
|
||||
"""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
@@ -444,7 +445,8 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
||||
mock_has_flashinfer,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||
and FlashInfer CUTLASS is available."""
|
||||
and FlashInfer CUTLASS is available.
|
||||
"""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
|
||||
@@ -83,8 +83,7 @@ def test_zero_expert_moe_router_is_zero_expert_router(zero_expert_moe, num_token
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32])
|
||||
def test_zero_expert_moe_forward(zero_expert_moe, num_tokens):
|
||||
"""
|
||||
Run a forward pass through FusedMoEFactory with zero experts
|
||||
"""Run a forward pass through FusedMoEFactory with zero experts
|
||||
and verify output shape.
|
||||
"""
|
||||
layer, vllm_config = zero_expert_moe
|
||||
|
||||
@@ -64,8 +64,7 @@ def make_dummy_moe_config(
|
||||
max_num_tokens: int = 512,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
) -> FusedMoEConfig:
|
||||
"""
|
||||
This is a dummy config for the mk constructor interface
|
||||
"""This is a dummy config for the mk constructor interface
|
||||
as most kernels like DeepGEMM, CUTLASSFp4, Triton, MARLIN
|
||||
do not actually use this config.
|
||||
|
||||
|
||||
@@ -159,7 +159,8 @@ def native_per_token_group_quant_fp8(
|
||||
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
|
||||
):
|
||||
"""Function to perform per-token-group quantization on an input tensor
|
||||
`x` using native torch."""
|
||||
`x` using native torch.
|
||||
"""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` must be divisible by `group_size`"
|
||||
)
|
||||
|
||||
@@ -118,6 +118,7 @@ def dequant_nvfp4_kv_cache(
|
||||
|
||||
Returns:
|
||||
[..., num_heads, block_size, head_size] float32.
|
||||
|
||||
"""
|
||||
data_dim = head_size // 2
|
||||
scale_dim = head_size // 16
|
||||
|
||||
@@ -304,7 +304,8 @@ def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel)
|
||||
|
||||
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
|
||||
"""Kernel output should match a dequantized fp32 reference within
|
||||
fp8 per-token / per-channel quantization noise."""
|
||||
fp8 per-token / per-channel quantization noise.
|
||||
"""
|
||||
weight_shape = (512, 4096) # (N, K)
|
||||
num_tokens = 32
|
||||
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
|
||||
|
||||
@@ -45,6 +45,7 @@ def quantize_weight_block_fp8(
|
||||
Returns:
|
||||
fp8_weight: [N, K] float8_e4m3fn
|
||||
scales: [n_tiles, k_tiles] float32
|
||||
|
||||
"""
|
||||
N, K = weight.shape
|
||||
block_n, block_k = block_size
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for cutlass kernels
|
||||
"""Tests for cutlass kernels.
|
||||
|
||||
Run `pytest tests/kernels/quantization/test_cutlass_scaled_mm.py`.
|
||||
"""
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user