[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -247,8 +247,7 @@ def test_extend_attention_cpu_matches_reference(prefix_lens, extend_lens):
|
||||
|
||||
class _FakeLinear:
|
||||
"""Minimal duck-typed stand-in for kv_b_proj: only .weight and
|
||||
.quant_method are read by get_and_maybe_dequant_weights().
|
||||
"""
|
||||
.quant_method are read by get_and_maybe_dequant_weights()."""
|
||||
|
||||
def __init__(self, weight: torch.Tensor):
|
||||
self.weight = weight
|
||||
@@ -319,8 +318,7 @@ class _FakeAttnMetadata:
|
||||
def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||
"""forward_mqa receives already-absorbed Q (as MLAAttention.forward_impl
|
||||
would produce via layer.W_UK_T) and must reproduce plain per-head MLA
|
||||
decode attention through W_UK/W_UV.
|
||||
"""
|
||||
decode attention through W_UK/W_UV."""
|
||||
set_random_seed(1)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
@@ -386,8 +384,7 @@ def test_amx_mla_impl_forward_mqa_matches_reference(default_vllm_config):
|
||||
def test_amx_mla_impl_forward_mha_matches_reference(default_vllm_config):
|
||||
"""forward_mha receives raw unabsorbed Q and must attend correctly for a
|
||||
prefill batch with a mix of fresh and continued (cached-prefix)
|
||||
sequences.
|
||||
"""
|
||||
sequences."""
|
||||
set_random_seed(2)
|
||||
num_heads, qk_nope_head_dim, v_head_dim, kv_lora_rank = 4, 32, 32, 64
|
||||
impl, kv_b_weight = _make_amx_mla_impl(
|
||||
|
||||
@@ -616,8 +616,7 @@ def test_non_causal_autoselect_backend():
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle.
|
||||
"""
|
||||
that it cannot handle."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
assert not FlashAttentionBackend.supports_kv_cache_dtype(kv_cache_dtype)
|
||||
@@ -628,8 +627,7 @@ def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'.
|
||||
"""
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
import vllm.v1.attention.backends.fa_utils as fa_utils_mod
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
|
||||
|
||||
@@ -225,8 +225,7 @@ def test_cutlass_mla_decode_cross_layer_view():
|
||||
"""The kernel must read the cache's page-dim stride instead of assuming
|
||||
pages are packed back-to-back. A per-layer view into a cross-layer
|
||||
(block-major) cache has stride(0) inflated by num_layers; outputs must
|
||||
match a contiguous cache holding the same data exactly.
|
||||
"""
|
||||
match a contiguous cache holding the same data exactly."""
|
||||
device = torch.device("cuda:0")
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
torch.set_default_device(device)
|
||||
|
||||
@@ -162,8 +162,7 @@ def test_fast_decode_plan_importable() -> None:
|
||||
@torch.inference_mode
|
||||
def test_fast_plan_decode_warmup_uses_full_plan(dtype: torch.dtype) -> None:
|
||||
"""On the first call fast_plan_decode must route through self.plan() and
|
||||
flip vllm_first_call to False on the wrapper object.
|
||||
"""
|
||||
flip vllm_first_call to False on the wrapper object."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import fast_plan_decode
|
||||
|
||||
@@ -134,8 +134,7 @@ def make_quantized_kv_cache(
|
||||
head_size: int,
|
||||
) -> tuple:
|
||||
"""Quantize kv_cache based on dtype. Returns (kv_cache, kv_cache_sf,
|
||||
kv_scale, ref_kv_cache, is_nvfp4_kv).
|
||||
"""
|
||||
kv_scale, ref_kv_cache, is_nvfp4_kv)."""
|
||||
is_nvfp4_kv = kv_quant_dtype == FP4_DTYPE
|
||||
if is_nvfp4_kv:
|
||||
data, scales, kv_scale, ref = make_nvfp4_kv_cache(
|
||||
|
||||
@@ -73,8 +73,7 @@ def _strided_context_inputs(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""The real prefill-context layouts: ``k_nope``/``v`` as the two strided
|
||||
halves of one kv_b_proj output and ``k_pe`` as a column slice of the gather
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache).
|
||||
"""
|
||||
workspace (left in the fp8 cache layout for a plain fp8 cache)."""
|
||||
kv_nope = _randn(num_tokens, num_heads, 256, dtype=dtype)
|
||||
k_nope, v = kv_nope.split((128, 128), dim=-1)
|
||||
workspace = _randn(num_tokens, 576, dtype=dtype)
|
||||
|
||||
@@ -89,8 +89,7 @@ all_case_info: list[tuple] = []
|
||||
@pytest.mark.parametrize("output_dtype", [torch.float32, torch.half, torch.bfloat16])
|
||||
def test_merge_attn_states_both_empty(merge_fn, output_dtype) -> None:
|
||||
"""When a token is empty on both sides (both LSE -inf), the 0/0 softmax
|
||||
scales must not surface as NaN in the merged output.
|
||||
"""
|
||||
scales must not surface as NaN in the merged output."""
|
||||
num_tokens, num_heads, head_size = 6, 8, 128
|
||||
prefix_output = torch.zeros(
|
||||
num_tokens, num_heads, head_size, device=DEVICE, dtype=output_dtype
|
||||
|
||||
@@ -45,8 +45,7 @@ def kv_layout(request) -> KVCacheLayout:
|
||||
|
||||
def _layer_stride_order(layout: KVCacheLayout, ndim: int) -> tuple[int, ...]:
|
||||
"""Per-layer physical stride order for the given layout; the 3-dim
|
||||
indexer side cache (H=1) is contiguous, so identity.
|
||||
"""
|
||||
indexer side cache (H=1) is contiguous, so identity."""
|
||||
if ndim == 3:
|
||||
return (0, 1, 2)
|
||||
stride_order = layout.layer_view_order
|
||||
@@ -66,8 +65,7 @@ def _main_spec() -> FullAttentionSpec:
|
||||
|
||||
def _main_kv_logical_shape(num_pages: int) -> tuple[int, ...]:
|
||||
"""Standardized per-layer logical shape [B, H, N, C] for the main cache,
|
||||
derived the same way the production allocator does.
|
||||
"""
|
||||
derived the same way the production allocator does."""
|
||||
shape_bytes = compute_layer_kv_cache_shape_bytes(_main_spec(), num_pages)
|
||||
return (*shape_bytes[:-1], shape_bytes[-1] // DTYPE.itemsize)
|
||||
|
||||
@@ -77,8 +75,7 @@ def _allocate_main_kv_via_contract(
|
||||
) -> torch.Tensor:
|
||||
"""Build the main KV cache exactly as the production allocator does for the
|
||||
given layout: allocate the physical (permuted) tensor, then expose the
|
||||
inverse-permuted logical [B, H, N, C] view the backend sees.
|
||||
"""
|
||||
inverse-permuted logical [B, H, N, C] view the backend sees."""
|
||||
logical_shape = _main_kv_logical_shape(num_pages)
|
||||
stride_order = _layer_stride_order(layout, len(logical_shape))
|
||||
physical_shape = tuple(logical_shape[i] for i in stride_order)
|
||||
@@ -1379,8 +1376,7 @@ def test_amd_decode_index_topk_end_to_end(
|
||||
@pytest.mark.parametrize("num_idx_heads", [1, 4])
|
||||
def test_decode_index_topk_fp8(num_idx_heads: int):
|
||||
"""The standalone Triton path must score FP8 inputs in FP32 so its top-k
|
||||
matches a reference computed from the dequantized FP8 values.
|
||||
"""
|
||||
matches a reference computed from the dequantized FP8 values."""
|
||||
torch.manual_seed(0)
|
||||
topk, init_blocks, local_blocks, head_dim = 8, 0, 1, 128
|
||||
decode_query_len = 1
|
||||
@@ -1653,8 +1649,7 @@ def test_prefill_sparse_attention_correctness(
|
||||
def test_main_cache_layout_contract():
|
||||
"""The standardized per-layer logical shape is [B, H, N, C] with packed
|
||||
K/V content, and the legacy layout aliases resolve to the expected
|
||||
per-layer stride orders.
|
||||
"""
|
||||
per-layer stride orders."""
|
||||
nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = _main_kv_logical_shape(nb)
|
||||
assert logical == (nb, h, bs, 2 * d)
|
||||
@@ -1778,8 +1773,7 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
|
||||
"""The indexer side cache is standardized 4D with H=1: under both layouts
|
||||
the allocator's logical view stays contiguous and squeezes (as
|
||||
`MiniMaxM3IndexerCache.bind_kv_cache` does) to the 3-dim
|
||||
[num_blocks, block_size, head_dim] cache the kernels consume.
|
||||
"""
|
||||
[num_blocks, block_size, head_dim] cache the kernels consume."""
|
||||
nb = 5
|
||||
ispec = MLAAttentionSpec(
|
||||
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
|
||||
@@ -1799,8 +1793,7 @@ def test_indexer_cache_squeezes_to_contiguous_3d():
|
||||
|
||||
def test_hnd_allocation_is_packed_head_major():
|
||||
"""Under HND the backend-visible logical view is the packed head-major
|
||||
physical allocation.
|
||||
"""
|
||||
physical allocation."""
|
||||
nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
|
||||
logical = _main_kv_logical_shape(nb)
|
||||
stride_order = KVCacheLayout.LBHNC.layer_view_order
|
||||
@@ -1828,8 +1821,7 @@ def test_hnd_allocation_is_packed_head_major():
|
||||
def test_main_cache_is_block_first_and_unpadded():
|
||||
"""The allocator's contiguous-view branch (not the padded-strided branch)
|
||||
is used for the main GQA cache: its spec is unpadded and the physical
|
||||
layout keeps num_blocks as the first dimension under both layouts.
|
||||
"""
|
||||
layout keeps num_blocks as the first dimension under both layouts."""
|
||||
spec = _main_spec()
|
||||
# Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
|
||||
assert spec.page_size_padded is None
|
||||
@@ -1851,8 +1843,7 @@ def _build_decode_inputs(
|
||||
):
|
||||
"""Shared decode setup: uniform query tokens per request, a non-identity
|
||||
block table, and topk indices selecting the current block plus older causal
|
||||
blocks for each query token.
|
||||
"""
|
||||
blocks for each query token."""
|
||||
active_batch = len(seq_lens_list)
|
||||
batch = active_batch + num_padded_reqs
|
||||
pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
|
||||
@@ -2188,8 +2179,7 @@ def test_decode_sparse_attention_correctness(
|
||||
):
|
||||
"""Decode (split-K) parity under both layouts: this is the only coverage of
|
||||
the decode-site cache feed, and the strided HND case fails if the kernel
|
||||
ignores the cache strides.
|
||||
"""
|
||||
ignores the cache strides."""
|
||||
torch.manual_seed(0)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
|
||||
seq_lens_list, decode_query_len, num_padded_reqs
|
||||
@@ -2239,8 +2229,7 @@ def test_decode_wrong_layout_breaks_parity():
|
||||
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
||||
reorders the K/V content, so the decode output no longer matches the
|
||||
reference computed on the correct logical view. The mislabeled tensor keeps
|
||||
the same shape as the correct view, so the kernel stays in bounds.
|
||||
"""
|
||||
the same shape as the correct view, so the kernel stays in bounds."""
|
||||
torch.manual_seed(0)
|
||||
seq_lens_list = (130, 257)
|
||||
q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
|
||||
@@ -2275,8 +2264,7 @@ def test_decode_wrong_layout_breaks_parity():
|
||||
def test_main_cache_byte_identical_through_production_allocator():
|
||||
"""AC-2: drive the real allocator (`create_kv_cache_views`) for the M3 main
|
||||
`FullAttentionSpec` under HND and assert the backend-visible view has the
|
||||
same shape, stride, and storage offset as the packed-HND allocation.
|
||||
"""
|
||||
same shape, stride, and storage offset as the packed-HND allocation."""
|
||||
nb = 4
|
||||
spec = _main_spec()
|
||||
raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
|
||||
@@ -2291,8 +2279,7 @@ def test_main_cache_byte_identical_through_production_allocator():
|
||||
def test_padded_main_cache_is_flagged():
|
||||
"""AC-2.1 negative: the M3 main cache relies on the allocator's
|
||||
contiguous-view branch (`page_size_padded is None`). A spec that sets
|
||||
`page_size_padded` is explicitly flagged rather than silently wrong-strided.
|
||||
"""
|
||||
`page_size_padded` is explicitly flagged rather than silently wrong-strided."""
|
||||
|
||||
def _require_unpadded_block_first(spec, stride_order):
|
||||
inv_order = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
@@ -2329,8 +2316,7 @@ def test_padded_main_cache_is_flagged():
|
||||
def test_reshape_and_cache_flash_write_persists(kv_layout: KVCacheLayout):
|
||||
"""AC-5 write path: the `reshape_and_cache_flash` write site now consumes
|
||||
packed-content K/V split views. Writing through those views must persist
|
||||
into the bound storage under both layouts.
|
||||
"""
|
||||
into the bound storage under both layouts."""
|
||||
torch.manual_seed(0)
|
||||
num_pages = 4
|
||||
kv_cache = _allocate_main_kv_via_contract(num_pages, kv_layout)
|
||||
|
||||
@@ -23,8 +23,7 @@ pytestmark = pytest.mark.skipif(
|
||||
def test_concat_and_cache_mla_into_unified_slot_view():
|
||||
"""concat_and_cache_mla must write correctly into a per-layer view whose
|
||||
block stride is the full unified slot (block-major), with zero bleed into
|
||||
the other layers' segments of the same slot.
|
||||
"""
|
||||
the other layers' segments of the same slot."""
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
torch.manual_seed(0)
|
||||
@@ -86,8 +85,7 @@ def test_concat_and_cache_mla_into_unified_slot_view():
|
||||
def test_flashmla_dense_decode_unified_slot_view():
|
||||
"""FlashMLA dense decode (FLASHMLA backend, e.g. Kimi-K2-style dense MLA
|
||||
on Hopper) must read a unified-slot block-major view bit-identically to a
|
||||
contiguous per-layer cache.
|
||||
"""
|
||||
contiguous per-layer cache."""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_dense_supported()
|
||||
@@ -152,8 +150,7 @@ def test_flashmla_dense_decode_unified_slot_view():
|
||||
def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
"""FlashInfer MLA dense decode must read a unified-slot block-major view
|
||||
(inflated stride(0), non-zero storage offset) bit-identically to a
|
||||
contiguous per-layer cache.
|
||||
"""
|
||||
contiguous per-layer cache."""
|
||||
try:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
except ImportError:
|
||||
@@ -224,8 +221,7 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
|
||||
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
|
||||
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
|
||||
cache, with finite nonzero output.
|
||||
"""
|
||||
cache, with finite nonzero output."""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_sparse_supported()
|
||||
@@ -314,8 +310,7 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
|
||||
def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
||||
"""indexer_k_quant_and_cache (DeepSeek V3.2/V4 DSA indexer K write) must
|
||||
write correctly into a per-layer view whose block stride is the full
|
||||
unified slot, with zero bleed into the other layers' segments.
|
||||
"""
|
||||
unified slot, with zero bleed into the other layers' segments."""
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
torch.manual_seed(0)
|
||||
@@ -360,8 +355,7 @@ def test_indexer_k_quant_and_cache_into_unified_slot_view():
|
||||
|
||||
def test_flashattn_mla_dense_decode_unified_slot_view():
|
||||
"""FA3 decode (FLASH_ATTN_MLA backend) must read a unified-slot
|
||||
block-major view bit-identically to a contiguous per-layer cache.
|
||||
"""
|
||||
block-major view bit-identically to a contiguous per-layer cache."""
|
||||
try:
|
||||
from vllm.vllm_flash_attn import flash_attn_varlen_func
|
||||
except ImportError:
|
||||
@@ -434,8 +428,7 @@ def test_flashattn_mla_dense_decode_unified_slot_view():
|
||||
def test_flashmla_dense_fp8_decode_unified_slot_view():
|
||||
"""FlashMLA dense fp8 decode (FLASHMLA backend with quantized KV cache)
|
||||
must read a unified-slot block-major view bit-identically to a contiguous
|
||||
per-layer fp8 cache.
|
||||
"""
|
||||
per-layer fp8 cache."""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
|
||||
ok, reason = fm.is_flashmla_dense_supported()
|
||||
@@ -506,8 +499,7 @@ def test_flashmla_dense_fp8_decode_unified_slot_view():
|
||||
def test_flashinfer_mla_dense_fp8_decode_unified_slot_view():
|
||||
"""FlashInfer MLA dense decode with an fp8 KV cache must read a
|
||||
unified-slot block-major view bit-identically to a contiguous per-layer
|
||||
cache.
|
||||
"""
|
||||
cache."""
|
||||
try:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
except ImportError:
|
||||
|
||||
@@ -283,8 +283,7 @@ def _run_direct_flash_attn_case(
|
||||
# Backend contract tests --------------------------------------------------
|
||||
def test_aiter_mha_backend_contract():
|
||||
"""The ROCm backend advertises the dtypes, shapes, and attention types it
|
||||
is designed to handle.
|
||||
"""
|
||||
is designed to handle."""
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
|
||||
@@ -312,8 +311,7 @@ def test_aiter_mha_backend_contract():
|
||||
|
||||
def test_aiter_mha_backend_validates_kv_cache_block_size():
|
||||
"""The backend should reject KV cache specs that cannot be gathered
|
||||
correctly by the ROCm kernel.
|
||||
"""
|
||||
correctly by the ROCm kernel."""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
@@ -333,8 +331,7 @@ def test_aiter_mha_backend_validates_kv_cache_block_size():
|
||||
|
||||
def test_aiter_mha_backend_supports_compute_capability_matches_mi3xx_probe():
|
||||
"""The backend should trust the ROCm MI3xx probe instead of the raw torch
|
||||
capability tuple.
|
||||
"""
|
||||
capability tuple."""
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionBackend
|
||||
|
||||
@@ -359,8 +356,7 @@ def test_aiter_mha_env_flags_control_enablement(
|
||||
use_aiter, use_mha, expected, monkeypatch
|
||||
):
|
||||
"""Both the global AITER flag and the MHA-specific flag must be enabled
|
||||
before the ROCm MHA path is considered active.
|
||||
"""
|
||||
before the ROCm MHA path is considered active."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -378,8 +374,7 @@ def test_aiter_mha_env_flags_control_enablement(
|
||||
|
||||
def test_aiter_mha_platform_gate_matches_install_and_arch():
|
||||
"""The global AITER availability check should only open on ROCm MI3xx when
|
||||
the aiter package is installed.
|
||||
"""
|
||||
the aiter package is installed."""
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, is_aiter_found_and_supported
|
||||
|
||||
assert is_aiter_found_and_supported() is (
|
||||
@@ -650,8 +645,7 @@ def test_aiter_mha_multi_batch(num_heads, head_size, dtype):
|
||||
@pytest.mark.parametrize("num_blocks", DIRECT_NUM_BLOCKS)
|
||||
def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
||||
"""The direct paged-KV path should stay stable for both normal and very
|
||||
large block tables.
|
||||
"""
|
||||
large block tables."""
|
||||
atol = 2e-2
|
||||
rtol = 2e-2
|
||||
_assert_aiter_supported()
|
||||
@@ -673,8 +667,7 @@ def test_aiter_fa_large_block_table_matches_reference(num_blocks):
|
||||
@pytest.mark.skipif(not on_mi3xx(), reason="MI300/MI350 ROCm only")
|
||||
def test_aiter_fa_sliding_window_matches_reference():
|
||||
"""The direct kernel should respect the same sliding-window causal mask as
|
||||
the naive reference implementation.
|
||||
"""
|
||||
the naive reference implementation."""
|
||||
atol = 2e-2
|
||||
rtol = 2e-2
|
||||
_assert_aiter_supported()
|
||||
|
||||
@@ -111,8 +111,7 @@ def _make_mixed_common_metadata():
|
||||
|
||||
def _patch_build_deps(monkeypatch, events=None):
|
||||
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
|
||||
on CPU.
|
||||
"""
|
||||
on CPU."""
|
||||
|
||||
def fake_generate_sparse_seqlen_triton(
|
||||
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
||||
@@ -153,8 +152,7 @@ def test_build_populates_decode_only_split_fields(monkeypatch):
|
||||
|
||||
def test_build_populates_mixed_split_fields(monkeypatch):
|
||||
"""Mixed decode+prefill batch: split is reported, prefill fields stay
|
||||
default because this impl always runs the MQA path.
|
||||
"""
|
||||
default because this impl always runs the MQA path."""
|
||||
builder = _make_builder()
|
||||
_patch_build_deps(monkeypatch)
|
||||
|
||||
|
||||
@@ -125,8 +125,7 @@ def _quantize_to_fp8(tensor: torch.Tensor):
|
||||
@pytest.mark.parametrize("PAGE_SIZE", [1, 16])
|
||||
def test_decode_attention_fp8(B, L, H_Q, H_KV, D_QK, D_V, CACHE_SIZE, PAGE_SIZE):
|
||||
"""Test FP8 KV cache path: quantize K/V to FP8, run kernel with scales,
|
||||
and compare against BF16 reference output.
|
||||
"""
|
||||
and compare against BF16 reference output."""
|
||||
assert CACHE_SIZE % PAGE_SIZE == 0
|
||||
dtype = torch.bfloat16
|
||||
seq_len = L
|
||||
@@ -249,8 +248,7 @@ def test_decode_attention_cross_layer_view(H_Q, H_KV, D_QK, D_V, is_mla, PAGE_SI
|
||||
"""The kernel must honor the cache's page-dim stride, not assume pages are
|
||||
packed back-to-back. A per-layer view into a cross-layer (block-major)
|
||||
cache has stride(0) inflated by num_layers; outputs must match a
|
||||
contiguous cache holding the same data exactly.
|
||||
"""
|
||||
contiguous cache holding the same data exactly."""
|
||||
B = 3
|
||||
seq_len = 1027
|
||||
CACHE_SIZE = 16384
|
||||
|
||||
@@ -41,8 +41,7 @@ def _rmsnorm(x: torch.Tensor, w: torch.Tensor, eps: float) -> torch.Tensor:
|
||||
def test_fused_embed_norm_matches_reference():
|
||||
"""Main-model fusion: the residual is the exact gather and the second output
|
||||
is a correct RMSNorm. The norm matches a full-precision reference to ~2 bf16
|
||||
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel.
|
||||
"""
|
||||
ulp (rtol 1e-2) -- that gap is bf16 rounding, not the kernel."""
|
||||
set_random_seed(13)
|
||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||
@@ -61,8 +60,7 @@ def test_fused_embed_norm_matches_reference():
|
||||
@torch.inference_mode()
|
||||
def test_fused_embed_eh_norm_matches_reference():
|
||||
"""MTP fusion (folded gather) is bit-exact vs gathering the embeds and
|
||||
feeding the untouched model-local ``fused_eh_norm``.
|
||||
"""
|
||||
feeding the untouched model-local ``fused_eh_norm``."""
|
||||
set_random_seed(13)
|
||||
table = torch.randn(VOCAB, HIDDEN, dtype=DTYPE, device="cuda")
|
||||
ids = torch.randint(0, VOCAB, (NUM_TOKENS,), dtype=torch.int32, device="cuda")
|
||||
|
||||
@@ -62,8 +62,7 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
||||
a [1, q_size] slice of a [1, q_size + kv_size] buffer used to produce a
|
||||
qr_out with row stride q_size + kv_size. Downstream dispatchers that
|
||||
require packed row-major inputs then reject the tensor and silently fall
|
||||
back to a slower GEMM path on every decode step.
|
||||
"""
|
||||
back to a slower GEMM path on every decode step."""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
@@ -89,8 +88,7 @@ def test_fused_q_kv_rmsnorm_outputs_are_packed(num_tokens: int):
|
||||
def test_fused_q_kv_rmsnorm_launches_past_grid_y_cap(num_tokens: int):
|
||||
"""Regression guard: grid used to be (2, num_tokens), hitting CUDA's
|
||||
65535 grid-y cap at num_tokens >= 65536. The new grid (num_tokens, 2)
|
||||
lifts that bound to 2**31-1.
|
||||
"""
|
||||
lifts that bound to 2**31-1."""
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
q_size, kv_size = 192, 576
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests that FusedRMSNormGated decomposes correctly under torch.compile,
|
||||
matching the eager triton kernel output.
|
||||
"""
|
||||
matching the eager triton kernel output."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -579,8 +579,7 @@ def test_rocm_aiter_mla_dual_rmsnorm_group_quant_vs_sequential(m):
|
||||
|
||||
def test_rocm_aiter_mla_dual_rmsnorm_group_quant_strided_split_view():
|
||||
"""Split-view inputs, as produced by the DeepSeek-V4 q/kv split, match
|
||||
contiguous inputs.
|
||||
"""
|
||||
contiguous inputs."""
|
||||
require_aiter()
|
||||
require_fp8()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
@@ -87,8 +87,7 @@ def _shard(full_table, si, rows_per_rank, dtype):
|
||||
@pytest.mark.parametrize("id_dtype", [torch.int32, torch.int64])
|
||||
def test_matches_masked_gather(tp_size, vocab_size, org_vocab_size, hidden, id_dtype):
|
||||
"""Every rank's partial output is bit-exact with the eager path, including
|
||||
the added-vocab (LoRA) rows and the ids owned by another rank.
|
||||
"""
|
||||
the added-vocab (LoRA) rows and the ids owned by another rank."""
|
||||
set_random_seed(7)
|
||||
dtype = torch.bfloat16
|
||||
rows = pad_vocab_size(vocab_size, 64) // tp_size
|
||||
|
||||
@@ -59,8 +59,7 @@ class TestAutotuneDisabledKernel:
|
||||
|
||||
def test_autotune_disabled_kernel_produces_valid_config(self):
|
||||
"""Register a kernel with no configs (disabled), run autotune,
|
||||
verify it produces a valid helion.Config.
|
||||
"""
|
||||
verify it produces a valid helion.Config."""
|
||||
with dummy_kernel_registry(configs={}) as register:
|
||||
wrapper = register(
|
||||
"autotune_test_kernel",
|
||||
|
||||
@@ -598,8 +598,7 @@ class TestHelionKernelWrapper:
|
||||
)
|
||||
def test_init_eagerly_initializes_hop_path(self):
|
||||
"""Test that register_kernel eagerly builds the configured kernel
|
||||
on the HOP path (no custom op registration needed).
|
||||
"""
|
||||
on the HOP path (no custom op registration needed)."""
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
configs: dict[CaseKey, helion.Config] = {
|
||||
@@ -634,8 +633,7 @@ class TestHelionKernelWrapper:
|
||||
)
|
||||
def test_init_eagerly_initializes(self):
|
||||
"""Test that register_kernel eagerly loads configs and detects GPU
|
||||
during construction so __call__ needs no further initialization.
|
||||
"""
|
||||
during construction so __call__ needs no further initialization."""
|
||||
from vllm.kernels.helion.utils import get_canonical_gpu_name
|
||||
|
||||
with (
|
||||
@@ -963,8 +961,7 @@ class TestTorchCompileHOP:
|
||||
|
||||
def test_compiled_graph_contains_helion_hop(self):
|
||||
"""Verify torch.compile on a HelionKernelWrapper emits a
|
||||
helion_kernel_wrapper_mutation HOP node in the FX graph.
|
||||
"""
|
||||
helion_kernel_wrapper_mutation HOP node in the FX graph."""
|
||||
configs: dict[CaseKey, helion.Config] = {
|
||||
CaseKey.default(): helion.Config(block_sizes=[4, 4])
|
||||
}
|
||||
|
||||
@@ -368,8 +368,7 @@ class TestFusedAddRMSNorm:
|
||||
@pytest.mark.parametrize("provider", ["vllm_c"])
|
||||
def test_inplace_semantics(self, dtype, n_tokens, hidden_size, epsilon, provider):
|
||||
"""Test that inplace implementations reuse inputs,
|
||||
for maybe_inplace overload but not for default overload.
|
||||
"""
|
||||
for maybe_inplace overload but not for default overload."""
|
||||
impl = ir.ops.fused_add_rms_norm.impls[provider]
|
||||
if not impl.supported:
|
||||
pytest.skip(f"{provider} impl not supported on this platform")
|
||||
|
||||
@@ -1055,8 +1055,7 @@ CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]
|
||||
|
||||
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
|
||||
"""High-precision conv reference: everything in float32, no bf16 rounding.
|
||||
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim].
|
||||
"""
|
||||
x: [total_tokens, dim] (no initial state). Returns [total_tokens, dim]."""
|
||||
xf = x.float()
|
||||
wf = weight.float().unsqueeze(1)
|
||||
bf = bias.float()
|
||||
@@ -1165,8 +1164,7 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
|
||||
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
|
||||
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
|
||||
conv_state view + is_vnni=True) must match the torch fallback. Validates
|
||||
the packing + layout handoff on any AVX-512BF16 CPU.
|
||||
"""
|
||||
the packing + layout handoff on any AVX-512BF16 CPU."""
|
||||
x, weight, bias = _conv_inputs(sum(seq_lens))
|
||||
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
|
||||
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
|
||||
@@ -1180,8 +1178,7 @@ def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
|
||||
@torch.inference_mode()
|
||||
def test_conv_update_cpp_matches_torch(batch):
|
||||
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
|
||||
including the in-place conv_state update (the next-step handoff).
|
||||
"""
|
||||
including the in-place conv_state update (the next-step handoff)."""
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
|
||||
@@ -122,8 +122,7 @@ def test_fallback_when_no_config(monkeypatch, tmp_path):
|
||||
|
||||
def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
|
||||
"""When effective_batch = batch*nheads is not an exact key, the closest
|
||||
key should be selected.
|
||||
"""
|
||||
key should be selected."""
|
||||
_write_config(
|
||||
tmp_path,
|
||||
dstate=32,
|
||||
@@ -168,8 +167,7 @@ def test_nearest_effective_batch_interpolation(monkeypatch, tmp_path):
|
||||
|
||||
def test_non_dict_json_returns_none(monkeypatch, tmp_path):
|
||||
"""A valid JSON file that is not a dict (e.g. a list) must be ignored
|
||||
and return None rather than raising AttributeError.
|
||||
"""
|
||||
and return None rather than raising AttributeError."""
|
||||
device_name = get_ssm_device_name()
|
||||
config_path = tmp_path / get_ssm_config_file_name(
|
||||
_HEADDIM, 16, _CACHE_DTYPE, device_name
|
||||
@@ -191,8 +189,7 @@ def test_non_dict_json_returns_none(monkeypatch, tmp_path):
|
||||
|
||||
def test_empty_config_falls_back_to_heuristic(monkeypatch, tmp_path):
|
||||
"""An empty JSON object {} must not crash min() — should fall back
|
||||
to the hard-coded heuristic.
|
||||
"""
|
||||
to the hard-coded heuristic."""
|
||||
_write_config(tmp_path, dstate=64, payload={})
|
||||
|
||||
monkeypatch.setenv("VLLM_TUNED_CONFIG_FOLDER", str(tmp_path))
|
||||
|
||||
@@ -68,8 +68,7 @@ def _run_prefill_decode_equivalence(
|
||||
the exact fp32 recurrence (and each other). All paths use the production dt
|
||||
flow (raw dt + softplus + a per-head dt_bias), so this also checks prefill
|
||||
and decode apply the softplus/bias preprocessing consistently. ``state_dtype``
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one.
|
||||
"""
|
||||
is the recurrent-state precision; ``act_dtype`` the activation/buffer one."""
|
||||
device = current_platform.device_type
|
||||
rtol, atol = _prefill_tolerances(act_dtype)
|
||||
set_random_seed(seed)
|
||||
|
||||
@@ -95,8 +95,7 @@ def _run_standard_decode(
|
||||
) -> None:
|
||||
"""Drive ``num_steps`` decode steps and check every path against the
|
||||
trusted recurrence. ``state_dtype`` is the recurrent-state precision;
|
||||
``act_dtype`` is the activation/buffer precision.
|
||||
"""
|
||||
``act_dtype`` is the activation/buffer precision."""
|
||||
device = "cuda"
|
||||
both_fp32 = state_dtype == torch.float32 and act_dtype == torch.float32
|
||||
rtol, atol = _tolerances(torch.float32 if both_fp32 else torch.bfloat16)
|
||||
@@ -540,8 +539,7 @@ def test_replayssm_standard_decode_tp_head_shard_equivalence(
|
||||
separable: one run over all heads must equal concatenating ``tp`` independent
|
||||
per-rank runs on ``nheads // tp`` heads and ``ngroups // tp`` groups. This
|
||||
guards the per-rank divisors and group->head mapping the state-shape wiring
|
||||
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite.
|
||||
"""
|
||||
relies on. The real TP1==TP2 engine check lives in the v1/e2e suite."""
|
||||
state_dtype, act_dtype = precision
|
||||
nheads, headdim, dstate, ngroups = geometry
|
||||
assert nheads % tp == 0 and ngroups % tp == 0
|
||||
|
||||
@@ -167,8 +167,7 @@ class Config:
|
||||
|
||||
def fe_supports_quant_scheme(self) -> bool:
|
||||
"""Check if the fused experts class supports this quant config.
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps.
|
||||
"""
|
||||
See https://github.com/ROCm/aiter/issues/2419 for AITER gaps."""
|
||||
if self.quant_config is None or self.quant_dtype is None:
|
||||
return True
|
||||
if self.quant_dtype != torch.float8_e4m3fn:
|
||||
|
||||
@@ -468,8 +468,7 @@ def test_deepseek_v4_flashinfer_cutlass_falls_through_to_w4a8(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Explicit flashinfer_cutlass must try the W4A8 variant when the BF16
|
||||
variant is unsupported (the BF16 variant is gated to SM90).
|
||||
"""
|
||||
variant is unsupported (the BF16 variant is gated to SM90)."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
|
||||
@@ -129,8 +129,7 @@ def test_batched_mm(
|
||||
per_act_token_quant: bool,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
@@ -269,8 +268,7 @@ def test_fused_moe_batched_experts(
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
set_random_seed(7)
|
||||
|
||||
use_fp8_w8a8 = dtype == torch.float8_e4m3fn
|
||||
@@ -651,8 +649,7 @@ def test_batched_experts_masked_activation_end_to_end(
|
||||
def test_batched_experts_end_to_end(m, n, k, e, topk):
|
||||
"""End-to-end BatchedTritonExperts via the reference (no-comms)
|
||||
BatchedPrepareAndFinalize, validated against a torch reference. Exercises
|
||||
the device-enablement path.
|
||||
"""
|
||||
the device-enablement path."""
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
set_random_seed(7)
|
||||
|
||||
@@ -54,8 +54,7 @@ SEEDS = [0]
|
||||
# For test
|
||||
def torch_w8a8_block_int8_moe(a, w1, w2, w1_s, w2_s, score, topk, block_shape):
|
||||
"""This function performs fused moe with block-wise quantization using
|
||||
native torch.
|
||||
"""
|
||||
native torch."""
|
||||
B, D = a.shape
|
||||
a = a.view(B, -1, D).repeat(1, topk, 1).reshape(-1, D)
|
||||
out = torch.zeros(B * topk, w2.shape[1], dtype=a.dtype, device=a.device)
|
||||
@@ -98,8 +97,7 @@ def setup_cuda():
|
||||
@torch.inference_mode()
|
||||
def test_w8a8_block_int8_fused_moe(M, N, K, E, topk, block_size, dtype, seed):
|
||||
"""Tests the fused_moe kernel with W8A8 INT8 block quantization against a
|
||||
native torch reference.
|
||||
"""
|
||||
native torch reference."""
|
||||
torch.manual_seed(seed)
|
||||
|
||||
a = torch.randn((M, K), dtype=dtype) / 10
|
||||
|
||||
@@ -397,8 +397,7 @@ UNALIGNED_INTERMEDIATE_DIM = 176
|
||||
class _StubMoELayer(torch.nn.Module):
|
||||
"""Minimal stand-in for the real MoE layer module, exposing just what
|
||||
the unquantized CPU experts read/replace (w13_weight, w2_weight, the
|
||||
router configuration, and optionally w13_bias/w2_bias).
|
||||
"""
|
||||
router configuration, and optionally w13_bias/w2_bias)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -501,8 +500,7 @@ def test_cpu_fused_moe_unaligned_intermediate_size(
|
||||
act: MoEActivation,
|
||||
):
|
||||
"""CPU kernels handle unaligned intermediate sizes by zero-padding the
|
||||
weights before prepacking.
|
||||
"""
|
||||
weights before prepacking."""
|
||||
set_random_seed(0)
|
||||
batch_size = 64
|
||||
intermediate_size = UNALIGNED_INTERMEDIATE_DIM
|
||||
|
||||
@@ -464,8 +464,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
):
|
||||
"""FlashInfer >= 0.6.18 applies the SwiGLU clamp for both block-scaled
|
||||
kernels with a SwiGLU activation (DeepSeek-V4 sets swiglu_limit); the
|
||||
per-tensor kernel has no clamp, and Relu2 rejects the parameters.
|
||||
"""
|
||||
per-tensor kernel has no clamp, and Relu2 rejects the parameters."""
|
||||
|
||||
class _Experts(TrtLlmFp8ExpertsModular):
|
||||
@staticmethod
|
||||
|
||||
@@ -377,8 +377,7 @@ def test_unit_shuffle():
|
||||
@pytest.mark.parametrize("n_experts,topk", [(32, 4), (128, 4)])
|
||||
def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
"""routing_data_from_sparse_topk must produce routing structures
|
||||
identical to make_routing_data for the same topk result.
|
||||
"""
|
||||
identical to make_routing_data for the same topk result."""
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
@@ -289,8 +289,7 @@ def test_marlin_vs_trtllm_mxint4_moe_kimik2(monkeypatch, m, n, k, e, topk, group
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_trtllm_mxint4_moe_wrapper(m, n, k, e, topk):
|
||||
"""Test that the flashinfer_trtllm_mxint4_moe wrapper matches the raw
|
||||
trtllm_mxint4_block_scale_moe kernel call.
|
||||
"""
|
||||
trtllm_mxint4_block_scale_moe kernel call."""
|
||||
pytest.importorskip("flashinfer")
|
||||
from flashinfer import RoutingMethodType
|
||||
from flashinfer.fused_moe import trtllm_mxint4_block_scale_moe
|
||||
|
||||
@@ -352,8 +352,7 @@ def test_modular_kernel_combinations_singlegpu(
|
||||
workspace_init,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware.
|
||||
"""
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
config = Config(
|
||||
Ms=Ms,
|
||||
K=k,
|
||||
|
||||
@@ -21,8 +21,7 @@ from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
|
||||
class TestUnquantizedDelegation:
|
||||
"""UnquantizedMoEKernelOracle methods must delegate to the existing
|
||||
module-level functions; behaviour is bit-identical.
|
||||
"""
|
||||
module-level functions; behaviour is bit-identical."""
|
||||
|
||||
def test_make_kernel_delegates(self) -> None:
|
||||
quant_config = object()
|
||||
|
||||
@@ -315,8 +315,7 @@ def tp_chunk_gate_up(
|
||||
device: torch.device | int | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""TP-chunk a combined [gate; up] weight, splitting each half separately
|
||||
so every rank gets a portion of both gate and up.
|
||||
"""
|
||||
so every rank gets a portion of both gate and up."""
|
||||
half = w.shape[dim] // 2
|
||||
gate = chunk_by_rank(
|
||||
w.narrow(dim, 0, half), tp_rank, tp_size, dim=dim, device=device
|
||||
|
||||
@@ -130,8 +130,7 @@ def test_is_supported_config(present, ep_size, supported, reason_substr):
|
||||
|
||||
def test_explicit_moe_backend_aiter():
|
||||
"""--moe-backend aiter: returns FlyDSL when usable (TP or EP), else a clear
|
||||
ValueError when the flydsl package is missing.
|
||||
"""
|
||||
ValueError when the flydsl package is missing."""
|
||||
with _gfx950(), _flydsl_installed(True):
|
||||
assert (
|
||||
_select_kernel_cls(Fp8MoeBackend.AITER_MXFP8, _config(1))
|
||||
@@ -163,8 +162,7 @@ def test_gfx950_picks_aiter():
|
||||
|
||||
def test_gfx942_picks_emulation():
|
||||
"""Flydsl unusable (e.g. gfx942, no FlyDSL support) -> native Triton
|
||||
dot_scaled backend wins instead.
|
||||
"""
|
||||
dot_scaled backend wins instead."""
|
||||
with patch(f"{_AITER_MOD}.current_platform.supports_mx", return_value=False):
|
||||
backend, experts_cls = select_mxfp8_moe_backend(_config())
|
||||
assert backend is Fp8MoeBackend.EMULATION
|
||||
|
||||
@@ -249,8 +249,7 @@ def _make_topk_ids(
|
||||
device: str = "cuda",
|
||||
) -> torch.Tensor:
|
||||
"""Generate distinct expert IDs per token with the same top-k shape as
|
||||
production routing.
|
||||
"""
|
||||
production routing."""
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
_, topk_ids = torch.topk(torch.softmax(router_logits, dim=-1), k=topk, dim=-1)
|
||||
return topk_ids.to(torch.int32)
|
||||
@@ -444,8 +443,7 @@ def test_aiter_fused_moe_custom_op_registered():
|
||||
|
||||
def test_aiter_asm_moe_tkw1_custom_op_registered():
|
||||
"""The tkw1 custom op should stay registered for FP8 apply-router-weight
|
||||
paths.
|
||||
"""
|
||||
paths."""
|
||||
_assert_aiter_supported()
|
||||
import vllm._aiter_ops as aiter_ops # noqa: F401
|
||||
|
||||
@@ -455,8 +453,7 @@ def test_aiter_asm_moe_tkw1_custom_op_registered():
|
||||
|
||||
def test_aiter_fused_moe_fake_tensor_support():
|
||||
"""The fused-MoE op should preserve fake-tensor compatibility for
|
||||
torch.compile-style tracing.
|
||||
"""
|
||||
torch.compile-style tracing."""
|
||||
_assert_aiter_supported()
|
||||
import vllm._aiter_ops # noqa: F401
|
||||
|
||||
@@ -519,8 +516,7 @@ def test_aiter_moe_enablement_follows_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""The fused-MoE gate should depend only on the main AITER toggle and the
|
||||
MoE-specific toggle.
|
||||
"""
|
||||
MoE-specific toggle."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -551,8 +547,7 @@ def test_aiter_moe_shared_experts_enablement_follows_env(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Shared-expert fusion should only be enabled when the fused-MoE path is
|
||||
enabled too.
|
||||
"""
|
||||
enabled too."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
@@ -693,8 +688,7 @@ def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
|
||||
"""The gfx950 AITER MXFP4 W4A16 MoE path should match the dequantized
|
||||
MXFP4 reference.
|
||||
"""
|
||||
MXFP4 reference."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
@@ -743,8 +737,7 @@ def test_aiter_fused_moe_mi350_mxfp4_w4a16_accuracy():
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fused_moe_mi350_mxfp4_w4a16_determinism():
|
||||
"""The gfx950 AITER MXFP4 W4A16 MoE path should stay bitwise
|
||||
deterministic.
|
||||
"""
|
||||
deterministic."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
@@ -837,8 +830,7 @@ def test_aiter_fused_moe_bf16_accuracy(
|
||||
intermediate_dim: int,
|
||||
):
|
||||
"""The ROCm AITER fused-MoE BF16 path should match the float32 reference
|
||||
on representative shapes.
|
||||
"""
|
||||
on representative shapes."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -883,8 +875,7 @@ def test_aiter_fused_moe_bf16_accuracy(
|
||||
|
||||
def test_aiter_fused_moe_gelu_accuracy():
|
||||
"""The GELU activation variant should stay aligned with the float32
|
||||
reference.
|
||||
"""
|
||||
reference."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -929,8 +920,7 @@ def test_aiter_fused_moe_gelu_accuracy():
|
||||
|
||||
def test_aiter_fused_moe_determinism():
|
||||
"""The BF16 fused-MoE kernel should stay bitwise deterministic for the
|
||||
same inputs.
|
||||
"""
|
||||
same inputs."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
@@ -1085,8 +1075,7 @@ def test_aiter_fused_moe_mi3xx_bf16_accuracy():
|
||||
)
|
||||
def test_aiter_fused_moe_mi3xx_fp8_accuracy():
|
||||
"""The MI3xx FP8 per-tensor MoE path should stay within the measured FP8
|
||||
error budget.
|
||||
"""
|
||||
error budget."""
|
||||
from tests.kernels.moe.utils import make_test_weights
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
|
||||
@@ -185,8 +185,7 @@ def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts(
|
||||
) -> None:
|
||||
"""The capture callback should receive the int16 routed-expert IDs the
|
||||
kernel actually used, the values should be valid expert indices, and
|
||||
each token should pick ``top_k`` distinct experts.
|
||||
"""
|
||||
each token should pick ``top_k`` distinct experts."""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
@@ -265,8 +264,7 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
|
||||
routing_method: RoutingMethodType,
|
||||
) -> None:
|
||||
"""Routing replay works for non-DeepSeekV3 routing methods too.
|
||||
FlashInfer's ``routing_replay_out`` is routing-method-agnostic.
|
||||
"""
|
||||
FlashInfer's ``routing_replay_out`` is routing-method-agnostic."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda:0")
|
||||
|
||||
@@ -319,8 +317,7 @@ def test_trtllm_bf16_monolithic_routing_replay_non_dsv3(
|
||||
|
||||
def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
|
||||
"""With no callback installed the kernel should not see a
|
||||
``routing_replay_out`` tensor — verify the helper short-circuits.
|
||||
"""
|
||||
``routing_replay_out`` tensor — verify the helper short-circuits."""
|
||||
torch.manual_seed(0)
|
||||
device = torch.device("cuda:0")
|
||||
experts, _, _ = _make_bf16_monolithic_experts(
|
||||
@@ -342,8 +339,7 @@ def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None:
|
||||
def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
|
||||
"""FlashInfer's ``routing_replay_out`` is supported by all routing
|
||||
methods, so ``supports_routing_replay_capture`` should be True
|
||||
regardless of routing method.
|
||||
"""
|
||||
regardless of routing method."""
|
||||
device = torch.device("cuda:0")
|
||||
for routing_method in (
|
||||
RoutingMethodType.DeepSeekV3,
|
||||
@@ -365,8 +361,7 @@ def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None:
|
||||
|
||||
def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None:
|
||||
"""When capture is installed, the allocated buffer is int16 and shaped
|
||||
``(num_tokens, experts_per_token)``.
|
||||
"""
|
||||
``(num_tokens, experts_per_token)``."""
|
||||
device = torch.device("cuda:0")
|
||||
experts, _, _ = _make_bf16_monolithic_experts(
|
||||
num_experts=_DSV3_NUM_EXPERTS,
|
||||
@@ -597,8 +592,7 @@ def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts(
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale
|
||||
path, DSV3 routing) captures valid expert IDs.
|
||||
"""
|
||||
path, DSV3 routing) captures valid expert IDs."""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
@@ -784,8 +778,7 @@ def _run_nvfp4_monolithic(
|
||||
routing_bias: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""The monolithic NVFP4 apply expects packed fp4 hidden states + the
|
||||
matching fp8 per-block scale stored in the ``a1q_scale`` slot.
|
||||
"""
|
||||
matching fp8 per-block scale stored in the ``a1q_scale`` slot."""
|
||||
# Stash the weight tensors on the experts in the locations the apply()
|
||||
# implementation reads from (it pulls them from quant_config / scales
|
||||
# already; w1/w2 come in as args).
|
||||
@@ -813,8 +806,7 @@ def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts(
|
||||
top_k: int,
|
||||
) -> None:
|
||||
"""End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs
|
||||
on the DSV3 routing path.
|
||||
"""
|
||||
on the DSV3 routing path."""
|
||||
if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP:
|
||||
pytest.skip(
|
||||
f"DSV3 requires top_k <= n_group * topk_group "
|
||||
|
||||
@@ -250,8 +250,7 @@ def assert_aiter_routing_valid(
|
||||
the Python baseline (different group selection, scoring internals),
|
||||
so numerical comparison is not meaningful. Instead we verify the
|
||||
outputs satisfy the routing contract: correct shapes, valid expert
|
||||
IDs, non-negative weights, and proper normalization.
|
||||
"""
|
||||
IDs, non-negative weights, and proper normalization."""
|
||||
n_tokens = topk_weights.shape[0]
|
||||
|
||||
# Shape
|
||||
|
||||
@@ -76,8 +76,7 @@ def test_basic_functionality(
|
||||
|
||||
def test_routing_strategy_integration(monkeypatch, device):
|
||||
"""Test that the routing strategy environment variable works with
|
||||
FusedMoEFactory.
|
||||
"""
|
||||
FusedMoEFactory."""
|
||||
pytest.importorskip("vllm.model_executor.layers.fused_moe.layer")
|
||||
|
||||
import vllm.envs as envs
|
||||
@@ -158,8 +157,7 @@ def test_routing_strategy_integration(monkeypatch, device):
|
||||
|
||||
def test_distribution_based_routing_with_custom_strategy():
|
||||
"""Test registering and using DistributionBasedRouting with custom
|
||||
parameters.
|
||||
"""
|
||||
parameters."""
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
# Register custom distribution-based strategy
|
||||
|
||||
@@ -70,8 +70,7 @@ def reference_with_clamp(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pre-clamp inputs (gate from above, up symmetric) at the input dtype to
|
||||
match the C++ compute() template, then run the standard silu_and_mul +
|
||||
quant reference.
|
||||
"""
|
||||
quant reference."""
|
||||
N_2 = x.size(1) // 2
|
||||
dtype = x.dtype
|
||||
gate = x[..., :N_2].to(torch.float32).clamp(max=clamp_limit).to(dtype)
|
||||
|
||||
@@ -28,8 +28,7 @@ if current_platform.is_fp8_fnuz():
|
||||
|
||||
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
"""Matrix multiplication function that supports per-token input
|
||||
quantization and per-column weight quantization
|
||||
"""
|
||||
quantization and per-column weight quantization"""
|
||||
A = A.to(torch.float32)
|
||||
B = B.to(torch.float32)
|
||||
|
||||
@@ -57,8 +56,7 @@ def fp8_mask(a, mask):
|
||||
|
||||
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, score, topk):
|
||||
"""This function performs fused moe with per-column int8
|
||||
quantization using native torch.
|
||||
"""
|
||||
quantization using native torch."""
|
||||
B, D = a.shape
|
||||
# Perform per-token quantization
|
||||
a_q, a_s = ops.scaled_fp8_quant(a, use_per_token_if_dynamic=True)
|
||||
|
||||
@@ -150,8 +150,7 @@ def test_select_default_backend_by_platform(
|
||||
expected_backend,
|
||||
):
|
||||
"""Test default backend selection per platform with all optional
|
||||
accelerators (FlashInfer, AITER) disabled.
|
||||
"""
|
||||
accelerators (FlashInfer, AITER) disabled."""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
@@ -445,8 +444,7 @@ def test_select_cuda_flashinfer_cutlass_backend(
|
||||
mock_has_flashinfer,
|
||||
):
|
||||
"""Test CUDA backend selection when FlashInfer TRTLLM is not available
|
||||
and FlashInfer CUTLASS is available.
|
||||
"""
|
||||
and FlashInfer CUTLASS is available."""
|
||||
with (
|
||||
patch.object(current_platform, "is_cuda", return_value=True),
|
||||
patch.object(current_platform, "is_rocm", return_value=False),
|
||||
|
||||
@@ -159,8 +159,7 @@ def native_per_token_group_quant_fp8(
|
||||
x, group_size, eps=1e-10, dtype=torch.float8_e4m3fn
|
||||
):
|
||||
"""Function to perform per-token-group quantization on an input tensor
|
||||
`x` using native torch.
|
||||
"""
|
||||
`x` using native torch."""
|
||||
assert x.shape[-1] % group_size == 0, (
|
||||
"the last dimension of `x` must be divisible by `group_size`"
|
||||
)
|
||||
|
||||
@@ -304,8 +304,7 @@ def test_hipb_mm_kernel_forward_matches_raw_aiter_hipb_mm(enable_hipb_mm_kernel)
|
||||
|
||||
def test_hipb_mm_kernel_forward_accuracy(enable_hipb_mm_kernel):
|
||||
"""Kernel output should match a dequantized fp32 reference within
|
||||
fp8 per-token / per-channel quantization noise.
|
||||
"""
|
||||
fp8 per-token / per-channel quantization noise."""
|
||||
weight_shape = (512, 4096) # (N, K)
|
||||
num_tokens = 32
|
||||
_check_bpreshuffle_runtime_support(weight_shape, num_tokens=num_tokens)
|
||||
|
||||
@@ -30,8 +30,7 @@ if current_platform.is_cuda_alike() and not current_platform.has_device_capabili
|
||||
|
||||
def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
"""Matrix multiplication function that supports per-token input
|
||||
quantization and per-column weight quantization
|
||||
"""
|
||||
quantization and per-column weight quantization"""
|
||||
A = A.to(torch.float32)
|
||||
B = B.to(torch.float32)
|
||||
|
||||
@@ -54,8 +53,7 @@ def native_w8a8_per_token_matmul(A, B, As, Bs, output_dtype=torch.float16):
|
||||
|
||||
def torch_w8a8_per_column_moe(a, w1, w2, w1_s, w2_s, topk, topk_weight, topk_ids):
|
||||
"""This function performs fused moe with per-column int8 quantization
|
||||
using native torch.
|
||||
"""
|
||||
using native torch."""
|
||||
B, D = a.shape
|
||||
# Perform per-token quantization
|
||||
a_q, a_s = per_token_quant_int8(a)
|
||||
|
||||
@@ -360,8 +360,7 @@ def test_gptq_marlin_padded_round_trip(shape, group_size):
|
||||
def test_fp8_block_marlin_padded_round_trip(shape):
|
||||
"""Block-quantized FP8 (e.g. Nemotron NVFP4 checkpoints' FP8 layers):
|
||||
group_size=128 exercises the lcm K-alignment in marlin_padded_nk and the
|
||||
weight_scale_inv group-wise scale padding.
|
||||
"""
|
||||
weight_scale_inv group-wise scale padding."""
|
||||
size_n, size_k = shape
|
||||
block = 128
|
||||
dtype = torch.float16
|
||||
@@ -411,8 +410,7 @@ def test_fp8_block_marlin_padded_round_trip(shape):
|
||||
@pytest.mark.parametrize("shape", [(200, 288), (4640, 512)])
|
||||
def test_mxfp8_marlin_padded_round_trip(shape):
|
||||
"""MXFP8 exercises the e8m0 scale path, where padded 0.0 scales clamp to
|
||||
2^-127 instead of zero and must still contribute nothing.
|
||||
"""
|
||||
2^-127 instead of zero and must still contribute nothing."""
|
||||
size_n, size_k = shape
|
||||
group_size = 32
|
||||
# The e8m0-scale Marlin kernels are only instantiated for bf16 activations.
|
||||
@@ -459,8 +457,7 @@ def test_mxfp8_marlin_padded_round_trip(shape):
|
||||
@pytest.mark.parametrize("shape", [(200, 512), (4640, 512)])
|
||||
def test_awq_zp_marlin_padded_round_trip(shape):
|
||||
"""AWQ-style uint4 with runtime zero-points, padded the way
|
||||
MarlinLinearKernel does: padded columns rely on (q=0 - zp=0) * scale=0.
|
||||
"""
|
||||
MarlinLinearKernel does: padded columns rely on (q=0 - zp=0) * scale=0."""
|
||||
size_n, size_k = shape
|
||||
group_size = 128
|
||||
dtype = torch.float16
|
||||
@@ -695,8 +692,7 @@ def test_check_moe_marlin_supports_layer_padding():
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
"""FP8 weight-only MoE: pad a tile-misaligned intermediate and check the
|
||||
real prepare + fused_marlin_moe against the dequantized reference.
|
||||
"""
|
||||
real prepare + fused_marlin_moe against the dequantized reference."""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
@@ -781,8 +777,7 @@ def test_fp8_marlin_moe_padded_round_trip(shape, quant):
|
||||
@pytest.mark.parametrize("shape", [(96, 256, 8), (160, 512, 4)])
|
||||
def test_mxfp8_marlin_moe_padded_round_trip(shape):
|
||||
"""MXFP8 weight-only MoE round-trip at a tile-misaligned intermediate, with
|
||||
unit e8m0 scales so the reference is the exact fp8 dequant.
|
||||
"""
|
||||
unit e8m0 scales so the reference is the exact fp8 dequant."""
|
||||
from tests.kernels.utils import torch_experts
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
|
||||
@@ -72,8 +72,7 @@ def test_true_w4a4_kernels_reject_unset_activation(kernel_cls):
|
||||
@pytest.mark.parametrize("kernel_cls", _TRUE_W4A4_KERNELS)
|
||||
def test_true_w4a4_kernels_reject_explicit_non_mxfp4_activation(kernel_cls):
|
||||
"""FlashInfer/XPU/Aiter quantize activations to MXFP4 internally, so an
|
||||
explicit request for a different activation format must be rejected.
|
||||
"""
|
||||
explicit request for a different activation format must be rejected."""
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=kMxfp6E3M2Dynamic)
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
assert not can_implement
|
||||
@@ -88,8 +87,7 @@ def test_weight_only_kernels_accept_unquantized_or_mxfp4_activation(
|
||||
"""Marlin/Humming never quantize activations, so an unset activation key,
|
||||
or one that already describes MXFP4-shaped data, is tolerated. When an
|
||||
activation key is explicitly set, a warning must be logged noting that it
|
||||
is ignored, since these kernels are weight-only (A16).
|
||||
"""
|
||||
is ignored, since these kernels are weight-only (A16)."""
|
||||
config = MxFp4LinearLayerConfig(activation_quant_key=activation_quant_key)
|
||||
with patch(f"{kernel_cls.__module__}.logger.warning_once") as warning_once:
|
||||
can_implement, reason = kernel_cls.can_implement(config)
|
||||
@@ -118,8 +116,7 @@ def test_weight_only_kernels_reject_non_mxfp4_activation(kernel_cls):
|
||||
)
|
||||
def test_emulation_kernel_accepts_any_config(activation_quant_key, monkeypatch):
|
||||
"""EmulationMxfp4LinearKernel is the universal fallback: it must accept
|
||||
every supported activation format.
|
||||
"""
|
||||
every supported activation format."""
|
||||
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
|
||||
# which we are not testing here.
|
||||
monkeypatch.setattr(
|
||||
@@ -137,8 +134,7 @@ def test_emulation_kernel_accepts_any_config(activation_quant_key, monkeypatch):
|
||||
|
||||
def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
|
||||
"""quant_dequant_func must be derived purely from the config's activation
|
||||
QuantKey, not set externally.
|
||||
"""
|
||||
QuantKey, not set externally."""
|
||||
# `EmulationMxfp4LinearKernel.can_implement` gates on `has_quark()`,
|
||||
# which we are not testing here.
|
||||
monkeypatch.setattr(
|
||||
@@ -162,8 +158,7 @@ def test_emulation_kernel_derives_quant_dequant_func_from_config(monkeypatch):
|
||||
|
||||
def test_aiter_kernel_is_supported_requires_native_mx_support():
|
||||
"""AiterMxfp4LinearKernel must not be selected on platforms without
|
||||
native MX compute, even if AITER itself is importable.
|
||||
"""
|
||||
native MX compute, even if AITER itself is importable."""
|
||||
with patch(
|
||||
"vllm.model_executor.kernels.linear.mxfp4.aiter.current_platform.supports_mx",
|
||||
return_value=False,
|
||||
@@ -199,8 +194,7 @@ class OOTMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
@patch("vllm.model_executor.kernels.linear.current_platform")
|
||||
def test_init_mxfp4_linear_kernel_dispatches_to_registered_kernel(platform_mock):
|
||||
"""init_mxfp4_linear_kernel should select a registered kernel that
|
||||
reports itself as supported, and construct it with a fresh config.
|
||||
"""
|
||||
reports itself as supported, and construct it with a fresh config."""
|
||||
platform_mock._enum = PlatformEnum.OOT
|
||||
register_linear_kernel(OOTMxFp4LinearKernel, PlatformEnum.OOT, "mxfp4")
|
||||
|
||||
|
||||
@@ -16,8 +16,7 @@ import torch
|
||||
|
||||
class TestTritonMoeForwardExpertMap:
|
||||
"""Test that triton_kernel_moe_forward applies expert_map remapping
|
||||
when expert_map is provided (EP active).
|
||||
"""
|
||||
when expert_map is provided (EP active)."""
|
||||
|
||||
def test_expert_map_remap(self):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
|
||||
@@ -41,8 +41,7 @@ def test_can_implement_is_abstract():
|
||||
|
||||
def test_emulation_kernel_rejects_non_mxfp6_weights():
|
||||
"""EmulationMxfp6LinearKernel must not implement a non-MXFP6 weight
|
||||
format.
|
||||
"""
|
||||
format."""
|
||||
config = MxFp6LinearLayerConfig(weight_quant_key=kMxfp4Static)
|
||||
can_implement, reason = EmulationMxfp6LinearKernel.can_implement(config)
|
||||
assert not can_implement
|
||||
@@ -58,8 +57,7 @@ def test_emulation_kernel_accepts_any_supported_config(
|
||||
weight_quant_key, activation_quant_key
|
||||
):
|
||||
"""EmulationMxfp6LinearKernel is the only backend today: it must accept
|
||||
every supported weight/activation format combination.
|
||||
"""
|
||||
every supported weight/activation format combination."""
|
||||
config = MxFp6LinearLayerConfig(
|
||||
weight_quant_key=weight_quant_key, activation_quant_key=activation_quant_key
|
||||
)
|
||||
@@ -104,8 +102,7 @@ class OOTMxFp6LinearKernel(MxFp6LinearKernel):
|
||||
def test_init_mxfp6_linear_kernel_dispatches_to_registered_kernel(platform_mock):
|
||||
"""init_mxfp6_linear_kernel should select a registered kernel that
|
||||
reports itself as supported/able to implement the given config, and
|
||||
construct it with that exact config.
|
||||
"""
|
||||
construct it with that exact config."""
|
||||
platform_mock._enum = PlatformEnum.OOT
|
||||
register_linear_kernel(OOTMxFp6LinearKernel, PlatformEnum.OOT, "mxfp6")
|
||||
|
||||
|
||||
@@ -118,8 +118,7 @@ def test_per_token_group_quant_fp8_packed(
|
||||
num_tokens, hidden_dim, group_size, poisoned_scales
|
||||
):
|
||||
"""Test the packed DeepGEMM quantization kernel against the Triton
|
||||
reference (row-major, UE8M0 scales).
|
||||
"""
|
||||
reference (row-major, UE8M0 scales)."""
|
||||
device = "cuda"
|
||||
torch.manual_seed(42)
|
||||
|
||||
@@ -257,8 +256,7 @@ def test_per_token_group_quant_fp8_packed_all_zero():
|
||||
def test_per_token_group_quant_fp8_packed_mantissa_rounds_up():
|
||||
"""Inputs whose absmax/max_8bit produces a non-power-of-2 force the
|
||||
mantissa-rounding-up branch (exp_byte += 1). Locks down this behavior
|
||||
before optimization.
|
||||
"""
|
||||
before optimization."""
|
||||
device = "cuda"
|
||||
num_tokens, hidden_dim, group_size = 4, 7168, 128
|
||||
|
||||
@@ -330,8 +328,7 @@ def test_per_token_group_quant_fp8_packed_zero_fills_padded_output_q(
|
||||
):
|
||||
"""When output_q is allocated with shape (tma_aligned_mn, k) instead of
|
||||
(mn, k), the kernel must overwrite the padded mn rows with zeros so
|
||||
callers can use ``torch.empty`` instead of ``torch.zeros``.
|
||||
"""
|
||||
callers can use ``torch.empty`` instead of ``torch.zeros``."""
|
||||
device = "cuda"
|
||||
group_size = 128
|
||||
torch.manual_seed(42)
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Triton dequant-gather kernel used by
|
||||
``CompressedTensorsEmbeddingWNA16Int`` (quantized embedding lookup).
|
||||
"""
|
||||
``CompressedTensorsEmbeddingWNA16Int`` (quantized embedding lookup)."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -29,8 +29,7 @@ pytestmark = pytest.mark.skipif(
|
||||
|
||||
def test_rocm_aiter_group_fp8_quant_fake_implementation():
|
||||
"""Test that the fake implementation is correctly
|
||||
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant.
|
||||
"""
|
||||
defined for torch.ops.vllm.rocm_aiter_group_fp8_quant."""
|
||||
# Create test tensors
|
||||
M = 128
|
||||
N = 4096
|
||||
@@ -50,8 +49,7 @@ def test_rocm_aiter_group_fp8_quant_fake_implementation():
|
||||
def test_rocm_aiter_group_fp8_quant_torch_compile_with_cudagraph():
|
||||
"""Test that rocm_aiter_ops.group_fp8_quant
|
||||
with group size 128 can be used with
|
||||
torch.compile in cudagraph mode.
|
||||
"""
|
||||
torch.compile in cudagraph mode."""
|
||||
# Create test tensors
|
||||
M = 128
|
||||
N = 4096
|
||||
|
||||
@@ -209,8 +209,7 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
@pytest.mark.parametrize("shape", LLAMA_SHAPES)
|
||||
def test_vllm_quant_dequant_mxfp4_matches_reference_on_large_shapes(shape, dtype):
|
||||
"""The public vLLM MXFP4 QDQ helper should stay exact against the torch
|
||||
reference even on larger Llama-like shapes.
|
||||
"""
|
||||
reference even on larger Llama-like shapes."""
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
quant_dequant_mxfp4,
|
||||
)
|
||||
|
||||
@@ -670,8 +670,7 @@ def test_v41_rope_insert_plain_row(compress_ratio: int, store_fp8: bool):
|
||||
def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
"""The ring group's generic slot mapping is disabled (all PAD), so the
|
||||
builder must map every real token to ``ring_block * capacity + pos %
|
||||
capacity`` and keep padding tokens at PAD.
|
||||
"""
|
||||
capacity`` and keep padding tokens at PAD."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
|
||||
@@ -1096,8 +1095,7 @@ def test_get_c128_boundary(starts, query_start_loc, expected):
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_deepseek_v4_attention_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
"""compressed_kv → quantize_and_insert_k_cache → dequantize_and_gather_k_cache
|
||||
→ compare against original.
|
||||
"""
|
||||
→ compare against original."""
|
||||
HEAD_DIM = 512
|
||||
NOPE_DIM = 448
|
||||
HEAD_BYTES = 584 # 448 fp8 + 128 bf16 + 8 uint8 scale
|
||||
@@ -1309,8 +1307,7 @@ def test_dequantize_and_gather_k_cache(
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
"""K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
|
||||
→ manual dequant → compare against original.
|
||||
"""
|
||||
→ manual dequant → compare against original."""
|
||||
HEAD_DIM = 128
|
||||
QUANT_BLOCK_SIZE = 128
|
||||
# cache_stride = head_dim + (head_dim * 4 / quant_block_size) = 128 + 4 = 132
|
||||
|
||||
@@ -432,8 +432,7 @@ def test_cp_gather_fp8_with_sequence_starts(gather_seq_lens, seq_starts):
|
||||
)
|
||||
def test_cp_gather_fp8_large_seqlens(seq_lens, block_size):
|
||||
"""Correctness test with large sequence lengths matching benchmark
|
||||
scenarios (8K-128K prefill).
|
||||
"""
|
||||
scenarios (8K-128K prefill)."""
|
||||
(
|
||||
cache,
|
||||
block_table,
|
||||
|
||||
@@ -85,8 +85,7 @@ def _decode_token_regions(
|
||||
cache: torch.Tensor, slot_mapping: torch.Tensor, block_size: int
|
||||
):
|
||||
"""Slice out, per valid token, the raw NoPE-fp8 / RoPE-bf16 / scale byte
|
||||
regions this layout defines.
|
||||
"""
|
||||
regions this layout defines."""
|
||||
valid = slot_mapping >= 0
|
||||
idx = valid.nonzero(as_tuple=True)[0]
|
||||
slots = slot_mapping[idx]
|
||||
@@ -111,8 +110,7 @@ def _assert_cache_parity(cache_fused, cache_ref, slot_mapping, block_size):
|
||||
"""NoPE fp8 + the UE8M0 scale byte are a deterministic quant of an
|
||||
un-rotated bf16 value, so they match tightly; the RoPE region is rotated
|
||||
in fp32 before its bf16 store and eager/kernel can round a tie to
|
||||
opposite sides, so allow <=1 ULP there.
|
||||
"""
|
||||
opposite sides, so allow <=1 ULP there."""
|
||||
nope_ref, rope_ref, scale_ref = _decode_token_regions(
|
||||
cache_ref, slot_mapping, block_size
|
||||
)
|
||||
@@ -192,8 +190,7 @@ def _dup_padded_block_cache(
|
||||
num_blocks: int, block_size: int, width: int, pad_elems: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Two independently-stored padded caches with identical values, for ops
|
||||
that mutate the cache in place.
|
||||
"""
|
||||
that mutate the cache in place."""
|
||||
page_stride = block_size * width + pad_elems
|
||||
template = torch.randn(num_blocks, page_stride, dtype=torch.float32)
|
||||
|
||||
@@ -210,8 +207,7 @@ def _run_save_partial_states_eager(
|
||||
):
|
||||
"""Reference: state_cache[block_idx, pos_in_block, :W] = kv;
|
||||
state_cache[..., W:] = score + ape[position % compress_ratio]; skip
|
||||
slot < 0. Mutates state_cache in place, matching the op's contract.
|
||||
"""
|
||||
slot < 0. Mutates state_cache in place, matching the op's contract."""
|
||||
state_width = kv.shape[-1]
|
||||
block_size = state_cache.shape[1]
|
||||
valid = slot_mapping >= 0
|
||||
@@ -271,8 +267,7 @@ def _run_compress_eager(
|
||||
):
|
||||
"""Reference for the per-channel-softmax-pooling + RMSNorm + NoPE-UE8M0
|
||||
quant + GPT-J RoPE math writing the main (head_dim=512) fp8_ds_mla
|
||||
layout.
|
||||
"""
|
||||
layout."""
|
||||
num_tokens, window = gather_slots.shape
|
||||
state_width = state_cache.shape[-1] // 2
|
||||
state_block_size = state_cache.shape[1]
|
||||
@@ -449,8 +444,7 @@ def test_compress_norm_rope_store_cpu_matches_eager(
|
||||
|
||||
def test_compress_norm_rope_store_cpu_non_contiguous_kv_cache_row_stride():
|
||||
"""The main (output) kv_cache is a separate physical buffer from the
|
||||
state cache and can independently be a strided view.
|
||||
"""
|
||||
state cache and can independently be a strided view."""
|
||||
eps = 1e-6
|
||||
compress_ratio, overlap = 128, False
|
||||
num_tokens = 5
|
||||
@@ -526,8 +520,7 @@ def _run_compress_indexer_eager(
|
||||
"""Same softmax-pool + RMSNorm + GPT-J RoPE prologue as the head=512
|
||||
path, but the indexer's cache quantizes all 128 post-RoPE dims as one
|
||||
FP8 block with a single raw fp32 scale per token (matching
|
||||
_indexer_k_quant_and_cache_cpu's layout), not 7 separate UE8M0 blocks.
|
||||
"""
|
||||
_indexer_k_quant_and_cache_cpu's layout), not 7 separate UE8M0 blocks."""
|
||||
num_tokens, window = gather_slots.shape
|
||||
state_width = state_cache.shape[-1] // 2
|
||||
state_block_size = state_cache.shape[1]
|
||||
@@ -655,8 +648,7 @@ def _pack_flash_mla_cache(kv_f32: torch.Tensor, block_size: int):
|
||||
"""Quantize kv_f32 ([N, HEAD_DIM]) into the fp8_ds_mla byte layout and
|
||||
return (cache, dequant_ref) -- the exact post-quantization value the
|
||||
kernel's dequant should reproduce. Slot id == row index, packed
|
||||
contiguously from block 0.
|
||||
"""
|
||||
contiguously from block 0."""
|
||||
n = kv_f32.shape[0]
|
||||
num_blocks = max(1, -(-n // block_size))
|
||||
cache = _make_cache(num_blocks, block_size)
|
||||
@@ -714,8 +706,7 @@ def _flash_mla_eager(
|
||||
"""MQA attention: gather the (already-dequantized) window+compressed KV
|
||||
rows referenced by *_slots (-1 = invalid/masked), append a learned
|
||||
per-head sink logit with zero value contribution, softmax, weighted
|
||||
sum.
|
||||
"""
|
||||
sum."""
|
||||
num_tokens, num_heads, _ = q.shape
|
||||
q_f32 = q.float()
|
||||
|
||||
@@ -814,8 +805,7 @@ def test_flash_mla_cpu_kernel_matches_eager(
|
||||
|
||||
def test_flash_mla_cpu_kernel_all_invalid_falls_back_to_sink():
|
||||
"""When every KV slot is masked out, softmax degenerates to the sink
|
||||
logit alone and the output must be all-zero (no valid value rows).
|
||||
"""
|
||||
logit alone and the output must be all-zero (no valid value rows)."""
|
||||
torch.manual_seed(2)
|
||||
dtype = torch.bfloat16
|
||||
num_tokens, num_heads, num_window = 3, 4, 5
|
||||
@@ -855,8 +845,7 @@ def _qnorm_kv_insert_eager(
|
||||
q, kv, positions, cache, slot_mapping, cos_sin_cache, q_head_padded, eps, bs
|
||||
):
|
||||
"""Reference: per-head weight-free RMSNorm + GPT-J RoPE on Q; GPT-J RoPE
|
||||
+ UE8M0 FP8 quant + paged insert on KV.
|
||||
"""
|
||||
+ UE8M0 FP8 quant + paged insert on KV."""
|
||||
num_quant_blocks = NOPE_DIM // QUANT_BLOCK
|
||||
|
||||
num_tokens_full, num_heads_q, _ = q.shape
|
||||
@@ -982,8 +971,7 @@ def test_qnorm_rope_kv_insert_cpu_kernel_matches_eager(
|
||||
@pytest.mark.parametrize("pad", [1, 5])
|
||||
def test_qnorm_rope_kv_insert_cpu_kernel_dp_padding(pad: int):
|
||||
"""slot_mapping shorter than q/kv rows: the KV branch must only touch
|
||||
the first num_tokens rows while Q-norm+RoPE still runs on all rows.
|
||||
"""
|
||||
the first num_tokens rows while Q-norm+RoPE still runs on all rows."""
|
||||
torch.manual_seed(1)
|
||||
dtype = torch.bfloat16
|
||||
eps = 1e-6
|
||||
@@ -1044,8 +1032,7 @@ def _inverse_gptj_rope_o_proj_eager(
|
||||
rope_dim: int,
|
||||
) -> torch.Tensor:
|
||||
"""NoPE dims pass through as fp32; the RoPE segment is de-rotated by
|
||||
R(-pos) in GPT-J (interleaved even/odd) convention.
|
||||
"""
|
||||
R(-pos) in GPT-J (interleaved even/odd) convention."""
|
||||
nope_dim = o.shape[-1] - rope_dim
|
||||
half_dim = rope_dim // 2
|
||||
|
||||
@@ -1111,8 +1098,7 @@ def _write_page(
|
||||
) -> None:
|
||||
"""Write k_fp8 ([block_size, INDEXER_HEAD_DIM]) and scale ([block_size]
|
||||
fp32) into one page row: K-region then scale-region, matching
|
||||
compress_norm_rope_store_indexer_cpu's layout.
|
||||
"""
|
||||
compress_norm_rope_store_indexer_cpu's layout."""
|
||||
k_region = page_row[: block_size * INDEXER_HEAD_DIM]
|
||||
k_region.copy_(k_fp8.view(torch.uint8).flatten())
|
||||
scale_region = page_row[block_size * INDEXER_HEAD_DIM :].view(torch.float32)
|
||||
@@ -1148,8 +1134,7 @@ def _build_paged_mqa_logits_batch(
|
||||
):
|
||||
"""Fill pages (a pre-sized [n_pages, buf_width] uint8 tensor, possibly
|
||||
non-contiguous in dim 0) with deterministic-per-seed random K/scale
|
||||
data.
|
||||
"""
|
||||
data."""
|
||||
torch.manual_seed(seed)
|
||||
batch_size = len(seq_lens)
|
||||
max_seq_len = max(seq_lens)
|
||||
@@ -1215,8 +1200,7 @@ def test_fp8_paged_mqa_logits_cpu_non_contiguous_kv_cache_row_stride():
|
||||
"""The paged K-cache in production is a per-layer view into a shared
|
||||
multi-layer allocation, so its page stride generally exceeds the page
|
||||
byte width -- check the kernel reads a padded/strided buffer identically
|
||||
to an equivalent tightly-packed one.
|
||||
"""
|
||||
to an equivalent tightly-packed one."""
|
||||
block_size = 64
|
||||
num_heads = 4
|
||||
seq_lens = [1, block_size + 7, 3 * block_size]
|
||||
@@ -1342,8 +1326,7 @@ def test_fused_indexer_q_rope_quant_dispatches_to_cpu_kernel():
|
||||
CPU op on a CPU tensor, not silently fall through to the Triton kernel
|
||||
(which would also "work" under triton-cpu, masking a dispatch
|
||||
regression). The dispatcher does its CPU import inside the CPU branch,
|
||||
so the patch target is the source module.
|
||||
"""
|
||||
so the patch target is the source module."""
|
||||
torch.manual_seed(0)
|
||||
num_tokens = 1
|
||||
|
||||
@@ -1410,8 +1393,7 @@ def _hc_head_torch(
|
||||
) -> torch.Tensor:
|
||||
"""Eager reference for hc_head_fused_cpu: round-trips the RMSNorm
|
||||
output through bf16 before the linear, matching the kernel's bf16
|
||||
arithmetic.
|
||||
"""
|
||||
arithmetic."""
|
||||
hc_mult, hidden_size = hidden_states.shape[-2:]
|
||||
outer_shape = hidden_states.shape[:-2]
|
||||
hs_flat = hidden_states.reshape(-1, hc_mult, hidden_size)
|
||||
@@ -1493,8 +1475,7 @@ def test_mhc_post_cpu(num_tokens: int, hidden_size: int):
|
||||
@pytest.mark.parametrize("hidden_size", [4096, 7168])
|
||||
def test_hc_head_cpu(num_tokens: int, hidden_size: int):
|
||||
"""hc_head_fused_cpu vs. the _hc_head_torch eager reference it
|
||||
replaces.
|
||||
"""
|
||||
replaces."""
|
||||
torch.set_default_device(DEVICE)
|
||||
set_random_seed(0)
|
||||
hc_mult = 4
|
||||
|
||||
@@ -383,8 +383,7 @@ def _hash_ids(requests, tables_meta, block_table, cache=None, capture=False):
|
||||
"""Run the op on a batch of (token_ids, start, window) requests; `window`
|
||||
is the runner's [depth] lookback for that request (None = all unknown).
|
||||
Token IDs 14 and 22 stand in for image tokens that break n-grams.
|
||||
Returns one [len(token_ids), ...] hash tensor per request.
|
||||
"""
|
||||
Returns one [len(token_ids), ...] hash tensor per request."""
|
||||
token_map, multipliers, primes, offsets, block_size = tables_meta
|
||||
depth = multipliers.shape[1] - 1
|
||||
|
||||
@@ -443,8 +442,7 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
|
||||
V2 supplies every lookback from its device token history and needs no
|
||||
slot cache. V1 supplies prompt positions only and reads generated
|
||||
positions from the slot cache it fills itself. Without any window the
|
||||
first decode token hashes stale slots (negative control).
|
||||
"""
|
||||
first decode token hashes stale slots (negative control)."""
|
||||
block_size = 4
|
||||
block_table = torch.tensor(
|
||||
[[3, 1, 5, 0], [2, 4, 6, 7]], dtype=torch.int32, device="cuda"
|
||||
@@ -710,8 +708,7 @@ def test_engram_lookup_reuses_jit_across_token_shapes():
|
||||
def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
|
||||
"""The fused gather must be bit-exact with the torch dequant path it
|
||||
replaces, from HBM and from pinned host memory alike, and must contribute
|
||||
zeros for rows another TP rank owns.
|
||||
"""
|
||||
zeros for rows another TP rank owns."""
|
||||
layer = _make_embedding(cpu_offload)
|
||||
cols, rows = 24, layer.part_num_embeddings
|
||||
ids = torch.randint(0, rows, (num_tokens, cols), dtype=torch.int32, device="cuda")
|
||||
|
||||
@@ -336,8 +336,7 @@ def test_layer_norm_rows_per_block(
|
||||
@torch.inference_mode()
|
||||
def test_strided_input(dtype: torch.dtype) -> None:
|
||||
"""Test that the kernel handles non-contiguous (strided)
|
||||
inputs correctly.
|
||||
"""
|
||||
inputs correctly."""
|
||||
set_random_seed(42)
|
||||
device = torch.device(DEVICE)
|
||||
num_tokens = 128
|
||||
|
||||
@@ -97,8 +97,7 @@ def test_topk_routing_consistency(num_tokens: int, hidden_dim: int, num_experts:
|
||||
"""The gate feeds top-k expert selection: the kernel's top-8 must match
|
||||
an fp64 reference's top-8 per token (ties tolerated). This is the
|
||||
business-level correctness of the router — numeric error only matters
|
||||
if it flips the argsort.
|
||||
"""
|
||||
if it flips the argsort."""
|
||||
_requires_sm90()
|
||||
top_k = 8
|
||||
device = torch.device("cuda")
|
||||
|
||||
@@ -854,8 +854,7 @@ def test_fused_q_no_indexer(num_tokens: int):
|
||||
@pytest.mark.parametrize("has_indexer", [True, False])
|
||||
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool):
|
||||
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
|
||||
produced (bf16, unquantized); ql_nope is consumed directly by the caller.
|
||||
"""
|
||||
produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
|
||||
torch.manual_seed(6)
|
||||
dev = "cuda"
|
||||
max_pos = 8192
|
||||
|
||||
@@ -176,8 +176,7 @@ def _call_fused(
|
||||
|
||||
def _as_stored_fp8(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Reinterpret a float8_e4m3fn-typed kernel output under the real (FNUZ on
|
||||
gfx942) encoding the kernel actually wrote, without touching the bytes.
|
||||
"""
|
||||
gfx942) encoding the kernel actually wrote, without touching the bytes."""
|
||||
return t.contiguous().view(torch.uint8).view(FP8_STORE_DTYPE)
|
||||
|
||||
|
||||
@@ -713,8 +712,7 @@ def test_q_interleaved_is_orthogonal_to_norm_and_rope(
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_kv_path_with_dp_padding(num_tokens: int, pad: int, block_size: int):
|
||||
"""slot_mapping.size(0) < q.size(0): the kernel must skip padded
|
||||
tokens in the KV branch while still processing Q on all rows.
|
||||
"""
|
||||
tokens in the KV branch while still processing Q on all rows."""
|
||||
torch.manual_seed(3)
|
||||
device = "cuda"
|
||||
dtype = torch.bfloat16
|
||||
|
||||
@@ -574,8 +574,7 @@ def test_indexer_k_store_roundtrips_through_rocm_gather(block_size, compress_rat
|
||||
def test_fused_indexer_q_rope_quant_writes_bf16_weights(use_cutedsl):
|
||||
"""The MXFP4 path can emit the per-head weights in bf16 (what DeepGEMM's
|
||||
sparse MQA-logits kernels take) instead of fp32; the values are the fp32
|
||||
result rounded once, so the scoring kernel needs no cast.
|
||||
"""
|
||||
result rounded once, so the scoring kernel needs no cast."""
|
||||
if use_cutedsl and not has_cutedsl():
|
||||
pytest.skip("cutedsl (cutlass) not installed")
|
||||
|
||||
|
||||
@@ -446,8 +446,7 @@ def test_output_strides(num_tokens, num_heads, n_groups):
|
||||
def test_per_group_contiguity(num_tokens):
|
||||
"""FP8 per-group slices must be contiguous. Scale per-group slices
|
||||
are column-major (T-stride=1) — not row-major contiguous, which is
|
||||
correct for TMA loads.
|
||||
"""
|
||||
correct for TMA loads."""
|
||||
num_heads, n_groups = 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
@@ -513,8 +512,7 @@ def test_scales_are_power_of_two():
|
||||
def test_nope_dims_unchanged():
|
||||
"""Nope dimensions (first 448 per head) should only be quantized,
|
||||
not rotated. Verify by dequantizing and comparing against
|
||||
quantize-only reference (no RoPE).
|
||||
"""
|
||||
quantize-only reference (no RoPE)."""
|
||||
num_tokens, num_heads, n_groups = 16, 64, 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
max_pos = 4096
|
||||
@@ -905,8 +903,7 @@ def test_einsum_end_to_end(num_tokens, num_heads, n_groups):
|
||||
@torch.inference_mode()
|
||||
def test_with_real_deepseek_v4_rope(num_tokens, default_vllm_config):
|
||||
"""Test with real DeepseekV4ScalingRotaryEmbedding (GPT-J style,
|
||||
mscale=0, YaRN scaling) matching the production config.
|
||||
"""
|
||||
mscale=0, YaRN scaling) matching the production config."""
|
||||
num_heads = 64
|
||||
n_groups = 8
|
||||
heads_per_group = num_heads // n_groups
|
||||
|
||||
@@ -1372,8 +1372,7 @@ def _patch_first_rank_pp_group(monkeypatch):
|
||||
|
||||
def test_deepseek_v4_mhc_broadcast_finalize_sums_hc_streams(monkeypatch):
|
||||
"""First finalize (at the end of load_weights) allocates
|
||||
hc_attn_fn_broadcast as hc_attn_fn summed over hc streams.
|
||||
"""
|
||||
hc_attn_fn_broadcast as hc_attn_fn summed over hc streams."""
|
||||
_patch_first_rank_pp_group(monkeypatch)
|
||||
layer = _make_mhc_decoder_layer(hc_mult=2, hidden_size=8)
|
||||
model = SimpleNamespace(start_layer=0, end_layer=1, layers=[layer])
|
||||
@@ -1388,8 +1387,7 @@ def test_deepseek_v4_mhc_broadcast_finalize_sums_hc_streams(monkeypatch):
|
||||
def test_deepseek_v4_mhc_broadcast_refit_refreshes_in_place(monkeypatch):
|
||||
"""Re-finalizing after a weight refit must copy into the existing
|
||||
broadcast tensor so its address stays stable for captured CUDA graphs,
|
||||
while picking up the new hc_attn_fn values.
|
||||
"""
|
||||
while picking up the new hc_attn_fn values."""
|
||||
_patch_first_rank_pp_group(monkeypatch)
|
||||
layer = _make_mhc_decoder_layer(hc_mult=2, hidden_size=8)
|
||||
model = SimpleNamespace(start_layer=0, end_layer=1, layers=[layer])
|
||||
|
||||
@@ -403,8 +403,7 @@ def test_mxfp8_linear_emulation_bf16_at_load(
|
||||
):
|
||||
"""EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
|
||||
``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
|
||||
same result; the dtype-match (BF16/FP16 activations) must also hold.
|
||||
"""
|
||||
same result; the dtype-match (BF16/FP16 activations) must also hold."""
|
||||
from vllm.model_executor.kernels.linear.mxfp8.emulation import (
|
||||
EmulationMxfp8LinearKernel,
|
||||
)
|
||||
@@ -518,8 +517,7 @@ def test_mxfp8_rocm_native_unaligned_k_dequantizes_at_load(shape):
|
||||
# produced EP garbage).
|
||||
def _capture_expert_mask(expert_mask, *, global_num_experts):
|
||||
"""Drive the real ``AiterMxfp8Experts.apply`` mask branch and capture the
|
||||
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``.
|
||||
"""
|
||||
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``."""
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
@@ -570,8 +568,7 @@ def _capture_expert_mask(expert_mask, *, global_num_experts):
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
|
||||
"""AiterMxfp8Experts.apply forwards the precomputed 0/1 mask to aiter as-is
|
||||
(no re-derivation that would collapse an already-0/1 mask to all-ones).
|
||||
"""
|
||||
(no re-derivation that would collapse an already-0/1 mask to all-ones)."""
|
||||
# 0/1 local-expert mask over global ids + trailing sentinel (rank owns 0..3).
|
||||
ep_mask = torch.tensor(
|
||||
[1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32, device=DEVICE
|
||||
@@ -585,8 +582,7 @@ def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
|
||||
def test_routed_experts_expert_map_delegates_to_kernel():
|
||||
"""RoutedExperts.expert_map returns the 0/1 mask only for kernels that set
|
||||
``consumes_expert_mask`` (AITER), and the canonical -1 map otherwise -- keyed
|
||||
on the resolved kernel, not the global aiter switch.
|
||||
"""
|
||||
on the resolved kernel, not the global aiter switch."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
|
||||
@@ -17,8 +17,7 @@ from vllm.platforms import current_platform
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
def test_shuffle_rows_basic(num_tokens: int, hidden_size: int, dtype: torch.dtype):
|
||||
"""Test basic functionality of shuffle_rows with various tensor sizes and
|
||||
dtypes.
|
||||
"""
|
||||
dtypes."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("shuffle_rows requires CUDA")
|
||||
|
||||
@@ -73,8 +72,7 @@ def test_shuffle_rows_permutation(
|
||||
@pytest.mark.parametrize("hidden_size", [256, 512])
|
||||
def test_shuffle_rows_expansion(num_tokens: int, hidden_size: int):
|
||||
"""Test shuffle_rows with expansion (more output tokens than input
|
||||
tokens).
|
||||
"""
|
||||
tokens)."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("shuffle_rows requires CUDA")
|
||||
|
||||
|
||||
@@ -1740,8 +1740,7 @@ def test_sparse_indexer_decode_topk_explicit_backends(
|
||||
) -> None:
|
||||
"""Every explicit sparse-indexer decode top-k backend must restrict each
|
||||
row to its seq_len (dirty data past the end must never be selected) and
|
||||
match torch.topk on the valid region.
|
||||
"""
|
||||
match torch.topk on the valid region."""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
@@ -1812,8 +1811,7 @@ def test_sparse_indexer_decode_topk_short_seq_lens(
|
||||
backend: str, workspace_init
|
||||
) -> None:
|
||||
"""seq_len < next_n: derived per-row ends must clamp at 0 (the reference
|
||||
kernels do max(0, ...)); negative ends are OOB for the ragged kernels.
|
||||
"""
|
||||
kernels do max(0, ...)); negative ends are OOB for the ragged kernels."""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
|
||||
Reference in New Issue
Block a user