Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.flash_attn import cascade_attention, merge_attn_states
try:
from vllm.vllm_flash_attn import (
fa_version_unsupported_reason,
flash_attn_varlen_func,
is_fa_version_supported,
)
except ImportError:
if current_platform.is_rocm():
pytest.skip(
"vllm_flash_attn is not supported for vLLM on ROCm.",
allow_module_level=True,
)
NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
HEAD_SIZES = [128, 192, 256]
BLOCK_SIZES = [16]
DTYPES = [torch.float16, torch.bfloat16]
@pytest.mark.parametrize("num_tokens", [1, 39, 16912])
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@torch.inference_mode()
def test_merge_kernel(
num_tokens: int,
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
):
torch.set_default_device("cuda")
set_random_seed(0)
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
# Prepare inputs.
prefix_output = torch.randn(num_tokens, num_query_heads, head_size, dtype=dtype)
suffix_output = torch.randn(num_tokens, num_query_heads, head_size, dtype=dtype)
prefix_lse = torch.randn(num_query_heads, num_tokens, dtype=torch.float32)
suffix_lse = torch.randn(num_query_heads, num_tokens, dtype=torch.float32)
# Run the kernel.
output = torch.empty(num_tokens, num_query_heads, head_size, dtype=dtype)
merge_attn_states(output, prefix_output, prefix_lse, suffix_output, suffix_lse)
# Reference implementation.
max_lse = torch.maximum(prefix_lse, suffix_lse)
p_lse = torch.exp(prefix_lse - max_lse)
s_lse = torch.exp(suffix_lse - max_lse)
p_scale = p_lse / (p_lse + s_lse)
s_scale = s_lse / (p_lse + s_lse)
p_scale = p_scale.transpose(0, 1).unsqueeze(2)
s_scale = s_scale.transpose(0, 1).unsqueeze(2)
ref_output = p_scale * prefix_output + s_scale * suffix_output
ref_output = ref_output.to(dtype)
# Compare the results.
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
CASES = [
# Case 1. A general case.
([(129, 871), (18, 280), (37, 988), (1023, 2304), (1, 257)], 256),
# Case 2. Flash-decoding case.
([(1, 1023), (1, 879), (1, 778), (1, 1777)] * 100, 512),
]
@pytest.mark.parametrize("seq_lens_and_common_prefix", CASES)
@pytest.mark.parametrize("num_heads", NUM_HEADS)
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
@pytest.mark.parametrize("soft_cap", [None, 50])
@pytest.mark.parametrize("num_blocks", [2048])
@pytest.mark.parametrize("fa_version", [2, 3])
@torch.inference_mode()
def test_cascade(
seq_lens_and_common_prefix: tuple[list[tuple[int, int]], int],
num_heads: tuple[int, int],
head_size: int,
dtype: torch.dtype,
block_size: int,
soft_cap: float | None,
num_blocks: int,
fa_version: int,
) -> None:
torch.set_default_device("cuda")
if not is_fa_version_supported(fa_version):
pytest.skip(
f"Flash attention version {fa_version} not supported due "
f'to: "{fa_version_unsupported_reason(fa_version)}"'
)
set_random_seed(0)
window_size = (-1, -1)
scale = head_size**-0.5
num_query_heads = num_heads[0]
num_kv_heads = num_heads[1]
assert num_query_heads % num_kv_heads == 0
key_cache = torch.randn(
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype
)
value_cache = torch.randn_like(key_cache)
seq_lens, common_prefix_len = seq_lens_and_common_prefix
num_seqs = len(seq_lens)
query_lens = [x[0] for x in seq_lens]
kv_lens = [x[1] for x in seq_lens]
max_query_len = max(query_lens)
max_kv_len = max(kv_lens)
total_num_query_tokens = sum(query_lens)
query = torch.randn(total_num_query_tokens, num_query_heads, head_size, dtype=dtype)
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
dim=0, dtype=torch.int32
)
kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32)
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
block_tables = torch.randint(
0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
)
assert common_prefix_len > 0
assert common_prefix_len % block_size == 0
num_common_kv_blocks = common_prefix_len // block_size
# Make sure the first `num_common_kv_blocks` blocks are the same.
block_tables[:, :num_common_kv_blocks] = block_tables[0, :num_common_kv_blocks]
# Run the regular attention.
ref_output = flash_attn_varlen_func(
q=query,
k=key_cache,
v=value_cache,
cu_seqlens_q=cu_query_lens,
seqused_k=kv_lens_tensor,
max_seqlen_q=max_query_len,
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=list(window_size),
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
)
# Run cascade attention.
assert all(common_prefix_len < kv_len for kv_len in kv_lens)
cu_prefix_query_lens = torch.tensor([0, total_num_query_tokens], dtype=torch.int32)
prefix_kv_lens = torch.tensor([common_prefix_len], dtype=torch.int32)
suffix_kv_lens = kv_lens_tensor - common_prefix_len
output = torch.empty_like(query)
cascade_attention(
output=output,
query=query,
key_cache=key_cache,
value_cache=value_cache,
cu_query_lens=cu_query_lens,
max_query_len=max_query_len,
cu_prefix_query_lens=cu_prefix_query_lens,
prefix_kv_lens=prefix_kv_lens,
suffix_kv_lens=suffix_kv_lens,
max_kv_len=max_kv_len,
softmax_scale=scale,
alibi_slopes=None,
sliding_window=window_size,
logits_soft_cap=soft_cap if soft_cap is not None else 0,
block_table=block_tables,
common_prefix_len=common_prefix_len,
max_num_splits=0, # no max
fa_version=fa_version,
)
# Compare the results.
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
@@ -5,6 +5,7 @@ import random
import pytest
import torch
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_ceil_to_ue8m0,
@@ -13,18 +14,29 @@ from vllm.utils.deep_gemm import (
fp8_fp4_paged_mqa_logits,
get_num_sms,
get_paged_mqa_logits_metadata,
mqa_logits_mx_inputs,
mqa_logits_use_mx_scales,
native_next_n_supported,
)
from vllm.utils.import_utils import has_deep_gemm
from vllm.utils.math_utils import cdiv
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
if not mqa_logits_use_mx_scales():
return sf
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
# x: (num_blocks, block_size, 1, head_dim)
num_blocks, block_size, num_heads, head_dim = x.shape
assert num_heads == 1
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
sf = x_amax / 448.0
sf = _ceil_to_ue8m0(x_amax / 448.0)
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
x_fp8 = torch.empty(
(num_blocks, block_size * (head_dim + 4)),
@@ -34,9 +46,9 @@ def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
x_fp8[:, : block_size * head_dim] = x_scaled.view(
num_blocks, block_size * head_dim
).view(dtype=torch.uint8)
x_fp8[:, block_size * head_dim :] = sf.view(num_blocks, block_size).view(
dtype=torch.uint8
)
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
sf.view(num_blocks, block_size)
).view(dtype=torch.uint8)
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
@@ -91,64 +103,72 @@ def _ref_fp8_mqa_logits(
return logits
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
# Two causal prefill requests packed into one KV buffer, so the second
# request's rows start at a nonzero K position.
half, half_kv = seq_len // 2, seq_len_kv // 2
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
ke = ks + half_kv - half + 1 + rows % half
return ks, ke
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
@pytest.mark.parametrize("clean_logits", [True, False])
def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
def test_deepgemm_fp8_mqa_logits(layout: str):
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
torch.manual_seed(0)
random.seed(0)
num_heads, head_dim = 32, 128
for seq_len in (512,):
for seq_len_kv in (1024,):
for disable_cp in (False, True):
q = torch.randn(
seq_len,
num_heads,
head_dim,
device="cuda",
dtype=torch.bfloat16,
)
kv = torch.randn(
seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16
)
weights = torch.randn(
seq_len, num_heads, device="cuda", dtype=torch.float32
)
seq_len, seq_len_kv = 512, 1024
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
if disable_cp:
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
seq_len_kv - seq_len
)
else:
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
if layout == "cp":
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
elif layout == "no_cp":
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
seq_len_kv - seq_len
)
else:
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
logits = fp8_fp4_mqa_logits(
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
)
q_fp8 = q.to(torch.float8_e4m3fn)
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
max_seqlen_k = int((ke - ks).max())
logits = fp8_fp4_mqa_logits(
(q_fp8, q_scale),
(kv_fp8, encode_indexer_scales(kv_sf)),
kernel_weights,
ks,
ke,
max_seqlen_k,
)
assert logits.shape == (seq_len, max_seqlen_k)
logits = as_fp32_logits(logits, ke - ks)
ref_logits = _ref_fp8_mqa_logits(
q=q,
kv=kv,
weights=weights,
cu_seqlen_ks=ks,
cu_seqlen_ke=ke,
)
ref_neginf_mask = ref_logits == float("-inf")
if clean_logits:
neginf_mask = logits == float("-inf")
assert torch.equal(neginf_mask, ref_neginf_mask)
ref_logits = ref_logits.masked_fill(ref_neginf_mask, 0)
logits = logits.masked_fill(ref_neginf_mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
ref_logits = _ref_fp8_mqa_logits(
q=q,
kv=kv,
weights=kernel_weights.float(),
cu_seqlen_ks=ks,
cu_seqlen_ke=ke,
)
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
valid = cols < ke[:, None]
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
ref_logits = ref_logits.masked_fill(~valid, 0)
logits = logits.masked_fill(~valid, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
def _ref_fp8_fp4_paged_mqa_logits(
@@ -212,9 +232,6 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
if not native_next_n_supported(next_n):
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
# NOTE: clean_logits=True is incompatible with the 2D context_lens
# required by csrc/apis/attention.hpp; only the False path is exercised.
clean_logits = False
torch.manual_seed(0)
random.seed(0)
@@ -273,26 +290,42 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
context_lens_2d = (
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
).contiguous()
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
kernel_q = (q_fp8, q_scale)
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
indices = None
if mqa_logits_use_mx_scales():
# SM100 only takes varlen rows: one Q token each, tied to its
# request by `indices`.
assert q_scale is not None
kernel_q = (
q_fp8.flatten(0, 1)[:, None],
q_scale.flatten(0, 1)[:, None],
)
kernel_context_lens = context_lens_2d.view(-1, 1)
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
indices = indices.repeat_interleave(next_n)
schedule_metadata = get_paged_mqa_logits_metadata(
context_lens_2d,
blocksize,
get_num_sms(),
kernel_context_lens, blocksize, get_num_sms(), indices=indices
)
logits = fp8_fp4_paged_mqa_logits(
(q_fp8, None),
kernel_q,
kv_cache_fp8,
weights,
context_lens_2d,
block_tables,
kernel_weights,
kernel_context_lens,
kernel_block_tables,
schedule_metadata,
max_model_len,
clean_logits=clean_logits,
indices=indices,
)
if indices is not None:
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
q,
kv_cache,
weights,
kernel_weights.float(),
context_lens,
block_tables,
max_model_len,
@@ -309,7 +342,69 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
context_lens[row_indices] - next_n + next_n_offset
).unsqueeze(1)
logits = logits.masked_fill(~mask, 0)
logits = logits.float().masked_fill(~mask, 0)
ref_logits = ref_logits.masked_fill(~mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
def test_deepgemm_paged_mqa_packed_manager_block_stride():
torch.manual_seed(0)
num_blocks, block_size, packed_rows = 4, 256, 320
num_heads, head_dim, row_bytes = 64, 128, 132
page_size, pages_per_block = 64, block_size // 64
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
compact_pages = kv_cache_cast_to_fp8(
torch.randn(
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
)
packed_backing = torch.zeros(
num_blocks,
packed_rows,
row_bytes,
device="cuda",
dtype=torch.uint8,
)
packed_manager = packed_backing[:, :block_size]
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
tables = [
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
]
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
torch.float8_e4m3fn
)
q_scale, weights = mqa_logits_mx_inputs(
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
)
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
indices = (
torch.zeros(1, device="cuda", dtype=torch.int32)
if mqa_logits_use_mx_scales()
else None
)
schedule = get_paged_mqa_logits_metadata(
context_lens, page_size, get_num_sms(), indices=indices
)
logits = [
fp8_fp4_paged_mqa_logits(
(q, q_scale),
cache,
weights,
context_lens,
table,
schedule,
block_size,
indices=indices,
)
for cache, table in zip((compact_pages, packed_pages), tables)
]
torch.testing.assert_close(*logits, rtol=0, atol=0)
@@ -175,7 +175,15 @@ def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypat
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
)
out, lse = impl._run_mqa_kernel(query.squeeze(1), kv_cache, indices, counts)
out, lse = impl._forward_mqa_kernel(
query.squeeze(1),
kv_cache,
indices,
counts,
layer=None,
block_size=block_size,
is_decode=True,
)
reference = trtllm_batch_decode_with_kv_cache_mla(
query=query,
kv_cache=kv_cache.unsqueeze(1),
@@ -1833,6 +1833,35 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
assert reason is None or "AITER sparse PA attend" not in reason
def test_aiter_indexer_rejects_unbuilt_index_heads(monkeypatch):
"""AITER builds the MSA score kernels for 1 or 2 index heads only.
MiniMax-M3 has 4, so at attention TP=1 the indexer has to fall back at load
time rather than fail at the first forward.
"""
import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod
monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr(
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True
)
def unsupported_reason(num_index_heads: int) -> str | None:
return indexer_aiter_mod.aiter_indexer_unsupported_reason(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
num_index_heads=num_index_heads,
index_head_dim=HEAD_DIM,
indexer_kv_dtype="fp8",
max_model_len=8192,
)
if reason := unsupported_reason(2):
pytest.skip(f"AITER MSA indexer unavailable: {reason}")
reason = unsupported_reason(4)
assert reason is not None and "num_index_heads" in reason
def test_aiter_consolidated_qknorm_enabled(monkeypatch):
from vllm._aiter_ops import rocm_aiter_ops
@@ -65,30 +65,40 @@ SM_SCALE = HEAD_DIM**-0.5
"decode_query_len",
"num_q_heads",
"num_kv_heads",
"is_sm103",
"expected",
),
[
pytest.param(8, 4, 64, 4, False, id="tp1-below-min-batch"),
pytest.param(16, 4, 64, 4, True, id="tp1-supported"),
pytest.param(16, 4, 16, 1, True, id="tp4-min-batch"),
pytest.param(24, 4, 16, 1, True, id="tp4-intermediate-batch"),
pytest.param(32, 4, 16, 1, True, id="tp4-supported"),
pytest.param(16, 1, 64, 4, True, id="tp1-query-len-1"),
pytest.param(16, 1, 16, 1, True, id="tp4-query-len-1"),
pytest.param(16, 2, 64, 4, True, id="tp1-query-len-2"),
pytest.param(16, 2, 16, 1, True, id="tp4-query-len-2"),
pytest.param(16, 32, 64, 4, True, id="query-len-upper-bound"),
pytest.param(16, 0, 64, 4, False, id="query-len-zero"),
pytest.param(16, 33, 64, 4, False, id="query-len-above-bound"),
pytest.param(15, 1, 64, 4, True, False, id="qlen1-below-row-floor"),
pytest.param(16, 1, 64, 4, True, True, id="qlen1-at-row-floor"),
pytest.param(10, 3, 32, 2, True, False, id="qlen3-below-row-floor"),
pytest.param(11, 3, 32, 2, True, True, id="qlen3-at-row-floor"),
pytest.param(3, 4, 64, 4, True, False, id="qlen4-below-row-floor"),
pytest.param(4, 4, 64, 4, True, True, id="qlen4-at-row-floor"),
pytest.param(7, 8, 16, 1, True, False, id="qlen8-below-row-floor"),
pytest.param(8, 8, 16, 1, True, True, id="qlen8-at-row-floor"),
pytest.param(15, 9, 64, 4, True, False, id="unmeasured-query-length"),
pytest.param(15, 4, 8, 1, True, False, id="unmeasured-heads"),
pytest.param(15, 4, 64, 4, False, False, id="sm100-keeps-old-floor"),
pytest.param(16, 32, 64, 4, True, True, id="query-len-upper-bound"),
pytest.param(16, 0, 64, 4, True, False, id="query-len-zero"),
pytest.param(16, 33, 64, 4, True, False, id="query-len-above-bound"),
],
)
def test_msa_cutlass_decode_static_dispatch(
batch_size: int,
decode_query_len: int,
expected: bool,
num_q_heads: int,
num_kv_heads: int,
is_sm103: bool,
expected: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
current_platform,
"is_device_capability",
lambda capability: is_sm103 and capability == (10, 3),
)
assert (
should_prepare_decode_metadata(
batch_size,
@@ -291,6 +301,7 @@ def test_msa_metadata_builder_prepares_cutlass_for_regular_decode(
builder.kv_cache_dtype = "fp8_e4m3"
builder.decode_backend = "cutlass"
builder.msa_cutlass_plan_cache = object()
builder.msa_cutlass_min_batch_size = None
metadata = builder.build(
0,
@@ -486,6 +497,10 @@ def _make_topk(
pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"),
pytest.param(16, 1, 8, 1, True, id="tp4-query-len-1"),
pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"),
# Measured B300 EAGLE3 dispatch boundaries.
pytest.param(64, 4, 2, 4, True, id="tp1-batch-4-query-len-4"),
pytest.param(32, 2, 4, 4, True, id="tp2-batch-8-query-len-4"),
pytest.param(16, 1, 8, 4, True, id="tp4-batch-16-query-len-4"),
],
)
def test_msa_cutlass_decode_matches_triton_with_interleaved_cache(
@@ -782,6 +797,46 @@ def _make_nvfp4_kv_cache(
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
def _e4m3_staged_kv_reference(
kv_reference: torch.Tensor, k_scale: torch.Tensor, v_scale: torch.Tensor
) -> torch.Tensor:
"""The dequantized NVFP4 cache as an E4M3-query kernel reads it: each
code * block_scale rounded to E4M3, then the global scale."""
sides = []
for side, global_scale in zip(
kv_reference.float().split(HEAD_DIM, dim=-1), (k_scale, v_scale)
):
sides.append(
(side / global_scale).to(torch.float8_e4m3fn).float() * global_scale
)
return torch.cat(sides, dim=-1).to(torch.bfloat16)
def _assert_close_fp8_prefill(actual: torch.Tensor, expected: torch.Tensor) -> None:
"""The fp8-q prefill rounds each split's probabilities to E4M3, and the splits
follow the SM count, so a rare element moves past the BF16 tolerance."""
close = torch.isclose(actual, expected, atol=0.05, rtol=0.05)
assert close.float().mean().item() > 1 - 1e-4
torch.testing.assert_close(actual, expected, atol=0.25, rtol=0.05)
def _record_query_dtypes(
monkeypatch: pytest.MonkeyPatch, kernel_name: str
) -> list[torch.dtype]:
"""Record the query dtype of every ``fmha_sm100.sparse.<kernel_name>`` call."""
from vllm.third_party.fmha_sm100 import sparse as msa_sparse
kernel = getattr(msa_sparse, kernel_name)
query_dtypes: list[torch.dtype] = []
def record(q, *args, **kwargs):
query_dtypes.append(q.dtype)
return kernel(q, *args, **kwargs)
monkeypatch.setattr(msa_sparse, kernel_name, record)
return query_dtypes
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
is one contiguous run of the page: a TP rank's page is a byte slice."""
@@ -898,27 +953,14 @@ def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
num_q_heads: int,
def _make_prefill_metadata(
q_lens: list[int],
seq_lens_list: list[int],
block_table: torch.Tensor,
num_kv_heads: int,
) -> None:
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
including chunked prefill over cached context, and takes q from its fp8
copy (the only one the fused insert writes on this path)."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
) -> tuple[MiniMaxM3SparseMetadata, torch.Tensor]:
"""Prefill-only metadata (chunks over cached context) and a top-k selecting
every causally visible page (<= TOPK per query), ending with the local one."""
total_q = sum(q_lens)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
@@ -948,7 +990,6 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
prefill=prefill,
)
# Every causally visible page (<= TOPK per query), ending with the local one.
topk = torch.full(
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
)
@@ -959,6 +1000,35 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
assert visible <= TOPK
topk[token, :, :visible] = torch.arange(visible, device="cuda")
token += 1
return metadata, topk
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
monkeypatch: pytest.MonkeyPatch,
num_q_heads: int,
num_kv_heads: int,
) -> None:
"""MSA prefill reads NVFP4 pages as their dequantized values, staged to E4M3
for the E4M3 q, including chunked prefill over cached context; it attends
with the fp8 q (the only one the fused insert writes on this path)."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
metadata, topk = _make_prefill_metadata(
q_lens, seq_lens_list, block_table, num_kv_heads
)
total_q = sum(q_lens)
layer_name = "model.layers.0.self_attn.attn"
q_scale = 0.5
@@ -976,6 +1046,7 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
/ q_scale
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
kv_reference = _e4m3_staged_kv_reference(kv_reference, k_scale, v_scale)
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
@@ -984,6 +1055,7 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
query_dtypes = _record_query_dtypes(monkeypatch, "sparse_atten_nvfp4_kv_func")
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
@@ -1001,7 +1073,8 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
query_fp8=query_fp8,
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
assert query_dtypes == [torch.float8_e4m3fn]
_assert_close_fp8_prefill(actual, expected)
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
@@ -1079,3 +1152,80 @@ def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_reads_query_fp8_over_fp8_cache(
monkeypatch: pytest.MonkeyPatch,
num_q_heads: int,
num_kv_heads: int,
) -> None:
"""With CUTLASS decode on, FP8 prefill (incl. chunks over cached context)
attends with the fp8 q the fused insert writes, not a BF16 copy, and
matches Triton on the dequantized q."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
kv_cache = (
torch.randn(
num_pages,
num_kv_heads,
BLOCK_SIZE,
2 * HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
)
* 0.5
).to(torch.float8_e4m3fn)
metadata, topk = _make_prefill_metadata(
q_lens, seq_lens_list, block_table, num_kv_heads
)
layer_name = "model.layers.0.self_attn.attn"
q_scale = 0.5
layer = SimpleNamespace(
layer_name=layer_name,
topk_indices_buffer=topk,
_q_scale_float=q_scale,
)
query_fp8 = (
torch.randn(
sum(q_lens), num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
/ q_scale
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=num_kv_heads,
kv_cache_dtype="fp8",
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
assert impl.use_cutlass_decode
query_dtypes = _record_query_dtypes(monkeypatch, "sparse_atten_func")
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
slot_mapping={},
)
with override_forward_context(forward_context):
expected = MiniMaxM3SparseTritonImpl(**common).forward(
layer, query, kv_cache, torch.empty_like(query)
)
actual = impl.forward(
layer,
torch.full_like(query, float("nan")),
kv_cache,
torch.zeros_like(query),
query_fp8=query_fp8,
)
current_platform.synchronize()
assert query_dtypes == [torch.float8_e4m3fn]
_assert_close_fp8_prefill(actual, expected)
@@ -346,7 +346,6 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
value_cache[..., :seq_len] = value.permute(1, 2, 0)
scale = torch.tensor(1.0, device=device)
metadata = SimpleNamespace(
use_cascade=False,
num_actual_tokens=query_len,
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32, device=device),
seq_lens=torch.tensor([seq_len], dtype=torch.int32, device=device),
@@ -47,16 +47,18 @@ pytestmark = pytest.mark.skipif(
#
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
# Cover all three regimes:
# Cover all four regimes:
# 8 -> padded to 16 (divisor, repeat_interleave), so the real heads sit at
# stride 2 in the padded output. Live case: Kimi-K2.x at TP8.
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
# 16 -> no padding, output aliases the caller's buffer.
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
# would need TP4, which does not fit in 288 GiB; 128-head models give
# 128/64/32/16/8), so this case exists to keep the general path honest
# for future architectures: it is the only one of the three where
# padding above 16 actually happens, so it is what proves the
# replicate-pad/slice-back argument holds for a target other than 16.
HEAD_COUNTS = [12, 16, 24]
# for future architectures: it is the only one where padding above
# 16 actually happens, so it is what proves the replicate-pad/slice-back
# argument holds for a target other than 16.
HEAD_COUNTS = [8, 12, 16, 24]
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
@@ -33,6 +33,26 @@ def _require_aiter():
pytest.skip("aiter is required on supported ROCm hardware for this test")
@torch.inference_mode()
def test_gather_kv_b_proj_op_schema() -> None:
"""Validate fake tensors and mutation of both expanded K/V outputs."""
_require_aiter()
from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
rows, num_heads = 32, 12
cache = torch.randn(rows, 1, 576, device="cuda").to(current_platform.fp8_dtype())
scale = torch.tensor([0.07], device="cuda")
cu_seq_lens = torch.tensor([0, 13, rows], dtype=torch.int32, device="cuda")
indices = torch.arange(rows, dtype=torch.int32, device="cuda").flip(0)
weight = torch.randn(num_heads * 256, 512, dtype=torch.bfloat16, device="cuda")
k = torch.empty(rows, num_heads, 192, dtype=torch.bfloat16, device="cuda")
v = torch.empty(rows, num_heads, 128, dtype=torch.bfloat16, device="cuda")
opcheck(
torch.ops.vllm.rocm_aiter_gather_kv_b_proj,
(cache, scale, cu_seq_lens, indices, cu_seq_lens, weight, k, v),
)
@torch.inference_mode()
def test_mla_decode_fwd_op_schema() -> None:
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
@@ -463,7 +463,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
set_random_seed(412)
num_tokens, num_heads, block_size = 2, 8, 16
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2 * block_size, 1, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
impl = object.__new__(ROCMAiterMLASparseImpl)
@@ -476,6 +476,8 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
(num_tokens, 128), -1, dtype=torch.int32, device="cuda"
)
impl.topk_indices_buffer[:, : selected.shape[1]] = selected
impl.owns_indexer = True
impl.index_epoch = 0
impl.q_concat_buffer = torch.empty_like(q)
metadata = SimpleNamespace(
attn_out_dtype=dtype,
@@ -504,7 +506,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
SimpleNamespace(_q_scale=None, _k_scale=None),
)
references = [
_sink_reference(q[i], kv[:, 0][selected[i].long()], sinks, SM_SCALE)
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
for i in range(num_tokens)
]
expected, _, value_scale, score_error = (
@@ -0,0 +1,295 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip(
"ROCm AITER sparse MLA index remap test requires ROCm.",
allow_module_level=True,
)
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip(
"ROCm AITER sparse MLA index remap test requires a supported AITER "
"installation.",
allow_module_level=True,
)
from vllm.model_executor.layers.attention import sparse_mla_attention as sparse_mla_mod
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
NUM_HEADS = 16
KV_LORA_RANK = 512
QK_NOPE_HEAD_DIM = 128
QK_ROPE_HEAD_DIM = 64
HEAD_SIZE = 576
NUM_TOKENS = 2
TOPK_TOKENS = 4
@pytest.fixture(autouse=True)
def _reset_index_epochs():
# The counter is keyed by id(), and a freed test buffer can hand its id to
# the next one, so a leftover count would leak across tests.
sparse_mod._INDEX_EPOCHS.clear()
yield
sparse_mod._INDEX_EPOCHS.clear()
def _make_metadata():
return SimpleNamespace(
num_actual_tokens=NUM_TOKENS,
req_id_per_token=torch.zeros(NUM_TOKENS, dtype=torch.int32, device="cpu"),
block_table=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
paged_kv_indptr=torch.zeros(NUM_TOKENS + 1, dtype=torch.int32, device="cpu"),
paged_kv_indices=torch.zeros(
NUM_TOKENS * TOPK_TOKENS, dtype=torch.int32, device="cpu"
),
block_size=1,
topk_tokens=TOPK_TOKENS,
remapped_buf=None,
remapped_epoch=-1,
)
def _patch_init_deps(monkeypatch, *, speculative: bool = False):
monkeypatch.setattr(
sparse_mod.AiterMLAHelper,
"check_num_heads_validity",
staticmethod(lambda num_heads: None),
)
vllm_config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=NUM_TOKENS),
model_config=SimpleNamespace(dtype=torch.bfloat16),
speculative_config=SimpleNamespace() if speculative else None,
attention_config=SimpleNamespace(hisparse_config=None),
)
monkeypatch.setattr(sparse_mod, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(sparse_mla_mod, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(
sparse_mla_mod, "get_tensor_model_parallel_world_size", lambda: 1
)
monkeypatch.setattr(
sparse_mod,
"current_workspace_manager",
lambda: SimpleNamespace(
get_simultaneous=lambda *specs: tuple(
torch.zeros(shape, dtype=dtype, device="cpu") for shape, dtype in specs
)
),
)
def _build_impl(indexer, topk_indices_buffer):
return sparse_mod.ROCMAiterMLASparseImpl(
num_heads=NUM_HEADS,
head_size=HEAD_SIZE,
scale=1.0,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="auto",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
topk_indices_buffer=topk_indices_buffer,
indexer=indexer,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
v_head_dim=QK_NOPE_HEAD_DIM,
kv_b_proj=None,
)
def _new_buffer():
return torch.zeros(NUM_TOKENS, TOPK_TOKENS, dtype=torch.int32, device="cpu")
def _patch_forward_deps(monkeypatch, impl, remap_calls):
monkeypatch.setattr(
sparse_mod,
"triton_convert_req_index_to_global_index",
lambda *args, **kwargs: remap_calls.append(args),
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper,
"get_mla_padded_q",
staticmethod(lambda num_heads, q: q),
)
attn_out = torch.zeros(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK, device="cpu")
monkeypatch.setattr(
type(impl),
"_forward_mla",
lambda self, layer, q, kv_cache, attn_metadata: (attn_out, None),
)
return attn_out
def _run_forward(impl, metadata):
return impl.forward_mqa(
torch.zeros(NUM_TOKENS, NUM_HEADS, HEAD_SIZE, device="cpu"),
torch.zeros(4, 1, HEAD_SIZE, device="cpu"),
metadata,
SimpleNamespace(_q_scale=None, _k_scale=None),
)
@pytest.mark.parametrize("has_indexer", [True, False])
def test_owns_indexer_is_structural(monkeypatch, has_indexer):
"""owns_indexer reflects the layer, not the serving config."""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
indexer = SimpleNamespace(topk_indices_buffer=buffer) if has_indexer else None
impl = _build_impl(indexer, buffer)
assert impl.owns_indexer is has_indexer
assert impl.topk_indices_buffer is buffer
def test_forward_mqa_remaps_once_per_metadata(monkeypatch):
"""Skip layers sharing a metadata share its remap; the first one pays.
All layers read one top-k buffer, so the reuse has to key off the metadata
that owns ``paged_kv_indices``, not off the buffer being distinct.
"""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
first = _build_impl(None, buffer)
second = _build_impl(None, buffer)
remap_calls: list[tuple] = []
attn_out = _patch_forward_deps(monkeypatch, first, remap_calls)
metadata = _make_metadata()
out, lse = _run_forward(first, metadata)
assert out is attn_out
assert lse is None
assert len(remap_calls) == 1
assert metadata.remapped_buf is buffer
_run_forward(second, metadata)
assert len(remap_calls) == 1
def test_forward_mqa_remaps_for_each_metadata(monkeypatch):
"""Skip layers on a second metadata must remap its own paged_kv_indices.
Regression: gating on ``indexer is not None`` skipped every layer in the
non-indexer metadata group, leaving its ``paged_kv_indices`` unwritten.
"""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
indexer_layer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
skip_layer = _build_impl(None, buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, indexer_layer, remap_calls)
indexer_metadata = _make_metadata()
_run_forward(indexer_layer, indexer_metadata)
assert len(remap_calls) == 1
assert indexer_metadata.remapped_buf is buffer
# A distinct metadata instance carries its own, still-unwritten indices.
skip_metadata = _make_metadata()
_run_forward(skip_layer, skip_metadata)
assert len(remap_calls) == 2
assert skip_metadata.remapped_buf is buffer
def test_forward_mqa_remaps_after_a_later_indexer_write(monkeypatch):
"""A skip metadata must be refreshed once per indexer write, not once ever.
Regression for cross-layer index sharing (``index_topk_freq``). With
freq=4 on a 78-layer model the 57 skip layers share one metadata while the
21 indexer layers share another, so the skip metadata has to pick up every
new selection. Recording only the buffer left layers 4 onwards attending
to the first selection of the pass, which collapsed RULER niah_single_2 to
0.026 on GLM-5.3 and took GPQA from 0.884 to 0.672.
"""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
# Construction order mirrors the layer stack: indexer, three skips, indexer.
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
first_skip = _build_impl(None, buffer)
reuse_skip = _build_impl(None, buffer)
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
next_skip = _build_impl(None, buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, first_skip, remap_calls)
skip_metadata = _make_metadata()
_run_forward(first_skip, skip_metadata)
assert len(remap_calls) == 1
# Same selection, same metadata: reuse.
_run_forward(reuse_skip, skip_metadata)
assert len(remap_calls) == 1
# A later indexer layer has since rewritten the buffer in place. The object
# is unchanged, so only the epoch can reveal it.
_run_forward(next_skip, skip_metadata)
assert len(remap_calls) == 2
assert skip_metadata.remapped_epoch == next_skip.index_epoch
def test_index_epoch_tracks_the_preceding_indexer_write(monkeypatch):
"""Layers between two indexer layers share the earlier one's epoch."""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
first_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
skip_a = _build_impl(None, buffer)
skip_b = _build_impl(None, buffer)
second_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
skip_c = _build_impl(None, buffer)
assert skip_a.index_epoch == first_indexer.index_epoch
assert skip_b.index_epoch == first_indexer.index_epoch
assert second_indexer.index_epoch == first_indexer.index_epoch + 1
assert skip_c.index_epoch == second_indexer.index_epoch
def test_forward_mqa_remaps_when_buffer_differs(monkeypatch):
"""A layer reading a different buffer cannot reuse the metadata's remap."""
_patch_init_deps(monkeypatch)
remapped_buffer = _new_buffer()
other_buffer = _new_buffer()
layer = _build_impl(None, other_buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, layer, remap_calls)
metadata = _make_metadata()
metadata.remapped_buf = remapped_buffer
_run_forward(layer, metadata)
assert len(remap_calls) == 1
assert metadata.remapped_buf is other_buffer
def test_forward_mqa_indexer_layer_always_remaps(monkeypatch):
"""The indexer rewrites its buffer in place, so its remap is never stale."""
_patch_init_deps(monkeypatch)
buffer = _new_buffer()
impl = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
remap_calls: list[tuple] = []
_patch_forward_deps(monkeypatch, impl, remap_calls)
metadata = _make_metadata()
metadata.remapped_buf = buffer
metadata.remapped_epoch = impl.index_epoch
_run_forward(impl, metadata)
assert len(remap_calls) == 1
@@ -33,6 +33,101 @@ class _FakeAiter(ModuleType):
get_mla_metadata_v1: Mock
@pytest.mark.parametrize("block_size", [16, 64, 128])
@pytest.mark.parametrize("topk_tokens", [128, 256])
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
"""Convert valid, masked, and out-of-range sparse indices exactly."""
device = torch.device("cuda")
num_tokens = 4
max_blocks = 4
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
block_table = torch.tensor(
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
)
token_indices = torch.arange(
num_tokens * topk_tokens, dtype=torch.int32, device=device
).view(num_tokens, topk_tokens)
token_indices %= max_blocks * block_size
token_indices[0, 0] = -1
token_indices[1, 1] = max_blocks * block_size
cu_seqlens = torch.arange(
0,
(num_tokens + 1) * topk_tokens,
topk_tokens,
dtype=torch.int32,
device=device,
)
output = torch.full(
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
)
sparse_mod.triton_convert_req_index_to_global_index(
req_ids,
block_table,
token_indices,
cu_seqlens,
output,
BLOCK_STRIDE_ROWS=block_size,
BLOCK_SIZE=block_size,
NUM_TOPK_TOKENS=topk_tokens,
)
block_ids = token_indices // block_size
valid = (token_indices >= 0) & (block_ids < max_blocks)
safe_blocks = block_ids.clamp(0, max_blocks - 1)
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
def test_rocm_index_conversion_rejects_mismatched_token_extents():
"""Reject full-batch request IDs paired with decode-subset indices."""
with pytest.raises(AssertionError, match="must cover the same tokens"):
sparse_mod.triton_convert_req_index_to_global_index(
torch.zeros(3, dtype=torch.int32),
torch.zeros(1, 4, dtype=torch.int32),
torch.zeros(2, 128, dtype=torch.int32),
torch.arange(3, dtype=torch.int32),
torch.zeros(256, dtype=torch.int32),
BLOCK_STRIDE_ROWS=64,
NUM_TOPK_TOKENS=128,
)
def test_rocm_index_conversion_respects_packed_valid_counts():
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
device = torch.device("cuda")
topk_tokens = 128
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
token_indices = torch.arange(
3 * topk_tokens, dtype=torch.int32, device=device
).view(3, topk_tokens)
token_indices %= 3 * 64
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
sparse_mod.triton_convert_req_index_to_global_index(
req_ids,
block_table,
token_indices,
cu_seqlens,
output,
BLOCK_STRIDE_ROWS=64,
BLOCK_SIZE=64,
NUM_TOPK_TOKENS=topk_tokens,
)
expected_parts = []
for token_id, count in enumerate(valid_counts.tolist()):
local = token_indices[token_id, :count]
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
expected = torch.cat(expected_parts)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
def _make_builder():
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
max_num_batched_tokens = 8
@@ -44,6 +139,18 @@ def _make_builder():
builder.kv_cache_dtype = "fp8"
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
builder.topk_tokens = topk_tokens
# SparseMLACommonMetadataBuilder state consumed by its build() method.
builder.cp_kv_cache_interleave_size = 1
builder.use_pcp = False
builder._prefill_backend = None
def cpu_token_to_req(metadata):
return torch.repeat_interleave(
torch.arange(metadata.num_reqs, dtype=torch.int32),
metadata.query_start_loc_cpu.diff(),
)
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
builder.req_id_per_token_buffer = torch.zeros(
max_num_batched_tokens, dtype=torch.int32, device="cpu"
)
@@ -62,6 +169,8 @@ def _make_builder():
builder._use_persistent_metadata = True
builder._num_attention_heads = 16
builder._num_compute_units = current_platform.num_compute_units()
builder._mla_q_dtype = torch.bfloat16
builder._mla_kv_dtype = torch.bfloat16
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
@@ -114,7 +223,7 @@ def _patch_build_deps(monkeypatch, events=None):
on CPU."""
def fake_generate_sparse_seqlen_triton(
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
):
return torch.zeros(num_tokens, dtype=torch.int32, device="cpu")
@@ -167,6 +276,25 @@ def test_build_populates_mixed_split_fields(monkeypatch):
assert md.prefill is None
def test_build_fills_paged_kv_indptr_tail(monkeypatch):
"""Rows past num_tokens must read as empty: the tail repeats the total."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
monkeypatch.setattr(
sparse_mod,
"generate_sparse_seqlen_triton",
lambda *args, **kwargs: torch.tensor([3, 5], dtype=torch.int32),
)
builder.paged_kv_indptr[1:].fill_(-1)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
)
assert md.paged_kv_indptr.tolist() == [0, 3, 8]
assert builder.paged_kv_indptr.tolist() == [0, 3, 8, 8, 8, 8, 8, 8, 8]
def test_sink_build_skips_persistent_metadata(monkeypatch):
builder = _make_builder()
builder._use_persistent_metadata = False
@@ -216,6 +344,8 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
assert events == ["metadata", "sync"]
assert fake_get_mla_metadata_v1_mock.call_count == 1
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
events.clear()
@@ -223,3 +353,399 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
assert events == []
assert fake_get_mla_metadata_v1_mock.call_count == 1
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
A mixed batch is routed as sparse MQA for its leading decode requests and
dense MHA for its trailing short prefill requests. The AITER metadata
buffers therefore must describe only the decode subset; including prefill
tokens would mismatch the query passed to ``forward_mqa``.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=4,
max_seq_len=16,
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=2,
max_query_len=4,
max_seq_len=16,
num_actual_tokens=5,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(5),
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
topk_tokens=4,
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
seq_lens=seq_lens,
prefill_max_seq_len=4,
prefill=SimpleNamespace(use_dense_mha=True),
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
sparse_call: dict[str, torch.Tensor] = {}
def fake_generate_sparse_seqlen(
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
):
sparse_call.update(
query_lens=torch.diff(sparse_query_start_loc),
seq_lens=sparse_seq_lens.clone(),
query_start_loc=sparse_query_start_loc.clone(),
num_tokens=num_tokens,
)
return torch.ones(num_tokens, dtype=torch.int32)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
)
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = Mock()
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
assert result.prefill is metadata.prefill
assert result.num_prefills == 1
assert result.req_id_per_token.tolist() == [0]
assert result.qo_indptr.numel() == 2
assert result.paged_kv_indptr.numel() == 2
assert sparse_call["num_tokens"] == 1
assert sparse_call["query_lens"].tolist() == [1]
assert sparse_call["seq_lens"].tolist() == [16]
assert sparse_call["query_start_loc"].tolist() == [0, 1]
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
"""A prefill longer than top-k must remain in shared sparse metadata.
Dense MHA is only selected when the prefill sequence fits within the
sparse top-k window. Once it exceeds that window, both decode and prefill
are handled by sparse MQA, so every query token must be represented by the
same sparse paging metadata.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
# One decode token followed by a five-token prefill. topk_tokens is four,
# so the prefill must not be split onto the dense-MHA path.
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=5,
max_seq_len=16,
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=2,
max_query_len=5,
max_seq_len=16,
num_actual_tokens=6,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(6),
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
topk_tokens=4,
num_decodes=1,
num_prefills=1,
num_decode_tokens=1,
seq_lens=seq_lens,
prefill_max_seq_len=5,
prefill=SimpleNamespace(use_dense_mha=False),
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
sparse_call: dict[str, torch.Tensor] = {}
def fake_generate_sparse_seqlen(
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
):
sparse_call.update(
query_lens=torch.diff(sparse_query_start_loc),
seq_lens=sparse_seq_lens.clone(),
query_start_loc=sparse_query_start_loc.clone(),
num_tokens=num_tokens,
)
return torch.ones(num_tokens, dtype=torch.int32)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
)
fake_aiter = _FakeAiter("aiter")
get_persistent_metadata = Mock()
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
assert result.num_decodes == 1
assert result.num_prefills == 1
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
assert result.qo_indptr.numel() == 7
assert result.paged_kv_indptr.numel() == 7
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
assert sparse_call["num_tokens"] == 6
assert sparse_call["query_lens"].tolist() == [1, 5]
assert sparse_call["seq_lens"].tolist() == [16, 5]
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
get_persistent_metadata.assert_called_once()
assert result.work_meta_data is builder._mla_work_meta_data
assert result.reduce_indptr is builder._mla_reduce_indptr
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
"""The sparse kernel inputs must have the same token extent as its query.
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
tokens while some metadata buffers may represent the full batch. This
guards the slicing that prevents the index conversion from writing beyond
the decode-sized output.
"""
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.kv_cache_dtype = "auto"
impl.num_heads = 4
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
impl.owns_indexer = True
impl.index_epoch = 0
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
forward_mla = Mock(return_value=(expected_output, None))
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
metadata = SimpleNamespace(
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
block_size=1,
topk_tokens=4,
)
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
captured = {}
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
captured["req_ids"] = req_ids.clone()
captured["token_indices"] = token_indices.clone()
monkeypatch.setattr(
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
)
output, lse = impl.forward_mqa(
query,
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
metadata,
SimpleNamespace(),
)
assert output is expected_output
assert lse is None
assert captured["req_ids"].tolist() == [0, 1]
assert captured["token_indices"].shape == (2, 4)
forward_mla.assert_called_once()
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.kv_cache_dtype = kv_cache_dtype
impl.num_heads = 4
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
impl.owns_indexer = True
impl.index_epoch = 0
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
forward_mla = Mock(return_value=(expected_output, None))
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
metadata = SimpleNamespace(
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
block_size=1,
topk_tokens=4,
)
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
captured = {}
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
captured["req_ids"] = req_ids.clone()
captured["token_indices"] = token_indices.clone()
monkeypatch.setattr(
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
)
monkeypatch.setattr(
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
)
if kv_cache_dtype == "fp8":
fp8_dtype = current_platform.fp8_dtype()
monkeypatch.setattr(
sparse_mod.ops,
"scaled_fp8_quant",
lambda q, scale: (q.to(fp8_dtype), None),
)
output, lse = impl.forward_mqa(
query,
kv_cache,
metadata,
SimpleNamespace(_q_scale=torch.tensor(1.0)),
)
assert output is expected_output
assert lse is None
assert captured["req_ids"].tolist() == [0, 1]
assert captured["token_indices"].shape == (2, 4)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("length_relation", ["short", "long"])
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
def test_sparse_metadata_workload_length_dtype_matrix(
monkeypatch, workload, length_relation, kv_cache_dtype
):
"""Cover routing and persistent metadata across the ROCm support matrix.
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
"""
builder = _make_builder()
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
)
if kv_cache_dtype == "fp8":
builder._mla_q_dtype = current_platform.fp8_dtype()
builder._mla_kv_dtype = current_platform.fp8_dtype()
is_long = length_relation == "long"
prefill_len = 5 if is_long else 3
decode_seq_len = 8 if is_long else 3
if workload == "prefill":
query_lens = [prefill_len]
seq_lens_list = [prefill_len]
num_decodes, num_prefills = 0, 1
elif workload == "mixed":
query_lens = [1, prefill_len]
seq_lens_list = [decode_seq_len, prefill_len]
num_decodes, num_prefills = 1, 1
else:
query_lens = [1, 1]
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
num_decodes, num_prefills = 2, 0
query_start_loc = torch.tensor(
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
num_tokens = sum(query_lens)
common_metadata = SimpleNamespace(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
max_query_len=max(query_lens),
max_seq_len=max(seq_lens_list),
)
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
num_reqs=len(query_lens),
max_query_len=max(query_lens),
max_seq_len=max(seq_lens_list),
num_actual_tokens=num_tokens,
query_start_loc=query_start_loc,
slot_mapping=torch.arange(num_tokens),
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
len(query_lens), 8
),
req_id_per_token=torch.repeat_interleave(
torch.arange(len(query_lens), dtype=torch.int32),
torch.tensor(query_lens),
),
topk_tokens=4,
num_decodes=num_decodes,
num_prefills=num_prefills,
num_decode_tokens=num_decodes,
seq_lens=seq_lens,
prefill_max_seq_len=prefill_len if num_prefills else 0,
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
)
monkeypatch.setattr(
sparse_mod.SparseMLACommonMetadataBuilder,
"build",
lambda *args, **kwargs: metadata,
)
monkeypatch.setattr(
sparse_mod,
"generate_sparse_seqlen_triton",
lambda *args, **kwargs: torch.ones(args[3], dtype=torch.int32),
)
get_persistent_metadata = Mock()
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(synchronize=lambda: None),
)
result = builder.build(0, common_metadata)
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
assert result.req_id_per_token.numel() == expected_sparse_tokens
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
if expected_sparse_tokens == 0 or uses_fallback:
get_persistent_metadata.assert_not_called()
else:
get_persistent_metadata.assert_called_once()
if uses_fallback or expected_sparse_tokens == 0:
assert result.work_meta_data is None
assert result.reduce_indptr is None
else:
assert result.work_meta_data is builder._mla_work_meta_data
assert result.reduce_indptr is builder._mla_reduce_indptr
@@ -253,6 +253,7 @@ def _prefill_metadata(
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
total_seq_lens=1,
max_seqlen_k=int(ends.max()),
token_start=t0,
token_end=t1,
num_reqs=req_hi - req_lo,
@@ -70,7 +70,7 @@ def enable_aiter_mqa(monkeypatch):
def test_unpaged_mqa_logits_preserves_intervals_and_scale_layout(
column_scales, monkeypatch, enable_aiter_mqa
) -> None:
"""The vLLM AITER dispatch preserves ragged masking and per-token scales."""
"""The vLLM AITER dispatch honors ragged intervals and per-token scales."""
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
@@ -108,7 +108,7 @@ def test_unpaged_mqa_logits_preserves_intervals_and_scale_layout(
valid = (positions >= starts[:, None]) & (positions < ends[:, None])
assert actual.shape == expected.shape
assert actual.dtype == torch.float32
assert torch.equal(torch.isneginf(actual), ~valid)
assert not torch.isneginf(actual[valid]).any()
torch.testing.assert_close(actual[valid], expected[valid], rtol=1e-3, atol=1e-3)
@@ -1983,6 +1983,272 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
@requires_gfx950
@torch.inference_mode()
def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
"""Replay V4.1 ROCm decode after device-only query reallocation."""
from tests.v1.attention.utils import create_vllm_config
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
compute_global_topk_ragged_indices_and_indptr,
)
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
from vllm.v1.kv_cache_interface import MLAAttentionSpec, SlidingWindowMLASpec
device = torch.device("cuda")
num_reqs = 3
upper_query_len = 6
graph_tokens = num_reqs * upper_query_len
block_size = 128
compress_ratio = 2
compressed_block_size = block_size // compress_ratio
vllm_config = create_vllm_config(
model_name="facebook/opt-125m",
max_model_len=1024,
block_size=block_size,
max_num_seqs=num_reqs,
max_num_batched_tokens=graph_tokens,
hf_config_override={
"compress_ratios": [0, 1, 2],
"index_topk": 8,
"sliding_window": 32,
},
)
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=upper_query_len - 1,
parallel_drafting=False,
enable_adaptive_verification=True,
use_dspark=lambda: True,
)
mla_spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_DIM,
dtype=torch.bfloat16,
tokens_per_state=compress_ratio,
cache_dtype_str="fp8_ds_mla",
model_version="deepseek_v4",
)
swa_spec = SlidingWindowMLASpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_DIM,
dtype=torch.uint8,
sliding_window=32,
cache_dtype_str="fp8_ds_mla",
model_version="deepseek_v4",
)
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
mla_spec, ["c2a"], vllm_config, device
)
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["c2a"], vllm_config, device
)
assert (
mla_builder.get_cudagraph_support(vllm_config, mla_spec)
== AttentionCGSupport.ALWAYS
)
assert (
swa_builder.get_cudagraph_support(vllm_config, swa_spec)
== AttentionCGSupport.ALWAYS
)
seq_lens = torch.tensor([100, 110, 120], dtype=torch.int32, device=device)
seq_lens_cpu = seq_lens.cpu()
query_start_loc_cpu = torch.arange(
0, graph_tokens + 1, upper_query_len, dtype=torch.int32
)
block_table = torch.arange(num_reqs, dtype=torch.int32, device=device).view(
num_reqs, 1
)
def build_metadata(query_lens: list[int]):
query_lens_tensor = torch.tensor(query_lens, dtype=torch.int32, device=device)
query_start_loc = torch.cat(
[
torch.zeros(1, dtype=torch.int32, device=device),
query_lens_tensor.cumsum(0),
]
)
active_tokens = sum(query_lens)
slot_mapping = torch.full((graph_tokens,), -1, dtype=torch.int64, device=device)
slot_mapping[:active_tokens] = torch.arange(
active_tokens, dtype=torch.int64, device=device
)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc_cpu,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=seq_lens_cpu,
num_reqs=num_reqs,
num_actual_tokens=graph_tokens,
max_query_len=upper_query_len,
max_seq_len=int(seq_lens_cpu.max()),
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=True,
)
return (
mla_builder.build_for_cudagraph_capture(common),
swa_builder.build_for_cudagraph_capture(common),
)
mla_metadata, swa_metadata = build_metadata([6, 6, 6])
metadata_ptrs = (
mla_metadata.req_id_per_token.data_ptr(),
mla_metadata.slot_mapping.data_ptr(),
swa_metadata.token_to_req_indices.data_ptr(),
swa_metadata.decode_swa_ragged_indices.data_ptr(),
swa_metadata.decode_swa_ragged_indptr.data_ptr(),
)
topk_indices = (
torch.arange(8, dtype=torch.int32, device=device)
.expand(graph_tokens, -1)
.contiguous()
)
torch.manual_seed(23)
num_heads = 16
q = (
torch.randn(
graph_tokens,
num_heads,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125
)
swa_cache = _pack_fp8_ds_mla_cache(
torch.randn(
num_reqs * block_size,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125,
block_size,
use_fnuz=False,
)
compressed_cache = _pack_fp8_ds_mla_cache(
torch.randn(
num_reqs * compressed_block_size,
HEAD_DIM,
dtype=torch.bfloat16,
device=device,
)
* 0.125,
compressed_block_size,
use_fnuz=False,
)
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
out = torch.empty_like(q)
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
def build_topk_ragged(mla_md, swa_md):
return compute_global_topk_ragged_indices_and_indptr(
topk_indices,
swa_md.token_to_req_indices,
mla_md.block_table,
compressed_block_size,
swa_md.is_valid_token,
)
def run_decode(mla_md, swa_md) -> None:
topk_ragged_indices, topk_ragged_indptr, topk_lens = build_topk_ragged(
mla_md, swa_md
)
mod.rocm_sparse_attn_decode(
q=q,
kv_cache=compressed_cache,
swa_k_cache=swa_cache,
swa_only=False,
topk_indices=None,
topk_lens=topk_lens,
swa_indices=swa_md.decode_swa_indices,
swa_lens=swa_md.decode_swa_lens,
swa_ragged_indices=swa_md.decode_swa_ragged_indices,
swa_ragged_indptr=swa_md.decode_swa_ragged_indptr,
topk_ragged_indices=topk_ragged_indices,
topk_ragged_indptr=topk_ragged_indptr,
attn_sink=attn_sink,
scale=HEAD_DIM**-0.5,
head_dim=HEAD_DIM,
nope_head_dim=NOPE_HEAD_DIM,
rope_head_dim=ROPE_HEAD_DIM,
output=out,
extra_cache_nan_free=True,
)
def reference(mla_md, swa_md) -> torch.Tensor:
topk_ragged_indices, topk_ragged_indptr, _ = build_topk_ragged(mla_md, swa_md)
return _ref_sparse_decode_ragged(
q=q,
main_cache=swa_cache,
main_rows=_rows_from_ragged(
swa_md.decode_swa_ragged_indices,
swa_md.decode_swa_ragged_indptr,
),
scale=HEAD_DIM**-0.5,
attn_sink=attn_sink,
block_size=block_size,
extra_cache=compressed_cache,
extra_rows=_rows_from_ragged(
topk_ragged_indices,
topk_ragged_indptr,
),
extra_block_size=compressed_block_size,
)
run_decode(mla_metadata, swa_metadata)
torch.accelerator.synchronize()
expected_full = reference(mla_metadata, swa_metadata)
torch.testing.assert_close(out, expected_full, atol=2e-2, rtol=2e-2)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run_decode(mla_metadata, swa_metadata)
torch.accelerator.synchronize()
captured_full = out.clone()
reallocated_mla, reallocated_swa = build_metadata([2, 6, 4])
assert metadata_ptrs == (
reallocated_mla.req_id_per_token.data_ptr(),
reallocated_mla.slot_mapping.data_ptr(),
reallocated_swa.token_to_req_indices.data_ptr(),
reallocated_swa.decode_swa_ragged_indices.data_ptr(),
reallocated_swa.decode_swa_ragged_indptr.data_ptr(),
)
expected_owners = torch.tensor(
[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2],
dtype=torch.int32,
device=device,
)
torch.testing.assert_close(reallocated_mla.req_id_per_token[:12], expected_owners)
torch.testing.assert_close(
reallocated_swa.token_to_req_indices[:12], expected_owners
)
assert reallocated_swa.is_valid_token[:12].all()
assert not reallocated_swa.is_valid_token[12:].any()
torch.testing.assert_close(
reallocated_swa.decode_swa_lens[12:],
torch.zeros(6, dtype=torch.int32, device=device),
)
expected_reallocated = reference(reallocated_mla, reallocated_swa)
run_decode(reallocated_mla, reallocated_swa)
torch.accelerator.synchronize()
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
assert not torch.equal(captured_full, expected_reallocated)
graph.replay()
torch.accelerator.synchronize()
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
# ---------------------------------------------------------------------------
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
# ---------------------------------------------------------------------------
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
import torch
@@ -9,11 +11,17 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.ops import triton_unified_attention as attention
from vllm.v1.attention.ops.triton_attention_helpers import (
apply_softcap,
compute_tile_loop_bounds,
softmax_step,
)
from vllm.v1.attention.ops.triton_unified_attention import (
MAX_3D_QUERY_LEN,
reduce_segments,
unified_attention,
)
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
from vllm.v1.kv_cache_interface import KVQuantMode
pytestmark = pytest.mark.skip_global_cleanup
@@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048]
SEQ_THRESHOLD_3D_VALUES = [0, 8]
def _scratch(rows):
shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)]
return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes]
def _uses_splitk(
monkeypatch,
query_len,
scratch,
*,
causal=True,
threshold=8,
mm_prefix=None,
num_seqs=2,
):
query = torch.empty(
(num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu"
)
cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu")
kernel = MagicMock()
monkeypatch.setattr(attention, "kernel_unified_attention", kernel)
monkeypatch.setattr(attention, "reduce_segments", MagicMock())
unified_attention(
query,
cache,
cache,
torch.empty_like(query),
torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len,
query_len,
torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"),
128,
128**-0.5,
causal,
(-1, -1),
torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"),
0.0,
None,
None,
None,
seq_threshold_3D=threshold,
num_par_softmax_segments=16,
softmax_segm_output=scratch[0],
softmax_segm_max=scratch[1],
softmax_segm_expsum=scratch[2],
mm_prefix_range=mm_prefix,
)
return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"]
def _spy_reduce_segments(monkeypatch):
original = attention.reduce_segments
grids = []
def launch(grid):
grids.append(grid)
return original[grid]
spy = MagicMock()
spy.__getitem__.side_effect = launch
monkeypatch.setattr(attention, "reduce_segments", spy)
return grids
@pytest.mark.parametrize(
"query_len,options,expected",
[
(1, {}, True),
(1, {"causal": False}, True),
(5, {}, True),
(6, {"num_seqs": 1}, True),
(8, {}, True),
(9, {"num_seqs": 1}, False),
(5, {"causal": False}, False),
(5, {"causal": "per_sequence"}, False),
(5, {"threshold": 1}, False),
(5, {"batch_invariant": True}, False),
(5, {"mm_prefix": True}, False),
# BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8.
(4, {"num_seqs": 4}, True),
(5, {"num_seqs": 4}, False),
(5, {"scratch_rows": 9}, False),
],
)
def test_splitk_admission_preserves_decode_and_bounds_verification(
monkeypatch, query_len, options, expected
):
options = options.copy()
monkeypatch.setattr(
attention, "is_batch_invariant", options.pop("batch_invariant", False)
)
num_seqs = options.setdefault("num_seqs", 2)
if options.get("causal") == "per_sequence":
options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu")
if options.get("mm_prefix"):
options["mm_prefix"] = torch.zeros(
(num_seqs, 1, 2), dtype=torch.int32, device="cpu"
)
scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len))
assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected
@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"])
def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None:
"""Empty segments must not suppress strongly negative, nonempty segments."""
output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE)
partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE)
maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE)
masses = torch.ones_like(maxima)
if empty_state != "all_empty":
partial[:, :, 0] = 3.0
maxima[:, :, 0] = -1000.0
if empty_state in ("masked", "all_empty"):
maxima[:, :, 1:] = 0.0
masses[:, :, 1:] = 0.0
reduce_segments[(1, 1)](
output,
partial,
maxima,
masses,
torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE),
1,
1,
1.0,
output.stride(0),
output.stride(1),
1,
16,
32,
32,
torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
16,
16,
False,
)
torch.testing.assert_close(
output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0)
)
@triton.jit
def _softmax_after_empty_tile(output_ptr):
maximum = tl.full((1,), float("-inf"), tl.float32)
mass = tl.full((1,), 1.0, tl.float32)
empty_scores = tl.full((1, 16), float("-inf"), tl.float32)
maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass)
scores = tl.full((1, 16), -1000.0, tl.float32)
_, mass, probabilities, _ = softmax_step(scores, maximum, mass)
tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass)
def test_splitk_softmax_recovers_after_fully_masked_tile() -> None:
"""A masked first tile must not suppress later, strongly negative logits."""
probabilities = torch.empty(16, device=DEVICE_TYPE)
_softmax_after_empty_tile[(1,)](probabilities)
torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16))
@pytest.mark.parametrize(
"query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])]
)
def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None:
"""A padded reducer row must return before reading sequence or scratch data."""
output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE)
partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE)
maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE)
masses = torch.ones_like(maxima)
query_starts = torch.tensor(
[0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE
).cumsum(0, dtype=torch.int32)
reduce_segments[(10, 1)](
output,
partial,
maxima,
masses,
torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE),
len(seq_lens),
1,
1.0,
output.stride(0),
output.stride(1),
1,
16,
32,
32,
query_starts,
16,
16,
False,
)
active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0)
torch.testing.assert_close(
output[:active_rows], torch.zeros_like(output[:active_rows])
)
torch.testing.assert_close(
output[active_rows:], torch.full_like(output[active_rows:], 123.0)
)
@pytest.mark.parametrize(
("seq_lens", "num_heads", "q_dtype"),
[
*[
pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}")
for w in range(2, MAX_3D_QUERY_LEN + 1, 2)
],
# Prefill chunk, verification, decode, padding, short query.
pytest.param(
[(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed"
),
*[
pytest.param(
[(6, 128), (6, 129), (6, 257)],
heads,
FP8_DTYPE,
id=f"w6-fp8q-kv{heads[1]}",
)
for heads in [(8, 1), (8, 8)]
],
],
)
@pytest.mark.parametrize("sliding_window", [None, 32])
def test_speculative_splitk_matches_reference(
monkeypatch, seq_lens, num_heads, q_dtype, sliding_window
):
reduce_grids = _spy_reduce_segments(monkeypatch)
test_triton_unified_attn(
seq_lens=seq_lens,
num_heads=num_heads,
head_size=128,
sliding_window=sliding_window,
dtype=torch.bfloat16,
block_size=128,
soft_cap=None,
num_blocks=8,
q_dtype=q_dtype,
seq_threshold_3D=32,
)
assert len(reduce_grids) == 1
def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch):
reduce_grids = _spy_reduce_segments(monkeypatch)
test_triton_unified_attn_bf16_query_fp8_kv(
seq_lens=[(6, 128), (6, 129), (6, 4097)],
num_heads=(8, 1),
head_size=128,
block_size=128,
num_blocks=64,
seq_threshold_3D=16,
)
assert len(reduce_grids) == 1
@triton.jit
def _compute_clamped_mm_tile_bounds(
output_ptr,
@@ -471,16 +732,19 @@ def test_triton_unified_attn(
num_par_softmax_segments = 16
head_size_padded = next_power_of_2(head_size)
scratch_rows = seq_threshold_3D * (
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
)
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
@@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv(
num_par_softmax_segments = 16
head_size_padded = next_power_of_2(head_size)
scratch_rows = seq_threshold_3D * (
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
)
softmax_segm_output = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
dtype=torch.float32,
)
softmax_segm_max = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
softmax_segm_expsum = torch.empty(
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
(scratch_rows, num_query_heads, num_par_softmax_segments),
dtype=torch.float32,
)
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the Triton DiffKV unified-attention kernel."""
from unittest.mock import patch
import pytest
import torch
@@ -16,6 +18,7 @@ from vllm.v1.attention.backends.fa_utils import (
get_flash_attn_version,
is_flash_attn_varlen_func_available,
)
from vllm.v1.attention.ops import triton_unified_attention_diffkv as diffkv_module
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
unified_attention_diffkv,
)
@@ -258,3 +261,95 @@ def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D):
seq_threshold_3D=seq_threshold_3D,
quant_query=True,
)
def _capture_diffkv_launch(
query_lens: list[int], max_seqlen_q: int, segm_first_dim: int
):
"""Run the launcher with both Triton kernels mocked; return grid/kwargs."""
num_query_heads, num_kv_heads = 8, 2
head_size_qk, head_size_v = 128, 128
num_seqs = len(query_lens)
num_tokens = sum(query_lens)
query = torch.zeros(
num_tokens, num_query_heads, head_size_qk, dtype=torch.bfloat16, device="cpu"
)
kv_cache = torch.zeros(
NUM_BLOCKS,
BLOCK_SIZES[0],
num_kv_heads,
head_size_qk + head_size_v,
dtype=torch.bfloat16,
device="cpu",
)
key_cache = kv_cache[..., :head_size_qk]
value_cache = kv_cache[..., head_size_qk:]
out = torch.zeros(
num_tokens, num_query_heads, head_size_v, dtype=torch.bfloat16, device="cpu"
)
cu_seqlens_q = torch.tensor(
[0] + query_lens, dtype=torch.int32, device="cpu"
).cumsum(dim=0, dtype=torch.int32)
seqused_k = torch.full((num_seqs,), 512, dtype=torch.int32, device="cpu")
block_tables = torch.zeros(num_seqs, 32, dtype=torch.int32, device="cpu")
segm_output, segm_max, segm_expsum = _alloc_segm_buffers(
segm_first_dim, num_query_heads, head_size_v
)
with (
patch.object(diffkv_module, "kernel_unified_attention_diffkv") as mock_kernel,
patch.object(diffkv_module, "kernel_reduce_segments_diffkv") as mock_reduce,
):
unified_attention_diffkv(
q=query,
k=key_cache,
v=value_cache,
out=out,
cu_seqlens_q=cu_seqlens_q,
seqused_k=seqused_k,
softmax_scale=head_size_qk**-0.5,
causal=True,
window_size=(-1, -1),
block_table=block_tables,
softcap=0,
max_seqlen_q=max_seqlen_q,
seq_threshold_3D=8,
num_par_softmax_segments=NUM_PAR_SOFTMAX_SEGMENTS,
softmax_segm_output=segm_output,
softmax_segm_max=segm_max,
softmax_segm_expsum=segm_expsum,
)
grid = mock_kernel.__getitem__.call_args.args[0]
kwargs = mock_kernel.__getitem__.return_value.call_args.kwargs
return grid, kwargs, mock_reduce.__getitem__.called
def test_spec_verify_block_m_grouping_and_launch_kwargs() -> None:
"""Pin program granularity for verify/decode without launching Triton."""
verify_grid, verify_kwargs, verify_reduce = _capture_diffkv_launch(
query_lens=[8, 8], max_seqlen_q=8, segm_first_dim=16
)
num_queries_per_kv = 4
expected_block_m = min(128, next_power_of_2(8 * num_queries_per_kv))
assert verify_grid == (4, 2, NUM_PAR_SOFTMAX_SEGMENTS)
assert verify_kwargs["IS_3D"] is True
assert verify_kwargs["BLOCK_M"] == expected_block_m
assert verify_kwargs["BLOCK_Q"] == expected_block_m // num_queries_per_kv
assert verify_kwargs["TILE_SIZE"] == 16
assert verify_kwargs["num_warps"] == 4
assert "num_stages" not in verify_kwargs
assert verify_reduce
decode_grid, decode_kwargs, decode_reduce = _capture_diffkv_launch(
query_lens=[1], max_seqlen_q=1, segm_first_dim=8
)
assert decode_grid == (1, 2, NUM_PAR_SOFTMAX_SEGMENTS)
assert decode_kwargs["IS_3D"] is True
assert decode_kwargs["BLOCK_M"] == 16
assert decode_kwargs["BLOCK_Q"] == 4
assert decode_kwargs["TILE_SIZE"] == 16
assert "num_warps" not in decode_kwargs
assert "num_stages" not in decode_kwargs
assert decode_reduce
+6
View File
@@ -21,6 +21,9 @@ DEVICES = [
@pytest.mark.parametrize("device", DEVICES)
def test_cpu_write(device):
torch.set_default_device(device)
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
# as the UVA view's ops never trigger lazy init.
torch.accelerator.set_device_index(device)
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
@@ -44,6 +47,9 @@ def test_cpu_write(device):
@pytest.mark.parametrize("device", DEVICES)
def test_gpu_write(device):
torch.set_default_device(device)
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
# as the UVA view's ops never trigger lazy init.
torch.accelerator.set_device_index(device)
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
assert gpu_view.device.type == DEVICE_TYPE
@@ -9,6 +9,8 @@ each rank, and summing the per-rank outputs (what the all-reduce does)
reconstructs a full-table lookup.
"""
from unittest.mock import Mock
import pytest
import torch
@@ -124,3 +126,43 @@ def test_rank_sum_is_full_lookup(tp_size):
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
@requires_cuda
@torch.inference_mode()
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
@pytest.mark.parametrize("use_fused", [False, True])
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
"""SP's deferred reduction preserves masked contributions and their dtype."""
with torch.device("cuda"):
values = torch.arange(128 * 16).reshape(128, 16)
table = ((values % 29 - 14) / 16).to(dtype)
ids = torch.tensor([0, 63, 64, 127, 12])
expected = table[ids].float()
for rank in range(2):
local = expected.clone()
local[ids // 64 != rank] = 0
remote = (expected - local).to(dtype)
group = Mock(world_size=2, rank_in_group=rank)
# Model the other vocabulary owner's contribution, including FP8 bytes.
group.all_reduce.side_effect = lambda output, remote=remote: (
output + remote.view(output.dtype)
)
for reduce_results in (True, False):
embedding = VocabParallelEmbedding(
128,
16,
params_dtype=dtype,
parallel_group=group,
reduce_results=reduce_results,
)
embedding.use_fused_embedding = use_fused
embedding.weight_loader(embedding.get_parameter("weight"), table)
actual = embedding(ids)
assert actual.dtype == dtype
torch.testing.assert_close(
actual.float(),
expected if reduce_results else local,
atol=0,
rtol=0,
)
@@ -18,17 +18,32 @@ pytestmark = pytest.mark.skipif(
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
@pytest.mark.parametrize(
"batch,num_key_heads,num_value_heads,dtype,state_dtype",
[
(32, 4, 8, torch.float16, torch.float16),
(32, 4, 8, torch.bfloat16, torch.bfloat16),
(32, 4, 8, torch.float32, torch.float32),
(1, 16, 64, torch.bfloat16, torch.float32),
(9, 4, 12, torch.bfloat16, torch.float32),
(1, 4, 12, torch.bfloat16, torch.bfloat16),
],
)
def test_fused_recurrent_packed_decode_matches_reference(
dtype: torch.dtype, strided_mixed_qkv: bool
dtype: torch.dtype,
strided_mixed_qkv: bool,
batch: int,
num_key_heads: int,
num_value_heads: int,
state_dtype: torch.dtype,
):
torch.manual_seed(0)
generator = torch.Generator(device=DEVICE).manual_seed(0)
# Small but representative GDN config (Qwen3Next defaults are K=128, V=128).
B = 32
H = 4
HV = 8 # grouped value attention: HV must be divisible by H
B = batch
H = num_key_heads
HV = num_value_heads
K = 128
V = 128
qkv_dim = 2 * (H * K) + (HV * V)
@@ -38,22 +53,33 @@ def test_fused_recurrent_packed_decode_matches_reference(
if strided_mixed_qkv:
# Simulate a packed view into a larger projection buffer:
# mixed_qkv.stride(0) > mixed_qkv.shape[1]
proj = torch.randn((B, qkv_dim + 64), device=device, dtype=dtype)
proj = torch.randn(
(B, qkv_dim + 64), device=device, dtype=dtype, generator=generator
)
mixed_qkv = proj[:, :qkv_dim]
else:
mixed_qkv = torch.randn((B, qkv_dim), device=device, dtype=dtype)
mixed_qkv = torch.randn(
(B, qkv_dim), device=device, dtype=dtype, generator=generator
)
a = torch.randn((B, HV), device=device, dtype=dtype)
b = torch.randn((B, HV), device=device, dtype=dtype)
A_log = torch.randn((HV,), device=device, dtype=dtype)
dt_bias = torch.randn((HV,), device=device, dtype=dtype)
a = torch.randn((B, HV), device=device, dtype=dtype, generator=generator)
b = torch.randn((B, HV), device=device, dtype=dtype, generator=generator)
A_log = torch.randn((HV,), device=device, dtype=dtype, generator=generator)
dt_bias = torch.randn((HV,), device=device, dtype=dtype, generator=generator)
# Continuous batching indices (include PAD_SLOT_ID=-1 cases). Index 0 is
# reserved as NULL_BLOCK_ID (CUDA graph padding), so valid slots start at 1.
ssm_state_indices = torch.arange(1, B + 1, device=device, dtype=torch.int32)
ssm_state_indices[-3:] = -1
if B > 3:
ssm_state_indices[-3:] = -1
ssm_state_indices[-1] = 0
state0 = torch.randn((B + 1, HV, V, K), device=device, dtype=dtype)
state0 = torch.randn(
(B + 1, HV, V, K),
device=device,
dtype=state_dtype,
generator=generator,
)
state_ref = state0.clone()
state_packed = state0.clone()
@@ -13,10 +13,19 @@ if current_platform.is_rocm():
)
import flashinfer.gdn_prefill # noqa: E402
import torch.nn.functional as F # noqa: E402
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
fi_chunk_gated_delta_rule,
) # noqa: E402
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
plan_flashinfer_checkpoints,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
@@ -51,3 +60,130 @@ def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
assert captured_cu_seqlens.dtype == torch.int64
assert output.shape == q.shape
assert final_state is None
@pytest.mark.skipif(
not (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
),
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
)
@pytest.mark.parametrize("use_cp", [True, False])
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
"""Each exported checkpoint is the state of the prefill stopped there.
The leading decode row is outside the prefill kernel. The two checkpointed
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
so the exporter must pick each state out of the shared emitted buffer;
the third prefill takes no checkpoint. Both FlashInfer routes are covered
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
slots after the window, which the conv kernels leave alone.
"""
torch.manual_seed(0)
device = "cuda"
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
num_spec = 2
offsets = [0, 128, 0, 192]
query_lens = [200, 50, 300]
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
num_blocks = 8
num_tokens = sum(query_lens)
q = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
k = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
initial_state = 0.1 * torch.randn(
len(query_lens), num_v_heads, head_dim, head_dim, device=device
)
prefill_cu = [0]
for query_len in query_lens:
prefill_cu.append(prefill_cu[-1] + query_len)
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
state_checkpoints = torch.empty(
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
)
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=torch.tensor(prefill_cu, device=device),
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
checkpoint_every_n_tokens=every_n_tokens,
use_cp=use_cp,
)
# The exporter sees the decode row's token in front of the prefills.
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
conv_state = torch.zeros(
num_blocks,
state_len + num_spec,
conv_dim,
device=device,
dtype=torch.bfloat16,
).transpose(-1, -2)
ssm_state = torch.zeros(
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
)
MambaPrefillCheckpointExporter(state_len=state_len).export(
MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
state_indices=torch.tensor(state_indices, device=device),
),
raw_qkv=raw_qkv,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=cu_seqlens,
recurrent_rows=torch.tensor(rows, device=device),
)
written = []
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
if not offset:
continue
written.append(block)
seq = row - 1
start = prefill_cu[seq]
_, expected_state = fi_chunk_gated_delta_rule(
q=q[:, start : start + offset],
k=k[:, start : start + offset],
v=v[:, start : start + offset],
g=g[:, start : start + offset],
beta=beta[:, start : start + offset],
initial_state=initial_state[seq : seq + 1],
output_final_state=True,
cu_seqlens=torch.tensor([0, offset], device=device),
use_qk_l2norm_in_kernel=False,
)
torch.testing.assert_close(
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
)
end = cu_seqlens[row].item() + offset
torch.testing.assert_close(
conv_state[block, :, :state_len],
raw_qkv[end - state_len : end].T,
atol=0,
rtol=0,
)
assert not conv_state[block, :, state_len:].any()
untouched = [i for i in range(num_blocks) if i not in written]
assert not ssm_state[untouched].any()
assert not conv_state[untouched].any()
@@ -178,7 +178,9 @@ def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
model="Qwen/Qwen3.5-0.8B",
revision="2fc06364715b967f1860aea9cf38778875588b17",
max_model_len=1024,
)
),
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
additional_config={"gdn_prefill_backend": "triton"},
)
with set_current_vllm_config(config):
prefill = ChunkGatedDeltaRule()
@@ -114,23 +114,27 @@ def test_checkpoint_does_not_perturb_the_final_state():
)
@pytest.mark.parametrize("spec_slots", [0, 3])
@pytest.mark.parametrize("dim_first", [True, False])
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots):
"""The store reads the checkpoint row out of every chunk state.
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
program's own row, so the store only passes if ``chunk_idx`` is used.
Row 1 declines and must write nothing. The conv input is a column slice of
a wider projection, the SD conv layout is a transposed view, and SSM rows
are padded pages, matching the strides the mixer passes.
are padded pages, matching the strides the mixer passes. Spec decoding
widens the conv state, and the store must still fill only the first
``state_len`` slots with the newest inputs.
"""
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
projection = torch.randn(64, dim + 8, device=DEVICE)
conv_input = projection[:, :dim]
num_slots = state_len + spec_slots
if dim_first:
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
conv_state = torch.zeros(3, dim, num_slots, device=DEVICE)
else:
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2)
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
# Blocks are pages holding more than the SSM state, so rows are padded.
row = heads * head_dim * d_state
@@ -152,12 +156,14 @@ def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
varlen_states,
ssm_state,
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
state_len=state_len,
)
exact = dict(rtol=0, atol=0)
torch.testing.assert_close(
conv_state[2], conv_input[32 - state_len : 32].T, **exact
conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact
)
assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots"
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
assert not conv_state[:2].any() and not ssm_storage[:2].any()
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
+6 -4
View File
@@ -96,15 +96,17 @@ def mixer2_gated_norm_tensor_parallel(
gate_states = torch.randn(batch_size, seq_len, hidden_size)
# create gated-norm with TP
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
with ensure_current_vllm_config():
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
mixer.weight.weight_loader(mixer.weight, weight) # load
# create gated-norm without TP to compute reference
# - utilize mock patching to disable TP when
with (
ensure_current_vllm_config(),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_world_size",
+24
View File
@@ -637,3 +637,27 @@ def test_replayssm_physical_ring_shape(
(8, expected_ring_len),
(2, expected_ring_len, 16),
)
def test_replayssm_ring_shape_extends_groups_for_head_shards():
"""A single group under TP is replicated to every head shard, as in
``mamba2_state_shape``, so each rank's B_cache holds one group."""
base_shapes = MambaStateShapeCalculator.mamba2_state_shape(
tp_world_size=2,
intermediate_size=256,
n_groups=1,
num_heads=4,
head_dim=64,
state_size=128,
conv_kernel=4,
)
shapes = MambaStateShapeCalculator.append_replayssm_ring(
base_shapes,
n_groups=1,
tp_world_size=2,
logical_window=16,
backend=MambaBackendEnum.TRITON,
)
assert shapes[4] == (1, 16, 128)
+5 -2
View File
@@ -439,13 +439,16 @@ def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carrie
return
# The post mapping and the collapse are what the unfused kernels produce.
# The two kernels may round a near-midpoint residual differently (FMA
# contraction is up to the compiler), so allow one bf16 ulp there and
# check the collapse against the fused kernel's own residual.
residual_ref = torch.ops.vllm.mhc_post_tilelang(
x, residual, post_layer_mix, comb_res_mix
)
layer_input_ref = mhc_pre_delayed_tilelang(
residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
residual_cur, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
)[2]
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0)
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=2**-7)
torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0)
post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref(
@@ -167,6 +167,9 @@ class Config:
vllm_config.model_config = SimpleNamespace(
enforce_eager=True,
is_moe=True,
enable_sleep_mode=False,
sleep_mode_backend="cumem",
sleep_mode_offload_cudagraph=False,
)
vllm_config.parallel_config.data_parallel_size = self.world_size
vllm_config.parallel_config.enable_expert_parallel = True
@@ -722,6 +725,9 @@ def _maybe_convert_weights_for_experts(
rank_weights: WeightTensors,
) -> WeightTensors:
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
@@ -734,7 +740,7 @@ def _maybe_convert_weights_for_experts(
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
# and per-token schemes. Those keep JIT-compiling until
# https://github.com/ROCm/aiter/issues/5766 is fixed.
if fe_name == "AiterExperts" and (
if issubclass(fe_type, AiterExperts) and (
config.quant_dtype is None or config.quant_block_shape is not None
):
return _shuffle_weights_for_aiter(rank_weights)
@@ -93,6 +93,7 @@ common_float_types: list[torch.dtype | str] = [
]
common_float_and_int_types = common_float_types + [torch.int8]
nvfp4_types = ["nvfp4"]
mxfp4_types = ["mxfp4"]
fp8_types = [current_platform.fp8_dtype()]
@@ -318,20 +319,37 @@ if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(
if has_aiter():
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterFp8BlockExperts,
AiterFp8ChannelExperts,
AiterFp8TensorExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
)
register_experts(
AiterExperts,
standard_format,
# AiterExperts also supports the fully-unquantized (None, None)
# scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8.
common_float_types,
blocked_quantization_support=True,
needs_aiter=True,
)
# Registration dtypes gate which quant configs the modular-kernel test
# harness will ever pair with a given class (see Config.is_valid()):
# they must match what each subclass's _supports_quant_scheme() actually
# declares, or the harness will generate a case the class rejects.
for aiter_experts_cls, aiter_supported_dtypes in (
(AiterUnquantizedExperts, common_float_types),
(AiterFp8BlockExperts, fp8_types),
(AiterFp8TensorExperts, fp8_types),
(AiterFp8ChannelExperts, fp8_types),
(AiterMxfp4Experts, mxfp4_types),
):
register_experts(
aiter_experts_cls,
standard_format,
aiter_supported_dtypes,
blocked_quantization_support=True,
needs_aiter=True,
)
else:
AiterExperts = None
AiterUnquantizedExperts = None
AiterFp8BlockExperts = None
AiterFp8TensorExperts = None
AiterFp8ChannelExperts = None
AiterMxfp4Experts = None
if has_deep_gemm() and is_deep_gemm_supported():
register_experts(
+2 -3
View File
@@ -3,7 +3,6 @@
"""DeepEP test utilities."""
import dataclasses
import os
import traceback
from collections.abc import Callable
from typing import Concatenate
@@ -14,7 +13,7 @@ from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUs
from typing_extensions import ParamSpec
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2
from vllm.utils.network_utils import get_open_port
from vllm.utils.network_utils import get_file_store_init_method
if has_deep_ep():
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
@@ -105,7 +104,7 @@ def parallel_launch(
world_size,
world_size,
0,
f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
get_file_store_init_method(),
worker,
)
+ args,
+83 -1
View File
@@ -179,7 +179,14 @@ def _make_b12x_moe_kernel(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=hidden_states.shape[1],
intermediate_size=w2.shape[2] * 2,
intermediate_size=w2.shape[2]
* (
32
if quant_config.weight_quant_dtype == "q8_0"
else 256
if quant_config.weight_quant_dtype in ("iq2_xs", "iq2_xxs")
else 2
),
in_dtype=hidden_states.dtype,
activation=activation,
)
@@ -1272,3 +1279,78 @@ def test_b12x_moe_cuda_graph_replay(
assert torch.isfinite(expected).all()
assert torch.isfinite(actual).all()
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
@pytest.mark.skipif(not _has_b12x_moe(), reason="requires b12x MoE on SM120")
@pytest.mark.parametrize("codec", ["iq2_xxs", "q8_0"])
@pytest.mark.parametrize("tokens", [1, 2, 4])
@torch.inference_mode()
def test_b12x_block_quant_relu2_moe_graph(codec, tokens, workspace_init):
from b12x.testing.iq2_xs_reference import moe_reference_iq2_xs
from tests.model_executor.kernels.test_b12x_linear import _block_weights
with set_current_vllm_config(VllmConfig()):
torch.manual_seed(71)
e, h, i, topk = 4, 256, 256, 2
raw1, raw2 = _block_weights(e * i, h, codec), _block_weights(e * h, i, codec)
w1 = raw1.view(e, i, *raw1.shape[1:]).cuda()
w2 = raw2.view(e, h, *raw2.shape[1:]).cuda()
x = torch.randn(tokens, h, device="cuda", dtype=torch.bfloat16) * 0.125
scores = torch.randn(tokens, e, device="cuda", dtype=torch.bfloat16)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantDesc
quant = FusedMoEQuantConfig(
_a1=FusedMoEQuantDesc(),
_a2=FusedMoEQuantDesc(),
_w1=FusedMoEQuantDesc(dtype=codec),
_w2=FusedMoEQuantDesc(dtype=codec),
)
activation = MoEActivation.RELU2_NO_MUL
kernel = _make_b12x_moe_kernel(x, w1, w2, topk, activation, quant)
weights, ids, _ = fused_topk(x, scores, topk, renormalize=False)
def apply():
return kernel.apply(
hidden_states=x,
w1=w1,
w2=w2,
topk_weights=weights,
topk_ids=ids,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
)
apply()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
output = apply()
for _ in range(2):
x.normal_(std=0.125)
scores.normal_()
new_weights, new_ids, _ = fused_topk(x, scores, topk, renormalize=False)
weights.copy_(new_weights)
ids.copy_(new_ids)
allocated = torch.accelerator.memory_allocated()
graph.replay()
torch.accelerator.synchronize()
assert torch.accelerator.memory_allocated() == allocated
expected = moe_reference_iq2_xs(
x,
raw1.view(e, i, *raw1.shape[1:]),
raw2.view(e, h, *raw2.shape[1:]),
ids,
weights,
activation="relu2",
)
assert torch.isfinite(output).all()
relative_l2 = (
output.float() - expected.float()
).norm() / expected.float().norm()
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), expected.float().flatten(), dim=0
)
assert cosine >= 0.999 and relative_l2 <= 0.01
graph.reset()
+7 -1
View File
@@ -119,6 +119,8 @@ class BatchedMMTensors:
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
@pytest.mark.parametrize("per_act_token_quant", [False, True])
# Tuned configs can pick a BLOCK_SIZE_K wider than the quantization block.
@pytest.mark.parametrize("fp8_block_k", [32, 256])
def test_batched_mm(
num_experts: int,
max_tokens_per_expert: int,
@@ -127,6 +129,7 @@ def test_batched_mm(
dtype: torch.dtype,
block_shape: list[int] | None,
per_act_token_quant: bool,
fp8_block_k: int,
):
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
and those tests will be skipped on unsupported hardware."""
@@ -149,6 +152,9 @@ def test_batched_mm(
if per_act_token_quant and block_shape is not None:
pytest.skip("Skip illegal quantization test.")
if fp8_block_k != 32 and not use_fp8_w8a8:
pytest.skip("BLOCK_SIZE_K is only varied for fp8.")
if dtype.itemsize == 1:
act_dtype = torch.bfloat16
quant_dtype = dtype
@@ -214,7 +220,7 @@ def test_batched_mm(
config={
"BLOCK_SIZE_M": 16,
"BLOCK_SIZE_N": 16,
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else 32,
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else fp8_block_k,
},
per_act_token_quant=per_act_token_quant,
block_shape=block_shape,
+196 -7
View File
@@ -9,7 +9,7 @@ import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_config
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import (
@@ -26,8 +26,10 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
fp8_w8a8_moe_quant_config,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
@@ -41,7 +43,10 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
)
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.fused_moe.utils import (
moe_kernel_quantize_input,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -183,6 +188,189 @@ def test_cutlass_moe_permutation_maps_padding_to_zero():
torch.testing.assert_close(restored[3], torch.zeros_like(restored[3]))
def test_remap_topk_ids_to_local():
device = "cuda"
expert_map = torch.tensor([-1, 0, -1, 1, -1, 2], dtype=torch.int32, device=device)
topk_ids = torch.tensor([[1, 3], [0, 5], [-1, 4]], dtype=torch.int32, device=device)
expected = torch.tensor(
[[0, 1], [-1, 2], [-1, -1]], dtype=torch.int32, device=device
)
actual = remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
torch.testing.assert_close(actual, expected)
def test_remap_topk_ids_to_local_requires_contiguous_topk_ids():
expert_map = torch.tensor([0, 1], dtype=torch.int32)
topk_ids = torch.tensor([[0, 1], [1, 0]], dtype=torch.int32).t()
assert not topk_ids.is_contiguous()
with pytest.raises(AssertionError):
remap_topk_to_local(topk_ids, expert_map)
def test_cutlass_fp4_moe_supports_expert_parallel_config():
ep_config = FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
)
assert CutlassExpertsMxfp4._supports_parallel_config(ep_config)
assert CutlassExpertsFp4._supports_parallel_config(ep_config)
def _slice_moe_quant_config(
quant_config: FusedMoEQuantConfig, start: int, end: int
) -> FusedMoEQuantConfig:
num_experts = quant_config.w1_scale.shape[0]
sliced_config = copy.deepcopy(quant_config)
for desc_name in ("_a1", "_a2", "_w1", "_w2"):
desc = getattr(sliced_config, desc_name)
for field_name in ("scale", "alpha_or_gscale", "bias", "zp"):
tensor = getattr(desc, field_name)
if (
isinstance(tensor, torch.Tensor)
and tensor.ndim > 0
and tensor.shape[0] == num_experts
):
setattr(desc, field_name, tensor[start:end])
return sliced_config
def _swizzle_mxfp4_moe_scales(
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
e, w1_n, w1_scale_k = w1_scale.shape
w1_k = w1_scale_k * 32
w2_m, w2_scale_n = w2_scale.shape[1], w2_scale.shape[2]
w2_n = w2_scale_n * 32
swizzled_w1 = []
swizzled_w2 = []
for expert in range(e):
swizzled_w1.append(
swizzle_mxfp4_scales(w1_scale[expert], w1_n, w1_k).reshape(w1_n, w1_scale_k)
)
swizzled_w2.append(
swizzle_mxfp4_scales(w2_scale[expert], w2_m, w2_n).reshape(w2_m, w2_scale_n)
)
return torch.stack(swizzled_w1), torch.stack(swizzled_w2)
def _make_cutlass_fp4_moe_weights(
quantization: str,
num_experts: int,
n: int,
k: int,
dtype: torch.dtype,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
if quantization == "nvfp4":
return make_test_quant_config(
num_experts,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
make_gate=True,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
w1, w1_scale = mxfp4_quantize(
torch.randn((num_experts, 2 * n, k), device=device, dtype=dtype) / 8
)
w2, w2_scale = mxfp4_quantize(
torch.randn((num_experts, k, n), device=device, dtype=dtype) / 8
)
w1_scale, w2_scale = _swizzle_mxfp4_moe_scales(w1_scale, w2_scale)
return (
w1.contiguous(),
w2.contiguous(),
mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale),
)
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
@torch.inference_mode()
def test_cutlass_fp4_moe_EP(
quantization: str,
workspace_init,
):
"""Sharded CUTLASS FP4/MXFP4 MoE with expert_map must match full weights."""
experts_cls = CutlassExpertsFp4 if quantization == "nvfp4" else CutlassExpertsMxfp4
if not experts_cls._supports_current_device():
pytest.skip(f"CUTLASS {quantization} MoE is not supported on this GPU")
set_random_seed(7)
m, n, k = 8, 128, 128
num_experts, num_local_experts, topk = 8, 2, 2
dtype = torch.bfloat16
device = torch.device("cuda")
w1, w2, quant_config = _make_cutlass_fp4_moe_weights(
quantization, num_experts, n, k, dtype, device
)
hidden_states = torch.randn((m, k), device=device, dtype=dtype) / 10
score = torch.randn((m, num_experts), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score, topk, renormalize=False
)
moe_config = make_dummy_moe_config(
num_experts=num_experts,
num_local_experts=num_experts,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=dtype,
max_num_tokens=m,
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts_cls(moe_config=moe_config, quant_config=quant_config),
)
apply_kwargs = {
"hidden_states": hidden_states,
"w1": w1,
"w2": w2,
"topk_weights": topk_weights,
"topk_ids": topk_ids,
"global_num_experts": num_experts,
"activation": MoEActivation.SILU,
"expert_map": None,
"apply_router_weight_on_input": False,
}
reference = kernel.apply(**apply_kwargs)
sharded = run_with_expert_maps(
num_experts,
num_local_experts,
quant_config,
experts_cls,
**apply_kwargs,
)
assert reference.abs().max() > 0
torch.testing.assert_close(sharded, reference, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
@torch.inference_mode()
def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization: str):
@@ -486,6 +674,7 @@ def run_with_expert_maps(
num_experts: int,
num_local_experts: int,
quant_config: FusedMoEQuantConfig,
experts_cls: type[mk.FusedMoEExperts],
**cutlass_moe_kwargs,
):
def slice_experts():
@@ -513,9 +702,7 @@ def run_with_expert_maps(
for k, t in full_tensors.items():
cutlass_moe_kwargs[k] = t[s:e]
new_quant_config = copy.deepcopy(quant_config)
new_quant_config._w1.scale = quant_config.w1_scale[s:e]
new_quant_config._w2.scale = quant_config.w2_scale[s:e]
new_quant_config = _slice_moe_quant_config(quant_config, s, e)
yield cutlass_moe_kwargs, new_quant_config
@@ -523,13 +710,14 @@ def run_with_expert_maps(
for kwargs, new_quant_config in slice_experts():
w2 = kwargs["w2"]
a = kwargs["hidden_states"]
w1 = kwargs["w1"]
moe_config = make_dummy_moe_config(
max_num_tokens=kwargs.get("hidden_states").shape[0],
experts_per_token=kwargs.get("topk_ids").shape[1],
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_dim=w2.shape[1],
intermediate_size=w2.shape[2],
intermediate_size=w1.shape[1] // 2,
in_dtype=a.dtype,
)
kernel = mk.FusedMoEKernel(
@@ -539,7 +727,7 @@ def run_with_expert_maps(
allow_new_interface=True,
use_monolithic=False,
),
CutlassExpertsFp8(
experts_cls(
moe_config=moe_config,
quant_config=new_quant_config,
),
@@ -624,6 +812,7 @@ def run_8_bit(
num_experts,
num_local_experts,
quant_config,
CutlassExpertsFp8,
**kwargs,
)
@@ -0,0 +1,422 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
ProcessGroupInfo,
parallel_launch_with_config,
)
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.expert_substitution import (
ConstantExpertSubstitution,
SubstitutedRoutedExperts,
get_expert_substitution_spec,
make_expert_substitution,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
FlashInferExperts,
)
from vllm.model_executor.models.mixtral import MixtralMoE
from vllm.platforms import current_platform
from vllm.v1.worker.workspace import init_workspace_manager
MOE_PREFIX = "model.layers.1.mlp.experts"
def _model_config(approximate_experts, **kwargs) -> SimpleNamespace:
return SimpleNamespace(
hf_config=SimpleNamespace(approximate_experts=approximate_experts),
**kwargs,
)
def _expert_substitution(targets: dict[str, dict[int, str]]) -> dict:
"""The ``compression_config.transform_config.expert_substitution`` format."""
return {
"version": 1,
"router_semantics": {
"preserve_logical_expert_ids": True,
"preserve_router_weights": True,
"renormalize_after_substitution": False,
},
"targets": {
path: {
"num_logical_experts": 5,
"weight_layout": "compact_retained_experts",
"replacements": {
str(expert_id): {
"format": "constant-v1",
"tensors": {"value": name},
}
for expert_id, name in replacements.items()
},
}
for path, replacements in targets.items()
},
}
def _compressed_model_config(expert_substitution: dict, **kwargs) -> SimpleNamespace:
return SimpleNamespace(
hf_config=SimpleNamespace(
compression_config={
"transform_config": {"expert_substitution": expert_substitution}
}
),
**kwargs,
)
def _substitution(hidden_size: int, dtype=torch.float32):
# Experts 1 and 3 of 5 are substituted; 0, 2, 4 map to physical 0, 1, 2.
return ConstantExpertSubstitution(1, 5, (1, 3), hidden_size, dtype)
def test_parse_approximate_experts():
assert get_expert_substitution_spec(None) is None
assert get_expert_substitution_spec(_model_config(None)) is None
spec = get_expert_substitution_spec(_model_config({"1": [3, 1], "2": [], 5: (0,)}))
assert spec.experts == {1: (1, 3), 5: (0,)}
assert spec.value_names == {}
text_config = SimpleNamespace(
hf_text_config=SimpleNamespace(approximate_experts={"0": [2]}),
hf_config=SimpleNamespace(),
)
assert get_expert_substitution_spec(text_config).experts == {0: (2,)}
@pytest.mark.parametrize(
"approximate_experts",
[[1, 2], {"layer": [1]}, {"1": [1, 1]}, {"1": [-1]}, {"1": ["x"]}],
)
def test_parse_approximate_experts_rejects_invalid_entries(approximate_experts):
with pytest.raises(ValueError, match="approximate_experts"):
get_expert_substitution_spec(_model_config(approximate_experts))
def test_parse_expert_substitution_with_explicit_value_names():
spec = get_expert_substitution_spec(
_compressed_model_config(
_expert_substitution(
{
MOE_PREFIX: {3: "shared", 1: "layer_1.expert_1"},
"model.layers.2.mlp.experts": {0: "shared"},
}
)
)
)
assert spec.experts == {1: (1, 3), 2: (0,)}
assert spec.value_names == {
"shared": ((1, 3), (2, 0)),
"layer_1.expert_1": ((1, 1),),
}
@pytest.mark.parametrize(
"mutate",
[
lambda raw: raw.update(version=2),
lambda raw: raw["router_semantics"].update(renormalize_after_substitution=True),
lambda raw: raw["targets"].update({"mlp.experts": raw["targets"][MOE_PREFIX]}),
lambda raw: raw["targets"][MOE_PREFIX].update(weight_layout="dense"),
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].update(
format="linear-v1"
),
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].pop("tensors"),
],
)
def test_parse_expert_substitution_rejects_unsupported_metadata(mutate):
raw = _expert_substitution({MOE_PREFIX: {1: "value"}})
mutate(raw)
with pytest.raises(ValueError, match="invalid expert_substitution metadata"):
get_expert_substitution_spec(_compressed_model_config(raw))
def test_parse_rejects_both_formats():
config = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
config.hf_config.approximate_experts = {"1": [1]}
with pytest.raises(ValueError, match="both"):
get_expert_substitution_spec(config)
def test_make_expert_substitution_matches_layer_index():
config = _model_config({"1": [3, 1]})
substitution = make_expert_substitution(config, MOE_PREFIX, 5, 16, torch.bfloat16)
assert substitution is not None
assert substitution.layer_idx == 1
assert substitution.substituted_expert_ids == (1, 3)
assert substitution.num_compute_experts == 3
assert substitution.values.dtype == torch.bfloat16
assert [substitution.physical_expert_id(i) for i in range(5)] == [0, -1, 1, -1, 2]
assert (
make_expert_substitution(config, "model.layers.2.mlp.experts", 5, 16, None)
is None
)
assert make_expert_substitution(None, MOE_PREFIX, 5, 16, None) is None
explicit = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
substitution = make_expert_substitution(explicit, MOE_PREFIX, 5, 16, None)
assert substitution is not None
assert substitution.substituted_expert_ids == (1,)
with pytest.raises(ValueError, match="exactly one layer index"):
make_expert_substitution(config, "mlp.experts", 5, 16, None)
with pytest.raises(ValueError, match="within"):
make_expert_substitution(config, MOE_PREFIX, 3, 16, None)
with pytest.raises(ValueError, match="retain at least one expert"):
make_expert_substitution(_model_config({"1": [0, 1]}), MOE_PREFIX, 2, 16, None)
def test_constant_substitution_loads_values():
substitution = _substitution(2)
substitution.weight_loader(substitution.values, torch.tensor([1.0, 2.0]), 3)
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
substitution.validate_loaded_values("layer")
substitution.weight_loader(substitution.values, torch.tensor([3.0, 4.0]), 1)
substitution.validate_loaded_values("layer")
torch.testing.assert_close(
substitution.values, torch.tensor([[3.0, 4.0], [1.0, 2.0]])
)
with pytest.raises(ValueError, match="not listed"):
substitution.weight_loader(substitution.values, torch.ones(2), 2)
with pytest.raises(ValueError, match="has shape"):
substitution.weight_loader(substitution.values, torch.ones(3), 1)
@pytest.mark.parametrize("num_tokens", [0, 1, 3, 7])
@pytest.mark.parametrize("hidden_size", [2, 4])
def test_constant_substitution_transforms_routes_and_computes_side_output(
num_tokens, hidden_size
):
substitution = _substitution(2)
substitution.values.data.copy_(torch.tensor([[10.0, 20.0], [30.0, 40.0]]))
rows = torch.arange(num_tokens) % 3
topk_ids = torch.tensor([[1, 2], [4, 3], [99, -1]])[rows]
topk_weights = torch.tensor([[0.25, 0.75], [0.6, 0.4], [0.5, 0.5]])[rows]
constant_output = substitution.transform_routes(
torch.zeros(num_tokens, hidden_size), topk_weights, topk_ids
)
torch.testing.assert_close(
topk_weights, torch.tensor([[0.0, 0.75], [0.6, 0.0], [0.0, 0.0]])[rows]
)
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1], [2, 0], [0, 0]])[rows])
expected = torch.zeros(num_tokens, hidden_size)
expected[:, :2] = torch.tensor([[2.5, 5.0], [12.0, 16.0], [0.0, 0.0]])[rows]
torch.testing.assert_close(constant_output, expected)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
"device",
[
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="requires a CUDA-like platform",
),
),
],
)
def test_constant_substitution_preserves_router_weight_precision(dtype, device):
substitution = _substitution(1, dtype).to(device)
substitution.values.data.copy_(
torch.tensor([[60000.0], [-60000.0]], dtype=dtype, device=device)
)
topk_weights = torch.tensor([[0.5001, 0.4999]], dtype=torch.float32, device=device)
topk_ids = torch.tensor([[1, 3]], device=device)
# Nearly cancelling constants must retain the difference between FP32 weights.
expected = (topk_weights.double() @ substitution.values.double()).to(dtype)
actual = substitution.transform_routes(
torch.zeros(1, 1, dtype=dtype, device=device), topk_weights, topk_ids
)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
def _bare_substituted_routed_experts(**attrs) -> SubstitutedRoutedExperts:
layer = SubstitutedRoutedExperts.__new__(SubstitutedRoutedExperts)
torch.nn.Module.__init__(layer)
layer.expert_substitution = _substitution(2)
for name, value in attrs.items():
setattr(layer, name, value)
return layer
@pytest.mark.parametrize("tp_rank", [0, 1])
@pytest.mark.parametrize("padded_input", [False, True])
def test_substituted_routed_experts_adds_constant_on_one_tp_rank(tp_rank, padded_input):
def apply(*, layer, x, topk_weights, topk_ids, **kwargs):
torch.testing.assert_close(topk_weights, torch.tensor([[0.0, 0.75]]))
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1]]))
return torch.ones(x.shape[0], 2)
layer = _bare_substituted_routed_experts(
moe_config=SimpleNamespace(tp_rank=tp_rank),
quant_method=SimpleNamespace(is_monolithic=False, apply=apply),
)
layer.expert_substitution.values.data.copy_(
torch.tensor([[10.0, 20.0], [30.0, 40.0]])
)
actual = layer.forward_modular(
torch.zeros(1, 4 if padded_input else 2),
torch.tensor([[0.25, 0.75]]),
torch.tensor([[1, 2]]),
)
expected = torch.ones(1, 2)
if tp_rank == 0:
expected += torch.tensor([[2.5, 5.0]])
torch.testing.assert_close(actual, expected)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
)
def test_standard_fused_moe_model_discovers_substitution(dist_init):
vllm_config = VllmConfig()
vllm_config.model_config = _model_config(
{"0": [1]}, dtype=torch.bfloat16, is_moe=True
)
vllm_config.kernel_config.moe_backend = "triton"
with set_current_vllm_config(vllm_config):
moe = MixtralMoE(
num_experts=4,
top_k=2,
hidden_size=256,
intermediate_size=512,
params_dtype=torch.bfloat16,
prefix="model.layers.0.block_sparse_moe",
)
routed_experts = moe.experts.routed_experts
assert isinstance(routed_experts, SubstitutedRoutedExperts)
assert routed_experts.expert_substitution.substituted_expert_ids == (1,)
assert routed_experts.w13_weight.shape[0] == 3
assert moe.experts.moe_config.require_decomposed_backend
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
)
@pytest.mark.parametrize(
"moe_backend",
[
pytest.param("triton", id="triton"),
pytest.param(
"flashinfer_cutlass",
id="flashinfer-cutlass",
marks=pytest.mark.skipif(
not FlashInferExperts._supports_current_device(),
reason="FlashInfer CUTLASS is unavailable on this platform",
),
),
],
)
def test_fused_moe_combines_compact_experts_and_substitution(dist_init, moe_backend):
hidden_size = 256
vllm_config = VllmConfig()
vllm_config.model_config = _model_config(
{"0": [1, 3]}, dtype=torch.bfloat16, is_moe=True
)
vllm_config.compilation_config.static_forward_context = {}
vllm_config.kernel_config.moe_backend = moe_backend
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
init_workspace_manager(torch.accelerator.current_device_index())
layer = FusedMoEFactory(
num_experts=4,
top_k=2,
hidden_size=hidden_size,
intermediate_size=512,
params_dtype=torch.bfloat16,
prefix="model.layers.0.mlp.experts",
renormalize=False,
).cuda()
substitution = layer.routed_experts.expert_substitution
assert layer.routed_experts.w13_weight.shape[0] == 2
with torch.no_grad():
layer.routed_experts.w13_weight.normal_(0, 0.01)
layer.routed_experts.w2_weight.normal_(0, 0.01)
substitution.values.normal_(0, 0.01)
layer._quant_method.process_weights_after_loading(layer.routed_experts)
hidden_states = torch.randn(8, hidden_size, dtype=torch.bfloat16, device="cuda")
router_logits = torch.randn(8, 4, dtype=torch.float32, device="cuda")
topk_weights, topk_ids = layer.router.select_experts(
hidden_states.clone(), router_logits
)
expected = substitution.transform_routes(hidden_states, topk_weights, topk_ids)
expected += layer._quant_method.apply(
layer=layer.routed_experts,
x=hidden_states.clone(),
topk_weights=topk_weights,
topk_ids=topk_ids,
shared_experts=None,
shared_experts_input=None,
)
get_forward_context().all_moe_layers = None
actual = layer(hidden_states.clone(), router_logits)
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
def _tp_substitution_worker(
process_group: ProcessGroupInfo,
vllm_config: VllmConfig,
_cpu_group,
) -> None:
with set_forward_context(None, vllm_config):
init_workspace_manager(process_group.local_rank)
layer = FusedMoEFactory(
num_experts=4,
top_k=1,
hidden_size=256,
intermediate_size=512,
params_dtype=torch.bfloat16,
prefix="model.layers.0.mlp.experts",
renormalize=True,
)
with torch.no_grad():
layer.routed_experts.w13_weight.zero_()
layer.routed_experts.w2_weight.zero_()
layer.routed_experts.expert_substitution.values.fill_(0.25)
layer._quant_method.process_weights_after_loading(layer.routed_experts)
hidden_states = torch.zeros(2, 256, dtype=torch.bfloat16)
router_logits = torch.tensor(
[[-100.0, 100.0, -100.0, -100.0]] * 2,
dtype=torch.float32,
)
get_forward_context().all_moe_layers = None
actual = layer(hidden_states, router_logits)
torch.testing.assert_close(
actual, torch.full_like(actual, 0.25), atol=0, rtol=0
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike() or current_platform.device_count() < 2,
reason="requires two CUDA-like devices",
)
def test_constant_substitution_is_added_once_with_tp2():
vllm_config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=2))
vllm_config.model_config = _model_config(
{"0": [1]},
dtype=torch.bfloat16,
is_moe=True,
sleep_mode_offload_cudagraph=False,
)
vllm_config.compilation_config.static_forward_context = {}
vllm_config.kernel_config.moe_backend = "triton"
parallel_launch_with_config(2, _tp_substitution_worker, vllm_config, None)
@@ -10,9 +10,14 @@ CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
32). The expected output is then known in closed form and compared bit for
bit.
A test that needs no SM100 also checks the adapter's kwargs against the
installed FlashInfer's config types, so config drift fails on any CI runner
with FlashInfer installed.
"""
from dataclasses import dataclass
from types import SimpleNamespace
import pytest
import torch
@@ -22,6 +27,7 @@ from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_devic
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
@@ -305,3 +311,59 @@ def test_init_warmup_forward_is_exact_on_grid_inputs(
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs")
distributed_run(_init_warmup_forward, world_size, geometry, backend)
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
def test_adapter_kwargs_match_installed_flashinfer(
monkeypatch: pytest.MonkeyPatch, backend: str
):
"""The adapter's kwargs construct the installed FlashInfer's config types.
Needs no SM100, so config kwarg drift fails in CI rather than at model load.
Fakes only the megakernel layer (symmetric memory), the EP group, the device
index and the DeepGEMM import.
"""
moe_ep = pytest.importorskip("flashinfer.moe_ep")
layer_args: list[tuple] = []
class FakeMegaLayer:
def __init__(self, *args) -> None:
layer_args.append(args)
def destroy(self) -> None:
pass
monkeypatch.setattr(moe_ep, "MoEEpMegaLayer", FakeMegaLayer)
monkeypatch.setattr(fi_ep, "_expose_deep_gemm_to_flashinfer", lambda: None)
monkeypatch.setattr(
fi_ep,
"get_ep_group",
lambda: SimpleNamespace(world_size=2, rank_in_group=0, device_group=None),
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0)
geometry = GEOMETRIES[0]
moe = SimpleNamespace(
moe_backend=backend,
num_experts=geometry.num_experts,
experts_per_token=geometry.top_k,
hidden_dim=geometry.hidden_size,
intermediate_size=geometry.intermediate_size,
max_num_tokens=MAX_TOKENS_PER_RANK,
swiglu_limit=10.0,
routing_method=_routing_method(backend),
)
weight = torch.zeros(1, dtype=torch.bfloat16)
FlashInferMoeEp(
moe,
FlashInferMoeEpWeights(w13=weight, w2=weight),
apply_topk_in_fc1=apply_topk_in_fc1(moe),
)
((_, _, _, mega_config),) = layer_args
expected = (
moe_ep.Nvfp4CutedslMegaMoeConfig
if backend == FLASHINFER_MOE_EP_CUTEDSL
else moe_ep.DeepGemmMegaMoeConfig
)
assert isinstance(mega_config.megakernel, expected)
@@ -0,0 +1,229 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness of the fused globalize + align + sort decode metadata kernel.
The fused op replaces the DeepEP-v2 INDEXED cudagraph-decode chain
(_globalize_recv_topk_idx + moe_align_block_size + count_and_sort). It is
checked against that exact reference chain over a grid of shapes / EP ranks /
fill distributions.
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
fused_globalize_align_block_size,
)
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
moe_align_block_size,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_ep_v2
if current_platform.is_cuda() and has_deep_ep_v2():
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
_globalize_recv_topk_idx,
)
else:
pytest.skip("Requires CUDA and DeepEP v2", allow_module_level=True)
def _expert_map(global_num_experts, rank_expert_offset, local_num_experts, device):
expert_map = torch.full((global_num_experts,), -1, device=device, dtype=torch.int32)
rng = torch.arange(local_num_experts, device=device, dtype=torch.int32)
expert_map[rank_expert_offset : rank_expert_offset + local_num_experts] = rng
return expert_map
def _make_case(n, topk, ep_size, local_e, block_size, seed, frac_recv, rank, fill):
torch.manual_seed(seed)
device = "cuda"
global_num_experts = local_e * ep_size
rank_idx = {"first": 0, "mid": ep_size // 2, "last": ep_size - 1}[rank]
rank_expert_offset = rank_idx * local_e
if fill == "all_nonlocal":
local_idx = torch.full((n, topk), -1, device=device, dtype=torch.int64)
elif fill == "single":
local_idx = torch.zeros((n, topk), device=device, dtype=torch.int64)
elif fill == "all_local":
local_idx = torch.randint(
0, local_e, (n, topk), device=device, dtype=torch.int64
)
else:
local_idx = torch.randint(
-1, local_e, (n, topk), device=device, dtype=torch.int64
)
num_recv = max(0, min(n, int(round(n * frac_recv))))
if num_recv < n:
local_idx[num_recv:] = torch.randint(
-1, local_e, (n - num_recv, topk), device=device, dtype=torch.int64
)
psum = torch.zeros(4, device=device, dtype=torch.int32)
psum[-1] = num_recv
return local_idx, psum, rank_expert_offset, global_num_experts
def _group_by_expert(sorted_ids, expert_ids, block_size, valid_len, total):
groups: dict[int, list[int]] = {}
for b in range(valid_len // block_size):
eid = int(expert_ids[b].item())
blk = sorted_ids[b * block_size : (b + 1) * block_size]
groups.setdefault(eid, []).extend(blk[blk < total].tolist())
return groups
@pytest.mark.parametrize("n", [1, 7, 64, 257])
# topk=16 with local_num_experts=28 is the Kimi K3 decode config. topk=6 and
# block_size=16, 128 are outside the old template dispatch table: they guard
# against a regression to fixed (topk, block_size) specialization.
@pytest.mark.parametrize("topk", [1, 6, 8, 16])
@pytest.mark.parametrize("block_size", [16, 32, 64, 128])
@pytest.mark.parametrize("local_e", [28, 32])
@pytest.mark.parametrize("rank", ["first", "mid", "last"])
@pytest.mark.parametrize("fill", ["rand", "all_local", "all_nonlocal", "single"])
def test_fused_matches_reference(n, topk, block_size, local_e, rank, fill):
ep_size = 4
local_idx, psum, reo, gne = _make_case(
n,
topk,
ep_size,
local_e,
block_size,
seed=n * 131 + topk,
frac_recv=0.75,
rank=rank,
fill=fill,
)
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
topk_ids=ref_global,
block_size=block_size,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
)
fused_global, fused_sorted, fused_expert_ids, fused_num = (
fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, block_size
)
)
total = n * topk
assert torch.equal(fused_global, ref_global)
assert torch.equal(fused_num, ref_num)
valid = int(fused_num.item())
nb = valid // block_size
assert torch.equal(fused_expert_ids[:nb], ref_expert_ids[:nb])
assert bool((fused_expert_ids[nb:] == -1).all())
assert bool((fused_sorted[valid:] == total).all())
ref_groups = _group_by_expert(
ref_sorted, ref_expert_ids, block_size, int(ref_num.item()), total
)
fused_groups = _group_by_expert(
fused_sorted, fused_expert_ids, block_size, valid, total
)
assert set(ref_groups) == set(fused_groups)
for eid in ref_groups:
assert sorted(ref_groups[eid]) == sorted(fused_groups[eid])
@pytest.mark.parametrize("n", [7, 257])
def test_fused_scatter_idx_matches_reference(n):
"""The optional scatter output masks invalid and padded routes."""
local_e = 28
local_idx, psum, reo, gne = _make_case(
n, 16, 4, local_e, 64, seed=n, frac_recv=0.75, rank="mid", fill="rand"
)
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
*_, ref_scatter = moe_align_block_size(
topk_ids=ref_global,
block_size=64,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
return_scatter_idx=True,
)
fused_global, *_, fused_scatter = fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, 64, return_scatter_idx=True
)
assert torch.equal(fused_global, ref_global)
assert torch.equal(fused_scatter, ref_scatter)
# These have effective_work 1504 (single block) and 6068 (cooperative).
@pytest.mark.parametrize("n", [1, 64])
def test_fused_cuda_graph_replay(n):
"""Both launch paths remain capturable and consume replay-time inputs."""
topk = 16
local_e = 28
block_size = 64
ep_size = 4
local_idx, psum, reo, gne = _make_case(
n,
topk,
ep_size,
local_e,
block_size,
seed=n,
frac_recv=0.75,
rank="mid",
fill="rand",
)
graph_idx = torch.empty_like(local_idx)
graph_psum = torch.empty_like(psum)
# Initialize lazy CUDA state before capture.
fused_globalize_align_block_size(
local_idx.clone(), psum, reo, gne, local_e, block_size
)
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
graph_idx.copy_(local_idx)
graph_psum.copy_(psum)
graph_outputs = fused_globalize_align_block_size(
graph_idx, graph_psum, reo, gne, local_e, block_size
)
for frac_recv in (0.75, 0.5):
psum[-1] = int(round(n * frac_recv))
ref_idx = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
topk_ids=ref_idx,
block_size=block_size,
num_experts=gne,
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
ignore_invalid_experts=True,
)
graph.replay()
torch.accelerator.synchronize()
fused_idx, fused_sorted, fused_expert_ids, fused_num = graph_outputs
assert torch.equal(fused_idx, ref_idx)
assert torch.equal(fused_num, ref_num)
valid = int(fused_num.item())
num_blocks = valid // block_size
assert torch.equal(fused_expert_ids[:num_blocks], ref_expert_ids[:num_blocks])
fused_groups = _group_by_expert(
fused_sorted, fused_expert_ids, block_size, valid, n * topk
)
ref_groups = _group_by_expert(
ref_sorted,
ref_expert_ids,
block_size,
int(ref_num.item()),
n * topk,
)
assert set(fused_groups) == set(ref_groups)
for eid in ref_groups:
assert sorted(fused_groups[eid]) == sorted(ref_groups[eid])
@@ -431,11 +431,6 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
if gptoss_moe.triton_kernels_version == "3.5.1":
pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+")
if gptoss_moe.triton_kernels_version == "3.8":
pytest.skip(
"3.8 rebuilds routing from raw ids, so routing_data_from_sparse_topk "
"just forwards to make_routing_data and the comparison is vacuous"
)
from triton_kernels.topk import topk as topk_fn
@@ -467,11 +462,52 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
elif hasattr(ref, "__dict__"):
for k, v in vars(ref).items():
assert_equivalent(getattr(new, k), v, f"{path}.{k}")
elif hasattr(type(ref), "__slots__"):
for k in type(ref).__slots__:
assert_equivalent(getattr(new, k), getattr(ref, k), f"{path}.{k}")
else:
assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}"
assert_equivalent(rd_new, rd_ref)
if isinstance(gather_ref, torch.Tensor):
torch.testing.assert_close(gather_new, gather_ref)
torch.testing.assert_close(scatter_new, scatter_ref)
return
torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx)
torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx)
torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx)
torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx)
@pytest.mark.parametrize("n_tokens", [1, 33, 4099])
@pytest.mark.parametrize("n_experts,topk", [(128, 4), (256, 6)])
def test_make_routing_data_tk38_matches_reference(n_tokens, n_experts, topk):
from vllm.model_executor.layers.fused_moe.experts import (
gpt_oss_triton_kernels_moe as gptoss_moe,
)
if gptoss_moe.triton_kernels_version != "3.8":
pytest.skip("requires triton_kernels 3.8")
torch.manual_seed(0)
topk_ids = torch.stack(
[torch.randperm(n_experts, device="cuda")[:topk] for _ in range(n_tokens)]
).to(torch.int32)
topk_ids[torch.rand(topk_ids.shape, device="cuda") < 0.5] = -1
topk_weights = torch.rand(topk_ids.shape, device="cuda")
routing, gather_tok, scatter_tok = gptoss_moe.make_routing_data(
topk_ids, topk_weights, n_experts
)
flat = topk_ids.reshape(-1).to(torch.int64)
order = torch.argsort(flat.clamp(min=0), stable=True)
counts = torch.bincount(flat.clamp(min=0), minlength=n_experts).to(torch.int32)
weights = topk_weights.reshape(-1) * (flat >= 0).to(torch.float32)
assert torch.equal(routing.ragged.slice_sizes, counts)
assert torch.equal(gather_tok, (order // topk).to(torch.int32))
assert torch.equal(scatter_tok, gather_tok)
assert torch.equal(routing.gate_scal, weights[order])
assert routing.n_expts_act == topk
assert routing.n_valid == flat.numel()
@@ -16,7 +16,13 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterFp8BlockExperts,
AiterFp8ChannelExperts,
AiterFp8TensorExperts,
AiterUnquantizedExperts,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8StaticTensorSym,
@@ -80,10 +86,10 @@ def assert_aiter_quant_scheme_case(config: Config) -> None:
indirectly through the general quant-config sweep.
See https://github.com/vllm-project/vllm/issues/54966."""
fe_cls = config.fused_experts_type
if fe_cls is not AiterExperts:
if not issubclass(fe_cls, AiterExperts):
return
if config.quant_config is None:
if config.quant_config is None or config.quant_dtype is None:
w_key, a_key = None, None
else:
w_key, a_key = config.fp8_quant_key_pair()
@@ -101,7 +107,7 @@ def assert_aiter_activation_case(config: Config) -> None:
AiterExperts being reached only indirectly through the general
activation sweep. See https://github.com/vllm-project/vllm/issues/54966."""
fe_cls = config.fused_experts_type
if fe_cls is not AiterExperts:
if not issubclass(fe_cls, AiterExperts):
return
assert fe_cls._supports_activation(config.activation), (
@@ -156,7 +162,7 @@ def rank_worker(
if (
topk == 1
and config.supports_apply_weight_on_input()
and config.fused_experts_type is AiterExperts
and issubclass(config.fused_experts_type, AiterExperts)
and config.quant_block_shape is not None
):
print(
@@ -172,7 +178,7 @@ def rank_worker(
# https://github.com/vllm-project/vllm/issues/57029
if (
config.world_size > 1
and config.fused_experts_type is AiterExperts
and issubclass(config.fused_experts_type, AiterExperts)
and getattr(config.prepare_finalize_type, "__name__", "")
in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize")
):
@@ -210,7 +216,7 @@ def rank_worker(
is_aiter_fp8 = (
_cp.is_rocm()
and config.fused_experts_type is AiterExperts
and issubclass(config.fused_experts_type, AiterExperts)
and config.quant_config is not None
)
if is_aiter_fp8:
@@ -270,7 +276,7 @@ def is_nyi_config(config: Config) -> bool:
return True
if config.activation != MoEActivation.SILU:
if config.fused_experts_type is not AiterExperts:
if not issubclass(config.fused_experts_type, AiterExperts):
return True # AITER-only for this axis, for now
if config.quant_dtype is not None:
return True # unquantized-only for this axis, for now
@@ -591,7 +597,7 @@ def test_aiter_moe_sorting_backend_dispatch_env_matrix(
dtype=torch.bfloat16,
quant_config=None,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
fused_experts_type=AiterUnquantizedExperts,
world_size=1,
)
assert config.is_valid()[0]
@@ -671,7 +677,7 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
dtype=torch.bfloat16,
quant_config=None,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
fused_experts_type=AiterUnquantizedExperts,
world_size=1,
)
assert config.is_valid()[0]
@@ -738,6 +744,20 @@ _PADDING_QUANT_IDS = [
]
def _padding_matrix_fe_cls(
quant_config: TestMoEQuantConfig | None,
) -> type[AiterExperts]:
"""Map a _PADDING_QUANT_CONFIGS entry to the AiterExperts leaf class that
supports it (mirrors Config.fp8_quant_key_pair's scheme derivation)."""
if quant_config is None:
return AiterUnquantizedExperts
if quant_config.block_shape is not None:
return AiterFp8BlockExperts
if quant_config.per_out_ch_quant:
return AiterFp8ChannelExperts
return AiterFp8TensorExperts
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
gate/up halves of `real_per_half` size each."""
@@ -950,7 +970,7 @@ def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig |
dtype=torch.bfloat16,
quant_config=quant_config,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
fused_experts_type=_padding_matrix_fe_cls(quant_config),
world_size=1,
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
intermediate_size_per_partition_unpadded=(
@@ -1113,7 +1133,7 @@ def test_aiter_moe_token_padding_garbage_rows(
dtype=torch.bfloat16,
quant_config=quant_config,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
fused_experts_type=_padding_matrix_fe_cls(quant_config),
world_size=1,
)
assert config.is_valid()[0]
+68 -17
View File
@@ -40,9 +40,37 @@ MNKS = [
]
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
def _mxfp4_aiter_weights(w1, w2, start, end):
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
from aiter.ops.triton.quant import dynamic_mxfp4_quant
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_quant_config,
)
def quantize(w):
w = w[start:end]
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
backend, None, q1, q2, s1, s2
)
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterFp8ChannelExperts,
AiterMxfp4Experts,
AiterUnquantizedExperts,
rocm_aiter_fused_experts,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
MoriPrepareAndFinalize,
)
@@ -50,6 +78,8 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
rocm_aiter_ops.refresh_env_variables()
init_workspace_manager(pgi.device)
set_random_seed(47)
fp8 = scheme == "fp8_token_channel"
mxfp4 = scheme == "mxfp4"
dtype = current_platform.fp8_dtype() if fp8 else None
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
@@ -83,28 +113,44 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
moe_parallel_config=parallel,
)
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
quant = FusedMoEQuantConfig.make(
dtype,
per_act_token_quant=fp8,
per_out_ch_quant=fp8,
w1_scale=None if s1 is None else s1[start:end].contiguous(),
w2_scale=None if s2 is None else s2[start:end].contiguous(),
)
if mxfp4:
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
else:
quant = FusedMoEQuantConfig.make(
dtype,
per_act_token_quant=fp8,
per_out_ch_quant=fp8,
w1_scale=None if s1 is None else s1[start:end].contiguous(),
w2_scale=None if s2 is None else s2[start:end].contiguous(),
)
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
w1[start:end].contiguous(), w2[start:end].contiguous()
)
local_w1.is_shuffled = local_w2.is_shuffled = True
prepare = maybe_make_prepare_finalize(config, quant)
assert isinstance(prepare, MoriPrepareAndFinalize)
assert prepare.use_fp8_dispatch == fp8
assert prepare.use_fp4_dispatch == mxfp4
assert prepare.num_dispatchers() == 2
experts = AiterExperts(config, quant)
experts_cls = {
"bf16": AiterUnquantizedExperts,
"fp8_token_channel": AiterFp8ChannelExperts,
"mxfp4": AiterMxfp4Experts,
}[scheme]
experts = experts_cls(config, quant)
assert not experts.expects_unquantized_inputs
kernel = FusedMoEKernel(prepare, experts)
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
w1[start:end].contiguous(), w2[start:end].contiguous()
)
local_w1.is_shuffled = local_w2.is_shuffled = True
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
expert_mask[start:end] = 1
def reference(route_weights):
if mxfp4:
# AITER's non-EP MXFP4 path quantizes the BF16 input the same way
# as FP4 dispatch, so the BF16 tolerances below still apply.
return rocm_aiter_fused_experts(
x, ref_w1, ref_w2, route_weights, ids, config, quant_config=ref_quant
)
return torch_experts(
x,
w1,
@@ -177,9 +223,14 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
)
@pytest.mark.parametrize("m,n,k", MNKS)
@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"])
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
@multi_gpu_test(num_gpus=2)
def test_mori_moe(m, n, k, fp8):
def test_mori_moe(m, n, k, scheme):
if scheme == "mxfp4":
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
pytest.skip("AITER MXFP4 W4A4 MoE requires gfx950")
config = VllmConfig(
parallel_config=ParallelConfig(
data_parallel_size=2,
@@ -193,4 +244,4 @@ def test_mori_moe(m, n, k, fp8):
"VLLM_ROCM_USE_AITER_MOE": "1",
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
}
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32)
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32)
+118
View File
@@ -0,0 +1,118 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.model_executor.layers.fused_moe.prepare_finalize import naive_dp_ep
from vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep import (
MoEPrepareAndFinalizeNaiveDPEPModular,
make_moe_prepare_and_finalize_naive_dp_ep,
)
class _RegionProbe(MoEPrepareAndFinalizeNaiveDPEPModular):
def __init__(self) -> None:
super().__init__()
self.body_ran = False
def run_comm(self) -> None:
with self._comm_region():
self.body_ran = True
@pytest.fixture
def dbo_mocks():
yield_switch = MagicMock()
switch_back = MagicMock()
with (
patch.object(
naive_dp_ep, "dbo_yield_and_switch_from_compute_to_comm", yield_switch
),
patch.object(naive_dp_ep, "dbo_switch_to_compute_sync", switch_back),
):
yield {"yield_switch": yield_switch, "switch_back": switch_back}
def _dbo_state(dbo_active: bool, capturing: bool):
return (
patch.object(naive_dp_ep, "dbo_enabled", return_value=dbo_active),
patch.object(torch.cuda, "is_current_stream_capturing", return_value=capturing),
)
def test_comm_region_noop_without_dbo(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=False, capturing=False)
with dbo, capturing:
probe.run_comm()
assert probe.body_ran
dbo_mocks["yield_switch"].assert_not_called()
dbo_mocks["switch_back"].assert_not_called()
def test_comm_region_handoff_when_dbo_active(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
with dbo, capturing:
probe.run_comm()
assert probe.body_ran
dbo_mocks["yield_switch"].assert_called_once_with()
dbo_mocks["switch_back"].assert_called_once_with()
def test_comm_region_skipped_during_graph_capture(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=True, capturing=True)
with dbo, capturing:
probe.run_comm()
assert probe.body_ran
dbo_mocks["yield_switch"].assert_not_called()
dbo_mocks["switch_back"].assert_not_called()
def test_comm_region_restores_compute_stream_on_error(dbo_mocks):
probe = _RegionProbe()
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
with (
dbo,
capturing,
pytest.raises(RuntimeError, match="collective failed"),
probe._comm_region(),
):
raise RuntimeError("collective failed")
dbo_mocks["yield_switch"].assert_called_once_with()
dbo_mocks["switch_back"].assert_called_once_with()
@pytest.mark.parametrize("is_rocm", [True, False])
def test_supports_dbo_only_on_rocm(is_rocm):
inst = MoEPrepareAndFinalizeNaiveDPEPModular()
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
with patch.object(naive_dp_ep, "current_platform", platform):
assert inst.supports_dbo() is is_rocm
# supports_async() also drives MK-internal shared-expert overlap, so
# it must not change with DBO support.
assert not inst.supports_async()
@pytest.mark.parametrize("is_rocm", [True, False])
def test_factory_returns_base_classes(is_rocm):
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
with patch.object(naive_dp_ep, "current_platform", platform):
mono = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=True)
modular = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=False)
assert type(mono) is naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic
assert type(modular) is MoEPrepareAndFinalizeNaiveDPEPModular
def test_default_supports_dbo_follows_supports_async():
# Classes that do not override supports_dbo() keep the old contract:
# DBO only with async prepare/finalize.
mono = naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic()
assert not mono.supports_async()
assert not mono.supports_dbo()
with patch.object(type(mono), "supports_async", return_value=True):
assert mono.supports_dbo()
+187
View File
@@ -2114,6 +2114,117 @@ def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
assert "parallel config" in reason
@pytest.mark.parametrize(
"all2all_backend",
[
"deepep_high_throughput",
"flashinfer_nvlink_two_sided",
"flashinfer_nvlink_one_sided",
],
)
def test_make_mxfp4_moe_kernel_rejects_cutlass_unsupported_expert_parallel(
monkeypatch: pytest.MonkeyPatch,
all2all_backend: str,
):
"""Compressed-tensors CUTLASS W4A4 bypasses the oracle; guard at assembly."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
)
monkeypatch.setattr(
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
)
moe_config = replace(
make_dummy_moe_config(
num_experts=256,
num_local_experts=32,
hidden_dim=4096,
intermediate_size=2048,
),
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend=all2all_backend,
enable_eplb=False,
),
)
w1_scale = torch.empty(32, 64, 128, dtype=torch.uint8)
w2_scale = torch.empty(32, 4096, 64, dtype=torch.uint8)
quant_config = mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale)
with pytest.raises(ValueError, match="parallel config"):
make_mxfp4_moe_kernel(
quant_config,
moe_config,
CutlassExpertsMxfp4,
Mxfp4MoeBackend.MARLIN,
)
def test_make_mxfp4_moe_kernel_validates_cutlass_expert_parallel(
monkeypatch: pytest.MonkeyPatch,
):
"""CUTLASS MXFP4 supports EP with allgather_reducescatter after expert_map."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_validate_mxfp4_experts_config,
)
monkeypatch.setattr(
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
)
moe_config = replace(
make_dummy_moe_config(
num_experts=256,
num_local_experts=32,
hidden_dim=4096,
intermediate_size=2048,
),
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
),
)
_validate_mxfp4_experts_config(
CutlassExpertsMxfp4, moe_config, Mxfp4MoeBackend.MARLIN
)
# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that
# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns
# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of
@@ -2255,3 +2366,79 @@ def test_emulation_a_mxfp6_moe_forward_quantizes_activations():
"w_mxfp4_a_mxfp6_e3m2 output is bit-identical to weight-only w_mxfp4:"
" the emulation never fake-quantized the activations"
)
# (backend, activation, hidden_in, inter_in, hidden_out, inter_out)
#
# The AITER_MXFP4_MXFP4 (W4A4) rows are the ones this table exists for: the
# per-partition intermediate is aligned to 128, not 256, so a model whose native
# shard is already 128-aligned is allocated at its real width instead of being
# padded up. `hidden_size` deliberately stays on 256 -- the gfx950 stage-1 mxfp4
# kernels are generated with tile_k 256 only, and stage-1 reduces over model_dim.
#
# The remaining rows pin the neighbouring branches so a future edit to this
# function cannot silently move a backend between them.
ROCM_MXFP4_ROUNDUP_CASES = [
# W4A4: 3072 / TP8 = 384 stays 384. This is the case the alignment change is
# for; the generic 256 round-up would allocate 512.
("AITER_MXFP4_MXFP4", "SILU", 6144, 384, 6144, 384),
# W4A4, already 256-aligned (e.g. moe_intermediate 2048 at TP8): identity in
# both the old and the new form, so such models are provably untouched.
("AITER_MXFP4_MXFP4", "SILU", 6144, 256, 6144, 256),
# W4A4, 1024 / TP8 = 128: the most aggressive case, halved rather than trimmed.
("AITER_MXFP4_MXFP4", "SILU", 4096, 128, 4096, 128),
# W4A4 ignores the activation -- the branch does not read it.
("AITER_MXFP4_MXFP4", None, 6144, 384, 6144, 384),
("AITER_MXFP4_MXFP4", "SWIGLUOAI", 6144, 384, 6144, 384),
# W4A4 hidden_size is still rounded to 256, unlike the intermediate.
("AITER_MXFP4_MXFP4", "SILU", 384, 384, 512, 384),
# W4A16 CK + SiTU/SILU keeps its own 128/128 carve-out.
("AITER_MXFP4_BF16", "SITU", 384, 384, 384, 384),
("AITER_MXFP4_BF16", "SILU", 384, 384, 384, 384),
# ... but only for those activations; anything else takes the generic ROCm arm.
("AITER_MXFP4_BF16", "SWIGLUOAI", 384, 384, 512, 512),
# Backends that stay on the generic ROCm 256/256 round-up.
("AITER_MXFP4_FP8", "SILU", 384, 384, 512, 512),
("AITER_TRITON_MXFP4_BF16", "SILU", 384, 384, 512, 512),
("TRITON", "SWIGLUOAI", 384, 384, 512, 512),
]
@pytest.mark.parametrize(
"backend_name,activation_name,hidden_in,inter_in,hidden_out,inter_out",
ROCM_MXFP4_ROUNDUP_CASES,
)
@pytest.mark.skipif(not ROCM_AVAILABLE, reason="ROCm is required for this test")
def test_rocm_mxfp4_round_up_hidden_and_intermediate(
backend_name: str,
activation_name: str | None,
hidden_in: int,
inter_in: int,
hidden_out: int,
inter_out: int,
):
"""Pin the ROCm arm of mxfp4_round_up_hidden_size_and_intermediate_size.
The allocation this function returns is what every MXFP4 MoE weight buffer,
scale buffer and aiter tuned-config lookup is sized from, so a change here is
invisible locally and expensive everywhere else. Asserting the table keeps
each backend on the alignment its kernels actually require.
"""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
mxfp4_round_up_hidden_size_and_intermediate_size,
)
backend = Mxfp4MoeBackend[backend_name]
activation = MoEActivation[activation_name] if activation_name else None
hidden, intermediate = mxfp4_round_up_hidden_size_and_intermediate_size(
backend, hidden_in, inter_in, activation
)
assert (hidden, intermediate) == (hidden_out, inter_out), (
f"{backend_name} / {activation_name}: "
f"hidden {hidden_in}->{hidden} (expected {hidden_out}), "
f"intermediate {inter_in}->{intermediate} (expected {inter_out})"
)
+2 -2
View File
@@ -848,13 +848,13 @@ def test_activation_method_enum_values():
def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
"""AITER MXFP4 MoE support should stay gfx950-only."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
AiterExperts,
AiterMxfp4Experts,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Static,
)
assert AiterExperts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950()
assert AiterMxfp4Experts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950()
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
+36 -5
View File
@@ -87,7 +87,7 @@ def _torch_topk_softplus_sqrt(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterExperts"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
@@ -302,15 +302,18 @@ def test_fused_topk_softplus_sqrt_hash(
@pytest.mark.parametrize(
("num_tokens", "num_experts", "indices_type"),
[
(0, 256, torch.uint32),
(17, 256, torch.uint32),
(0, 256, torch.int32),
(17, 256, torch.int32),
(17, 384, torch.int64),
],
)
@pytest.mark.parametrize("use_padding", [False, True])
def test_dsv4_fast_topk(
num_tokens: int,
num_experts: int,
indices_type: torch.dtype,
use_padding: bool,
monkeypatch: pytest.MonkeyPatch,
):
torch.manual_seed(0)
gating_output = torch.randn(
@@ -325,8 +328,25 @@ def test_dsv4_fast_topk(
routed_scaling_factor=1.5,
e_score_correction_bias=correction_bias,
)
topk_weights, topk_ids = dsv4_topk(
gating_output, correction_bias, indices_type, 1.5
is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1
if use_padding:
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.router."
"fused_topk_bias_router._get_padding_mask",
lambda _: is_padding,
)
gating_output[is_padding] = float("nan")
topk_weights_ref[is_padding] = 0
topk_ids_ref[is_padding] = -1
topk_weights, topk_ids = fused_topk_bias(
gating_output,
gating_output,
"sqrtsoftplus",
correction_bias,
6,
True,
indices_type=indices_type,
routed_scaling_factor=1.5,
)
assert topk_ids.dtype == indices_type
@@ -338,6 +358,17 @@ def test_dsv4_fast_topk(
rtol=2e-5,
)
if use_padding and num_tokens:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
weights, ids = dsv4_topk(
gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding
)
is_padding.fill_(True)
graph.replay()
assert (ids == -1).all()
assert (weights == 0).all()
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
@@ -136,7 +136,13 @@ def test_arm_cpu_unquantized_kernel_selection(
],
)
@patch(
"vllm.utils.flashinfer.has_flashinfer",
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe."
"has_flashinfer_cutlass_fused_moe",
return_value=False,
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe."
"has_flashinfer_trtllm_fused_moe",
return_value=False,
)
@patch(
@@ -145,13 +151,18 @@ def test_arm_cpu_unquantized_kernel_selection(
)
def test_select_default_backend_by_platform(
mock_aiter_enabled,
mock_has_flashinfer,
mock_has_flashinfer_trtllm,
mock_has_flashinfer_cutlass,
monkeypatch,
platform_method,
expected_backend,
):
"""Test default backend selection per platform with all optional
accelerators (FlashInfer, AITER) disabled."""
accelerators (FlashInfer, AITER) disabled.
Patch the feature probes directly because their cached results can outlive
changes to has_flashinfer().
"""
with patch(
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
) as mock_platform:
@@ -407,6 +418,41 @@ def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
)
@pytest.mark.parametrize("require_decomposed_backend", [False, True])
def test_require_decomposed_backend_rejects_monolithic_kernels(
mock_supports_current_device,
require_decomposed_backend,
):
"""Expert substitution adjusts routes, so the kernel must not route itself."""
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
TrtLlmBf16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEActivationFormat,
)
moe_config = make_dummy_moe_config()
moe_config.routing_method = RoutingMethodType.Renormalize
moe_config.require_decomposed_backend = require_decomposed_backend
supported, reason = TrtLlmBf16ExpertsMonolithic.is_supported_config(
TrtLlmBf16ExpertsMonolithic,
moe_config,
None,
None,
FusedMoEActivationFormat.Standard,
)
assert supported != require_decomposed_backend
assert (reason is not None and "routes adjusted" in reason) == (
require_decomposed_backend
)
@patch(
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
return_value=True,
@@ -580,3 +626,17 @@ def test_select_explicit_triton_backend(is_lora_enabled):
assert selected_backend == UnquantizedMoeBackend.TRITON
assert experts_cls is not None
@skipif_not_cuda_rocm
@pytest.mark.parametrize("moe_backend", ["humming", "marlin"])
def test_quantization_only_backend_falls_back_to_auto(moe_backend):
"""A quantization-only --moe-backend also reaches the unquantized MoE
layers of a quantized model (e.g. a BF16 MTP draft), which must use the
auto-selected backend instead of failing."""
moe_config = make_dummy_moe_config()
moe_config.moe_backend = moe_backend
assert select_unquantized_moe_backend(
moe_config=moe_config
) == select_unquantized_moe_backend(moe_config=make_dummy_moe_config())
@@ -5,7 +5,10 @@
import pytest
import torch
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.input_quant_fp8 import (
_FP8_MIN_SCALING_FACTOR,
QuantFP8,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.utils.torch_utils import set_random_seed
@@ -186,3 +189,22 @@ def test_quantfp8_group_edge_cases(default_vllm_config, seed: int) -> None:
assert x_quant_large.shape == x_large.shape
# FP8 max is typically 448 or 224, so scales should be > 1
assert (scales_large > 1.0).all(), "Large values should have scales > 1"
# Test UE8M0 clamping still returns a correctly rounded value
quant_op_ue8m0 = QuantFP8(
static=False,
group_shape=group_shape,
column_major_scales=False,
use_ue8m0=True,
)
x_quant_zero_ue8m0, scales_zero_ue8m0 = quant_op_ue8m0.forward_native(
x_zero.clone()
)
assert x_quant_zero_ue8m0.shape == x_zero.shape
# 0x807FFFFF selects only the sign and mantissa bits.
assert ((scales_zero_ue8m0.view(torch.int32) & 0x807FFFFF) == 0).all(), (
"Clamped scales did not have zero sign and mantissa bits."
)
assert (scales_zero_ue8m0 >= _FP8_MIN_SCALING_FACTOR).all(), (
"Clamped scales were not larger than or equal to _FP8_MIN_SCALING_FACTOR."
)
@@ -19,7 +19,7 @@ if not current_platform.is_rocm():
pytest.importorskip("triton")
from vllm.platforms.rocm import on_gfx1x # noqa: E402
from vllm.platforms.rocm import on_gfx1x, on_gfx115x # noqa: E402
device = "cuda"
@@ -446,22 +446,28 @@ def _hip_skinny_reference(
scales_nkg: torch.Tensor,
*,
group_size: int,
zp_bias: int,
zp_nkg: torch.Tensor | None,
) -> torch.Tensor:
"""Reference for symmetric HIP skinny: C = A @ (W - zp_bias) * S."""
"""Reference for HIP skinny: C = A @ (W - zp) * S.
``zp_nkg`` is [N, K//G] int32 raw zero points for the asymmetric case, or
None for symmetric uint4b8, where the kernel subtracts a constant 8.
"""
K = a_mk.shape[1]
N = w_int4_nk.shape[0]
num_groups = K // group_size
w_fp = (w_int4_nk.to(torch.float32) - zp_bias).view(N, num_groups, group_size)
w_g = w_int4_nk.to(torch.float32).view(N, num_groups, group_size)
zp = 8.0 if zp_nkg is None else zp_nkg.to(torch.float32).unsqueeze(-1)
s = scales_nkg.to(torch.float32).unsqueeze(-1)
w_dequant = (w_fp * s).view(N, K)
w_dequant = ((w_g - zp) * s).view(N, K)
return (a_mk.to(torch.float32) @ w_dequant.t()).to(a_mk.dtype)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize(
"M,K,N,G",
[
@@ -470,13 +476,50 @@ def _hip_skinny_reference(
(1, 512, 256, 128),
(2, 512, 256, 64),
(3, 256, 512, 64),
# M is the kernel's batch dimension; 4 and 5 reach dispatch tuples that
# 1..3 never take, and 5 is MAX_SKINNY_BATCH_SIZE.
(4, 512, 256, 32),
(4, 256, 512, 64),
(5, 512, 256, 128),
(5, 256, 256, 32),
# N == 8 is exactly one packed zero-point word.
(1, 256, 8, 32),
# K must exceed THRDS * A_CHUNK * UNRL for the K loop to run a whole
# unrolled block; every K above is below that bound, so without these
# the deep-K path goes untested. 2560 is not a multiple of the 1024 K
# step the batch 2 and 4 tuples take, so it also hits the ragged tail.
(1, 4096, 256, 128),
(2, 2560, 256, 32),
(3, 4096, 256, 64),
(4, 2560, 256, 128),
(5, 4096, 256, 32),
# On gfx115x, batch 1 with K == 4096, or symmetric with K a multiple of
# 8192, takes a tuple with A_CHUNK = 32 instead of 16. G = 32 is the
# tight case there: each lane's A_CHUNK spans exactly one scale group.
(1, 4096, 256, 32),
(1, 8192, 256, 128),
(1, 16384, 256, 64),
# K * batch beyond what LDS holds (32768 fp16 elements), so gfx115x
# walks K in chunks; elsewhere the op rejects these, as the layer
# routes them to Triton. The per-row window shrinks as the batch grows,
# so these cover a single reload and several, with and without a short
# final chunk. 20992 is not a multiple of the K step either, so it
# crosses a chunk boundary *and* has a ragged K tail.
(2, 20480, 512, 64),
(3, 16384, 512, 32),
(4, 16384, 512, 128),
(5, 21504, 512, 32),
(5, 20992, 512, 128),
],
)
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G, has_zp):
"""Test HIP wvSplitK_int4_g kernel directly via _custom_ops."""
import vllm._custom_ops as ops
from vllm.utils.platform_utils import num_compute_units
if K * M > LDS_CAPACITY_ELEMENTS and not on_gfx115x():
pytest.skip("deep-K HIP path is gfx115x only")
set_random_seed(0)
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
@@ -489,10 +532,64 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
dtype
)
cu_count = num_compute_units()
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G)
zp_nkg = None
zp_packed = None
if has_zp:
zp_nkg = torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
cu_count = num_compute_units()
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G, zp_packed)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("G", [32, 128])
@pytest.mark.parametrize("has_zp", [False, True])
@pytest.mark.parametrize("M", [2, 64], ids=["M=2_decode", "M=64_prefill"])
def test_hip_skinny_padded_group_stride(dtype, G, has_zp, M):
"""Scale and zero-point rows may be padded past K // group_size.
Both paths index them by their own row stride, so a padded row is read
correctly rather than bleeding into the next row. M picks the path: the
HIP skinny kernel at 2, the Triton prefill kernel at 64.
"""
from vllm.utils.platform_utils import num_compute_units
set_random_seed(0)
K, N = 2048, 256
num_groups = K // G
pad = 7
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
b_packed_i8 = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
scales = (0.05 * torch.rand((N, num_groups), device=device)).to(dtype)
scales_padded = torch.zeros((N, num_groups + pad), device=device, dtype=dtype)
scales_padded[:, :num_groups] = scales
scales_view = scales_padded[:, :num_groups]
assert scales_view.stride(0) == num_groups + pad
zp_nkg = zp_view = None
if has_zp:
zp_nkg = torch.randint(0, 16, (N, num_groups), device=device, dtype=torch.int32)
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
zp_padded = torch.zeros(
(N // 8, num_groups + pad), device=device, dtype=torch.int32
)
zp_padded[:, :num_groups] = zp_packed
zp_view = zp_padded[:, :num_groups]
cu_count = num_compute_units()
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
a, b_packed_i8, scales_view, zp_view, None, cu_count, G
)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@@ -510,6 +607,8 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
(1, 256, 256, 64),
(1, 512, 256, 32),
(1, 512, 256, 128),
# Past LDS: HIP on gfx115x, Triton elsewhere.
(2, 20480, 256, 64),
(32, 512, 256, 64),
(64, 1024, 256, 128),
],
@@ -535,7 +634,7 @@ def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
a, b_packed_i8, scales, None, None, cu_count, G
)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=None)
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
@@ -748,17 +847,30 @@ def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
@pytest.mark.parametrize(
"M,K,expected_path",
"M,K,gfx115x,expected_path",
[
(4, 8192, "hip"),
(4, 9728, "hip"),
(4, 9856, "triton"),
(MAX_SKINNY_BATCH_SIZE + 1, 1024, "triton"),
(4, 8192, False, "hip"),
(4, 9728, False, "hip"),
(4, 9856, False, "triton"),
(4, 9856, True, "hip"),
(MAX_SKINNY_BATCH_SIZE + 1, 1024, True, "triton"),
],
ids=[
"regular_limit",
"medium_range",
"above_medium",
"above_medium_gfx115x",
"batch_too_large",
],
ids=["regular_limit", "medium_range", "above_medium", "batch_too_large"],
)
def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch):
"""Route only supported regular and medium skinny shapes to HIP."""
def test_rdna_hybrid_w4a16_dispatch_boundaries(
M, K, gfx115x, expected_path, monkeypatch
):
"""Route only supported regular and medium skinny shapes to HIP.
gfx115x drops the K*M bound, so a shape past the medium limit still goes to
HIP there; the batch bound holds on every target.
"""
import vllm._custom_ops as ops
N, G = 16, 128
@@ -777,6 +889,7 @@ def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch)
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
monkeypatch.setattr(hybrid_module, "_on_gfx115x", lambda: gfx115x)
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
x_mk, w_q, scales, None, None, 40, G
+1 -1
View File
@@ -30,7 +30,7 @@ fp8_only = pytest.mark.skipif(
ENV_DEFAULTS = {
"VLLM_ROCM_FP8_PADDING": True,
"VLLM_ROCM_USE_AITER_FP8BMM": True,
"VLLM_ROCM_USE_AITER_FP8BMM": False,
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False,
}
@@ -148,7 +148,7 @@ def test_fp4_env_defaults():
import vllm.envs as envs
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
@pytest.mark.parametrize(
@@ -32,11 +32,9 @@ def torch_scaled_mm(
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
out = scale_a * out
out = scale_b.T * out
out = out.to(out_dtype)
if bias is not None:
out = out + bias
return out
out = out + bias.float()
return out.to(out_dtype)
def get_8bit_types():
@@ -125,6 +123,25 @@ def test_scaled_mm(
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
@pytest.mark.parametrize("in_dtype", get_8bit_types())
@pytest.mark.parametrize("use_heuristic", [True, False])
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
a = torch.ones((1, 32), device=device).to(in_dtype)
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
scale_a = torch.ones((1, 1), device=device)
scale_b = torch.full((1, 1), 2**-10, device=device)
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
expected = torch_scaled_mm(*args)
assert (expected == 1 + 2**-7).all()
torch.testing.assert_close(out, expected, rtol=0, atol=0)
# TD operand loads must be bit-exact vs the plain masked-load path.
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
+1 -4
View File
@@ -185,10 +185,7 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None:
let (3216, 7168) ship without its DISPATCH_DSV3_SHAPE(7168, 3216).
"""
source = (
Path(__file__).resolve().parents[2]
/ "csrc"
/ "libtorch_stable"
/ "dsv3_fused_a_gemm.cu"
Path(__file__).resolve().parents[2] / "csrc" / "dsv3_fused_a_gemm.cu"
).read_text(encoding="utf-8")
# Benchmark-only shapes live behind VLLM_K3_BENCH_SHAPES and are not built
# by default, so they must not count as available.
+110 -10
View File
@@ -1205,6 +1205,84 @@ def test_get_c128_boundary(starts, query_start_loc, expected):
assert _get_c128_boundary(metadata) is expected
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@torch.inference_mode()
def test_c128_skip_is_not_baked_into_cuda_graph(monkeypatch):
"""A graph captured on a batch with no C128 boundary must still launch the
compressor on replay; only eager steps may skip it."""
from vllm.config import CUDAGraphMode
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v4 import compressor as compressor_mod
from vllm.models.deepseek_v4.compressor import DeepseekCompressor
from vllm.models.deepseek_v4.nvidia.ops import sparse_attn_compress_cutedsl
launches = torch.zeros(1, dtype=torch.int32, device="cuda")
monkeypatch.setattr(
compressor_mod, "_SAVE_PARTIAL_STATES_KERNEL", lambda **kw: None
)
monkeypatch.setattr(
sparse_attn_compress_cutedsl,
"_SPARSE_ATTN_COMPRESSOR_CUTEDSL_KERNEL",
lambda **kw: launches.add_(1),
)
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
torch.nn.Module.__init__(compressor)
compressor.head_dim, compressor.rope_head_dim = 512, 64
compressor.compress_ratio, compressor.coff, compressor.overlap = 128, 1, False
compressor.use_fp4_cache = False
compressor._use_two_stage_fused_compressor = False
compressor._quant_block = 64
compressor._token_stride = 576
compressor._scale_dim = 8
compressor.rms_norm_eps = 1e-6
compressor.ape = torch.zeros(128, 512, device="cuda")
compressor.norm = SimpleNamespace(weight=torch.ones(512, device="cuda"))
compressor.state_cache = SimpleNamespace(
prefix="state", kv_cache=torch.zeros(2, 8, 1024, device="cuda")
)
compressor.k_cache_prefix = "main"
kv_cache = torch.zeros(1, 64, 584, dtype=torch.uint8, device="cuda")
compressor._static_forward_context = {"main": SimpleNamespace(kv_cache=kv_cache)}
num_tokens = 4
slots = torch.arange(num_tokens, device="cuda")
metadata = {
"state": SimpleNamespace(
slot_mapping=slots,
block_table=torch.zeros(num_tokens, 1, dtype=torch.int32, device="cuda"),
block_size=8,
token_to_req_indices=torch.arange(
num_tokens, dtype=torch.int32, device="cuda"
),
c128_boundary=False,
),
"main": SimpleNamespace(slot_mapping=slots),
}
kv_score = torch.zeros(num_tokens, 1024, device="cuda")
positions = torch.arange(num_tokens, device="cuda")
rotary = SimpleNamespace(cos_sin_cache=torch.zeros(1, 64, device="cuda"))
context = ForwardContext(
{}, metadata, {}, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
)
def run():
with override_forward_context(context):
compressor(kv_score, positions, rotary)
run()
torch.accelerator.synchronize()
assert launches.item() == 0, "eager step with no boundary should skip"
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
launches.zero_()
graph.replay()
torch.accelerator.synchronize()
assert launches.item() == 1, "captured graph must not bake in the skip"
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
@@ -1422,7 +1500,10 @@ def test_dequantize_and_gather_k_cache(
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
@pytest.mark.parametrize("block_size", [16, 64])
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
@pytest.mark.parametrize("scale_fmt", ["ue8m0", "mxfp8"])
def test_indexer_quant_cache_roundtrip(
num_tokens: int, block_size: int, scale_fmt: str
):
"""K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
→ manual dequant → compare against original."""
HEAD_DIM = 128
@@ -1442,7 +1523,9 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
ops.indexer_k_quant_and_cache(
k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, scale_fmt
)
# ── Gather ──────────────────────────────────────────────────────────
max_blocks_per_seq = num_blocks
@@ -1465,7 +1548,12 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
# ── Manual dequant ──────────────────────────────────────────────────
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
if scale_fmt == "mxfp8":
# One UE8M0 byte per 32 elements, all equal to the token's scale.
assert (dst_scale == dst_scale[:, :1]).all()
scale = (dst_scale[:, :1].to(torch.int32) << 23).view(torch.float32)
else:
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
k_recovered = k_fp8 * scale # [num_tokens, 128]
# ── Compare ─────────────────────────────────────────────────────────
@@ -1749,8 +1837,13 @@ def _reference_kv_compress_norm_rope(
),
],
)
def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: bool):
@pytest.mark.parametrize("mx_scale", [False, True])
def test_fused_kv_insert_indexer(
num_tokens: int, kv_block_size: int, use_fp4: bool, mx_scale: bool
):
"""Fused K compress+norm+rope+quant+insert for the indexer KV cache."""
if use_fp4 and mx_scale:
pytest.skip("MX scales only change the FP8 cache")
HEAD_DIM = 128
ROPE_DIM = 64
BLOCK_SIZE = 16
@@ -1833,6 +1926,7 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
TOKEN_STRIDE=TOKEN_STRIDE,
SCALE_DIM=SCALE_DIM,
KV_BLOCK_STRIDE=kv_cache.stride(0),
MX_SCALE=mx_scale,
num_warps=1,
)
@@ -1876,12 +1970,18 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
), f"token {i}: FP8 bytes differ"
scale_off = kv_block_size * TOKEN_STRIDE + pos * SCALE_DIM
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM].view(
torch.float32
)
assert torch.equal(actual_scale, scale[i : i + 1]), (
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
)
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM]
if mx_scale:
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = (scale[i : i + 1].view(torch.int32) >> 23).to(torch.uint8)
assert torch.equal(actual_scale, ue8m0.repeat(SCALE_DIM)), (
f"token {i}: mx scale {actual_scale.tolist()} != {ue8m0.item()}"
)
else:
actual_scale = actual_scale.view(torch.float32)
assert torch.equal(actual_scale, scale[i : i + 1]), (
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
)
@pytest.mark.parametrize("compress_ratio", [4, 128])
@@ -32,6 +32,7 @@ import torch
from vllm.models.deepseek_v32.common import kernels as K
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
FP8 = torch.float8_e4m3fn
FP8_MAX = 448.0
@@ -116,6 +117,16 @@ def rope(
return out
def indexer_scales(scale_bytes: torch.Tensor) -> torch.Tensor:
"""Per-token fp32 scales from the indexer cache's 4 scale bytes per token."""
if not mqa_logits_use_mx_scales():
return scale_bytes.view(torch.float32)
# MX scales: the power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = scale_bytes.view(-1, 4)
assert (ue8m0 == ue8m0[:, :1]).all(), "MX scale bytes differ within a token"
return (ue8m0[:, 0].to(torch.int32) << 23).view(torch.float32)
def ue8m0_quant(vals: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Per-row (last dim) UE8M0 fp8 quant matching kernels._fp8_ue8m0_quantize."""
amax = vals.float().abs().amax(dim=-1, keepdim=True)
@@ -243,7 +254,7 @@ def test_fused_norm_rope(
q_ref, s_ref = ue8m0_quant(ik_ref)
flat = idx_cache[0].reshape(-1)
vals = flat[: bs * cfg.index_head_dim].view(FP8).reshape(bs, cfg.index_head_dim)
scales = flat[bs * cfg.index_head_dim :].view(torch.float32)
scales = indexer_scales(flat[bs * cfg.index_head_dim :])
assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8")
torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0)
@@ -318,7 +329,7 @@ def test_fused_norm_rope_packed_indexer_block_stride(cfg: ModelConfig):
.view(FP8)
.view(block_size, cfg.index_head_dim)
)
scales = packed[block_size * cfg.index_head_dim :].view(torch.float32)
scales = indexer_scales(packed[block_size * cfg.index_head_dim :])
assert_fp8(values, q_ref, "packed indexer-K fp8")
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
)
from vllm.models.deepseek_v4.common.ops import fused_indexer_q_rope_quant
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.utils.import_utils import has_cutedsl
HEAD_DIM = 128
@@ -347,6 +348,10 @@ def _indexer_k_reference(
exponent = k.float().abs().amax(dim=-1, keepdim=True).clamp(min=1e-4) / 448
scale = exponent.log2().ceil().exp2()
values = (k.float() / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
if mqa_logits_use_mx_scales():
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = (scale.view(torch.int32) >> 23).to(torch.uint8)
return values.view(torch.uint8), ue8m0.repeat(1, 4)
return values.view(torch.uint8), scale.view(torch.uint8)
@@ -29,6 +29,7 @@ import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
if current_platform.is_rocm():
from vllm.models.glm5next.amd.ops.kpool_compress import (
@@ -206,7 +207,12 @@ def _torch_reference(
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
s_off = page_base + k_region + loc_tok * 4
kv_flat[k_off] = quantized.view(torch.uint8)
kv_flat[s_off : s_off + 4] = scale.detach().reshape(1).view(torch.uint8)
scale_bytes = scale.detach().reshape(1).view(torch.uint8)
if mqa_logits_use_mx_scales():
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
ue8m0 = scale.reshape(1).view(torch.int32) >> 23
scale_bytes = ue8m0.to(torch.uint8).repeat(4)
kv_flat[s_off : s_off + 4] = scale_bytes
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
# pos_valid keys off the pool-granular cache_loc, which is -1 for
@@ -220,12 +226,19 @@ def _torch_reference(
return kv_out, tail_cpu.to(torch.bfloat16).to(device="cuda")
def _assert_eq(r_ref, r_kern):
def _assert_eq(r_ref, r_kern, k_byte_tol=0):
kv_ref, tail_ref = r_ref
kv_kern, tail_kern = r_kern
assert torch.equal(kv_ref, kv_kern), (
"kv_cache differs: max diff "
f"{(kv_ref.int() - kv_kern.int()).abs().max().item()}"
# Each page holds the fp8 K bytes followed by the fp32 scales.
k_region = PAGE_SIZE * HEAD_DIM
pages_ref = kv_ref.view(kv_ref.shape[0], -1)
pages_kern = kv_kern.view(kv_kern.shape[0], -1)
k_diff = (pages_ref[:, :k_region].int() - pages_kern[:, :k_region].int()).abs()
assert k_diff.max().item() <= k_byte_tol, (
f"kv_cache K differs: max diff {k_diff.max().item()}"
)
assert torch.equal(pages_ref[:, k_region:], pages_kern[:, k_region:]), (
"kv_cache scales differ"
)
assert torch.equal(tail_ref, tail_kern), (
"tail_kv_cache differs: max diff "
@@ -652,4 +665,6 @@ def test_batched_matches_reference_fuzz(seed):
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
_assert_eq(r_ref, r_kern)
# Random inputs can land the pooled value on a bf16 rounding midpoint,
# where the kernel and the CPU reference may round apart by one fp8 step.
_assert_eq(r_ref, r_kern, k_byte_tol=1)
+544
View File
@@ -0,0 +1,544 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for cuteDSL low-latency GEMM with fp32 weights."""
import pytest
import torch
import torch.nn.functional as F
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not torch.accelerator.is_available(), reason="CUDA required"
)
@pytest.fixture(autouse=True, scope="module")
def _require_sm90_and_cutedsl():
if not current_platform.has_device_capability(90):
pytest.skip("Requires SM90+ (Hopper/Blackwell)")
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
is_available,
)
if not is_available():
pytest.skip("cuteDSL (CUTLASS Python) not installed")
def _ref(a, b):
return torch.mm(a.float(), b.float().T)
def _assert_close(out, ref, *, min_cos_sim=0.99, context=""):
assert out.device.type == "cuda", f"{context}: not on CUDA"
assert torch.isfinite(out).all(), f"{context}: NaN/Inf"
cos = F.cosine_similarity(
out.reshape(-1).float(), ref.reshape(-1).float(), dim=0
).item()
assert cos > min_cos_sim, (
f"{context}: cos_sim {cos:.6f} < {min_cos_sim} "
f"(abs_err={(out.float() - ref.float()).abs().max().item():.2e})"
)
def _gemm(a, b, output_dtype=torch.float32):
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
LL_FP32W_GEMM_KERNEL,
)
return LL_FP32W_GEMM_KERNEL(a, b, output_dtype)
SHAPES = [
(256, 3072, "MiniMax-M2"),
(256, 7168, "DSV3-sized"),
(128, 5120, "DeepSeek-V2"),
(64, 2880, "non-aligned-K"),
(8, 4096, "Mixtral"),
]
A_DTYPES = [
pytest.param(torch.bfloat16, id="bf16"),
pytest.param(torch.float16, id="fp16"),
pytest.param(torch.float32, id="fp32"),
]
# =================================================================
# Correctness: all dtype combos x shapes x M values
# =================================================================
@pytest.mark.parametrize("a_dtype", A_DTYPES)
@pytest.mark.parametrize("M", [1, 2, 4, 8, 16])
@pytest.mark.parametrize("N,K,desc", SHAPES, ids=[s[2] for s in SHAPES])
def test_correctness(M, N, K, desc, a_dtype):
torch.manual_seed(42)
a = torch.randn(M, K, device="cuda").to(a_dtype)
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
out = _gemm(a, b)
assert out.dtype == torch.float32
assert out.shape == (M, N)
_assert_close(out, _ref(a, b), context=f"{a_dtype} M={M} {desc}")
# =================================================================
# Small K (k_main=0, only tail/scalar paths)
# =================================================================
@pytest.mark.parametrize("K", [16, 32, 64, 128, 256, 512, 1024, 1536])
def test_small_K(K):
torch.manual_seed(42)
a = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"small_K={K}")
# =================================================================
# K sweep (covers main loop + various tail combinations)
# =================================================================
@pytest.mark.parametrize("K", [2048, 2304, 2880, 3072, 4096, 5120, 7168])
def test_K_sweep(K):
torch.manual_seed(42)
a = torch.randn(8, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"K={K}")
# =================================================================
# MiniMax-M2 specific shapes
# =================================================================
@pytest.mark.parametrize("M", [1, 4, 8, 16, 32])
def test_minimax_m2(M):
torch.manual_seed(42)
a = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
b = torch.randn(256, 3072, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context=f"MiniMax-M2 M={M}")
# =================================================================
# Single token (M=1, decode path)
# =================================================================
@pytest.mark.parametrize(
"N,K",
[(256, 3072), (256, 7168), (8, 4096)],
ids=["MiniMax-M2", "DSV3-sized", "Mixtral"],
)
def test_single_token(N, K):
torch.manual_seed(42)
a = torch.randn(1, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
out = _gemm(a, b)
assert out.shape == (1, N)
_assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
# =================================================================
# Arbitrary N
# =================================================================
@pytest.mark.parametrize("N", [1, 3, 16, 64, 256])
def test_arbitrary_N(N):
torch.manual_seed(42)
a = torch.randn(4, 3072, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, 3072, dtype=torch.float32, device="cuda")
out = _gemm(a, b)
assert out.shape == (4, N)
_assert_close(out, _ref(a, b), context=f"N={N}")
# =================================================================
# Numerical robustness
# =================================================================
def test_large_values():
torch.manual_seed(42)
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda") * 100
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda") * 100
out = _gemm(a, b)
assert torch.isfinite(out).all()
_assert_close(out, _ref(a, b), context="large_values")
def test_near_zero():
torch.manual_seed(42)
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda") * 1e-4
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda") * 1e-4
out = _gemm(a, b)
assert torch.isfinite(out).all()
assert out.abs().max() < 1.0
def test_zeros():
a = torch.zeros(4, 2048, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
assert (_gemm(a, b) == 0).all()
def test_ones():
a = torch.ones(1, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(32, 4096, dtype=torch.float32, device="cuda")
_assert_close(_gemm(a, b), _ref(a, b), context="ones")
# =================================================================
# Output dtype
# =================================================================
def test_output_fp32():
torch.manual_seed(42)
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
assert _gemm(a, b).dtype == torch.float32
# =================================================================
# Determinism
# =================================================================
@pytest.mark.parametrize("M", [1, 4, 8, 16])
@pytest.mark.parametrize("K", [1024, 3072, 7168])
def test_deterministic(M, K):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
torch.testing.assert_close(_gemm(a, b), _gemm(a, b), atol=0, rtol=0)
# =================================================================
# Cross-dtype consistency
# =================================================================
def test_bf16_vs_fp32_activations():
"""bf16 and fp32 activations on same data should roughly agree."""
torch.manual_seed(42)
K, N = 4096, 128
a_bf16 = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
out_bf16 = _gemm(a_bf16, b)
a_fp32 = a_bf16.float()
out_fp32 = _gemm(a_fp32, b)
_assert_close(
out_bf16, out_fp32, min_cos_sim=0.999, context="bf16_vs_fp32_activations"
)
# =================================================================
# CUDA graph
# =================================================================
@pytest.mark.parametrize(
"M,K,N", [(4, 3072, 256), (16, 5120, 128)], ids=["MiniMax", "DeepSeek"]
)
def test_cudagraph(M, K, N):
torch.manual_seed(42)
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
for _ in range(5):
g.replay()
torch.accelerator.synchronize()
_assert_close(out, _ref(a, b), context=f"cudagraph M={M} K={K}")
def test_cudagraph_20x_replay():
torch.manual_seed(42)
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
results = []
for _ in range(20):
g.replay()
torch.accelerator.synchronize()
results.append(out.clone())
for i in range(1, len(results)):
torch.testing.assert_close(
results[0], results[i], atol=0, rtol=0, msg=f"Replay {i} differs"
)
def test_cudagraph_input_update():
torch.manual_seed(42)
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
_gemm(a, b)
torch.accelerator.synchronize()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
out = _gemm(a, b)
a.copy_(torch.randn_like(a))
g.replay()
torch.accelerator.synchronize()
_assert_close(out, _ref(a, b), context="cudagraph input update")
# =================================================================
# Negative tests
# =================================================================
def test_invalid_weight_dtype():
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.bfloat16, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_device_cpu():
a = torch.randn(4, 4096, dtype=torch.bfloat16)
b = torch.randn(64, 4096, dtype=torch.float32)
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_1d_input():
a = torch.randn(4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_activation_dtype():
a = torch.ones(4, 4096, dtype=torch.int32, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_output_dtype():
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b, torch.bfloat16)
def test_invalid_k_mismatch():
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
def test_invalid_non_contiguous():
base = torch.randn(4, 8192, dtype=torch.bfloat16, device="cuda")
a = base[:, ::2]
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(ValueError):
_gemm(a, b)
# =================================================================
# GateLinear dispatch integration
# =================================================================
def _make_gate_linear_fp32w(
monkeypatch,
*,
params_dtype,
out_dtype=torch.float32,
input_size=2048,
output_size=64,
):
monkeypatch.setattr(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
lambda: 1,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
lambda: 0,
)
monkeypatch.setattr(
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
lambda: 1,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.is_available",
lambda: False,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.is_available",
lambda: True,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
return GateLinear(
input_size=input_size,
output_size=output_size,
bias=False,
out_dtype=out_dtype,
params_dtype=params_dtype,
).cuda()
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float16, torch.float32])
def test_gate_linear_uses_ll_fp32w_for_fp32_weight_fast_path(monkeypatch, x_dtype):
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
x = torch.randn(4, 2048, dtype=x_dtype, device="cuda")
calls = []
def fake_ll_fp32w_gemm(hidden_states, router_weight):
calls.append((hidden_states, router_weight))
return torch.full(
(hidden_states.shape[0], router_weight.shape[0]),
1.0,
dtype=torch.float32,
device=hidden_states.device,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fake_ll_fp32w_gemm,
)
out, bias = gate(x)
assert bias is None
assert out.shape == (4, 64)
assert out.dtype == torch.float32
assert len(calls) == 1
assert calls[0][0] is x
assert calls[0][1] is gate.weight
PREFERRED_FP32W_GATE_CASES = [
*((6144, 128, M) for M in range(1, 33)),
*((6144, 256, M) for M in (1, 8, 12, 14, 16, 20, 22, 25, 27, 32)),
]
@pytest.mark.parametrize(("input_size", "output_size", "M"), PREFERRED_FP32W_GATE_CASES)
def test_gate_linear_uses_ll_fp32w_when_cpp_tier_disabled(
monkeypatch, input_size, output_size, M
):
gate = _make_gate_linear_fp32w(
monkeypatch,
params_dtype=torch.float32,
input_size=input_size,
output_size=output_size,
)
gate.allow_fp32_router_gemm = False
x = torch.randn(M, input_size, dtype=torch.bfloat16, device="cuda")
calls = []
def fake_ll_fp32w_gemm(hidden_states, router_weight):
calls.append((hidden_states, router_weight))
return torch.full(
(hidden_states.shape[0], router_weight.shape[0]),
2.0,
dtype=torch.float32,
device=hidden_states.device,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fake_ll_fp32w_gemm,
)
out, _ = gate(x)
assert len(calls) == 1
assert calls[0][0] is x
assert calls[0][1] is gate.weight
assert torch.all(out == 2.0)
@pytest.mark.parametrize("M", [8, 16, 32])
def test_gate_linear_uses_cpp_fp32_router_for_supported_cpp_shape(monkeypatch, M):
gate = _make_gate_linear_fp32w(
monkeypatch,
params_dtype=torch.float32,
input_size=3072,
output_size=256,
)
assert gate.allow_fp32_router_gemm
x = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
def fail_ll_fp32w_gemm(hidden_states, router_weight):
raise AssertionError(
"ll_fp32w_gemm should not bypass fp32_router_gemm_dispatch"
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fail_ll_fp32w_gemm,
)
out, _ = gate(x)
assert out.shape == (M, 256)
assert out.dtype == torch.float32
def test_gate_linear_bf16_weight_skips_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.bfloat16)
assert not gate.allow_ll_fp32w_gemm
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
def fail_ll_fp32w_gemm(hidden_states, router_weight):
raise AssertionError("ll_fp32w_gemm should not run for bf16 weights")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fail_ll_fp32w_gemm,
)
out, _ = gate(x)
assert out.shape == (4, 64)
assert out.dtype == torch.float32
def test_gate_linear_set_out_dtype_enables_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(
monkeypatch, params_dtype=torch.float32, out_dtype=None
)
assert not gate.allow_ll_fp32w_gemm
gate.set_out_dtype(torch.float32)
assert gate.allow_ll_fp32w_gemm
def test_gate_linear_non_fp32_out_dtype_disables_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(
monkeypatch, params_dtype=torch.float32, out_dtype=None
)
gate.set_out_dtype(torch.bfloat16)
assert not gate.allow_ll_fp32w_gemm
def test_gate_linear_m_gt_32_skips_ll_fp32w(monkeypatch):
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
x = torch.randn(33, 2048, dtype=torch.bfloat16, device="cuda")
def fail_ll_fp32w_gemm(hidden_states, router_weight):
raise AssertionError("ll_fp32w_gemm should not run for M > 32")
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
fail_ll_fp32w_gemm,
)
out, _ = gate(x)
assert out.shape == (33, 64)
assert out.dtype == torch.float32
if __name__ == "__main__":
pytest.main([__file__, "-v"])