Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 22:06:10 +07:00
327 changed files with 13704 additions and 5433 deletions
@@ -107,3 +107,37 @@ def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
@torch.inference_mode()
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
Production allocates index_q with the index-K cache dtype, so the scorers
normally see a matched pair and keep the native FP8 MMA. This pins the
other branch: a bf16 query against an e4m3 cache still compiles and still
ranks the planted blocks.
"""
dev = "cuda"
q, keys = _make_inputs(dev)
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
def select(cache: torch.Tensor) -> set[int]:
idx = minimax_m3_index_decode(
q, # bf16 query, deliberately not cast to the cache dtype
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
block_table,
seq_lens,
SEQ_LEN,
TOPK,
0,
0,
HEADS,
1,
1,
)
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
@@ -596,6 +596,114 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
device = torch.device("cuda")
set_random_seed(7)
head_dim = nope_dim + rope_dim
return dict(
q=torch.randn(3, 3, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
kv=torch.randn(5, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
indices=torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device),
indptr=torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device),
scale=head_dim**-0.5,
attn_sink=torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device),
nope_head_dim=nope_dim,
rope_head_dim=rope_dim,
)
# 448+64 is DeepSeek V4/V4.1, 512+64 is V3.2, and 256+0 stands in for the
# NoPE-only layouts where the destination spans the whole head dim.
SPARSE_PREFILL_DIMS = [(NOPE_HEAD_DIM, ROPE_HEAD_DIM), (512, 64), (256, 0)]
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_matches_allocated(
nope_dim: int, rope_dim: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
allocated = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)
q = kwargs["q"]
dest = torch.empty(q.shape[0], q.shape[1], nope_dim, dtype=q.dtype, device=q.device)
returned = _rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
assert returned is dest
torch.testing.assert_close(dest, allocated[..., :nope_dim], atol=0.0, rtol=0.0)
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_view_leaves_neighbors_untouched(
nope_dim: int, rope_dim: int
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
q = kwargs["q"]
num_queries, num_heads = q.shape[0], q.shape[1]
buffer = torch.full(
(num_queries + 4, num_heads + 2, nope_dim),
-7.0,
dtype=q.dtype,
device=q.device,
)
dest = buffer[2 : 2 + num_queries, :num_heads]
assert not dest.is_contiguous()
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :nope_dim]
torch.testing.assert_close(dest, expected, atol=0.0, rtol=0.0)
untouched = torch.ones_like(buffer, dtype=torch.bool)
untouched[2 : 2 + num_queries, :num_heads] = False
assert torch.all(buffer[untouched] == -7.0)
@pytest.mark.parametrize("out_dtype", [torch.float32, torch.float16])
@torch.inference_mode()
def test_sparse_attn_prefill_ragged_out_casts_to_dest_dtype(
out_dtype: torch.dtype,
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(512, 64)
q = kwargs["q"]
dest = torch.empty(q.shape[0], q.shape[1], 512, dtype=out_dtype, device=q.device)
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :512]
assert dest.dtype == out_dtype
torch.testing.assert_close(dest.float(), expected.float(), atol=2e-2, rtol=2e-2)
@torch.inference_mode()
def test_sparse_attn_prefill_head_dim_wide_out_keeps_rope_rows() -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_rocm_sparse_attn_prefill_ragged_triton,
)
kwargs = _sparse_prefill_ragged_inputs(NOPE_HEAD_DIM, ROPE_HEAD_DIM)
q = kwargs["q"]
dest = torch.empty_like(q)
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
expected = _ref_sparse_prefill_ragged(
q, kwargs["kv"], [[0, 2], [1, 3, 4], []], kwargs["scale"], kwargs["attn_sink"]
)
torch.testing.assert_close(dest, expected, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("num_queries", "on_gfx950", "expected"),
[(1023, True, False), (1024, True, True), (1024, False, False)],
@@ -718,10 +826,11 @@ def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> Non
output = torch.empty_like(q)
dense_fallback_calls = 0
def fake_dense_fallback(*args, **kwargs):
def fake_dense_fallback(*args, out, **kwargs):
nonlocal dense_fallback_calls
dense_fallback_calls += 1
return torch.zeros_like(q)
out.zero_()
return out
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)
+1 -1
View File
@@ -194,7 +194,7 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
layer = types.SimpleNamespace(
prefix=PREFIX,
enable_fused_gdn_decode=True,
enable_fused_gdn_spec_decode=True,
norm=types.SimpleNamespace(
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
),
+1 -1
View File
@@ -167,7 +167,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
draft = SimpleNamespace(
use_mega_moe=False,
use_native_mega_moe=False,
use_sequence_parallel=False,
hc_mult=hc_mult,
layers=[make_layer(mix) for mix in mixes],
@@ -88,6 +88,15 @@ class Config:
torch_trace_dir_path: str | None = None
# Force AiterExperts's hidden_pad/intermediate_pad computation
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
# None (default) preserves today's behavior: FusedMoEConfig defaults both
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
# intermediate_pad come out to 0.
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
hidden_dim_unpadded: int | None = None
intermediate_size_per_partition_unpadded: int | None = None
def __post_init__(self):
if self.quant_config is None:
self.quant_config = TestMoEQuantConfig(None, False, False, None)
@@ -664,6 +673,10 @@ def make_modular_kernel(
activation=config.activation,
device=vllm_config.device_config.device,
routing_method=RoutingMethodType.DeepSeekV3,
hidden_dim_unpadded=config.hidden_dim_unpadded,
intermediate_size_per_partition_unpadded=(
config.intermediate_size_per_partition_unpadded
),
)
prepare_finalize = maybe_make_prepare_finalize(
+2
View File
@@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
expert_map=None,
apply_router_weight_on_input=True,
routed_scaling_factor=1.0,
routing_sink=None,
)
check_accuracy(
@@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
router_logits=logits,
e_score_correction_bias=bias,
routed_scaling_factor=scale,
routing_sink=None,
**common,
)
topk_weights, topk_ids = fused_topk_bias(
+307
View File
@@ -0,0 +1,307 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
MoE geometry of a real model, warms it up and runs a forward whose inputs are
chosen so that every quantization step in the megakernel is exact, for the
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
32). The expected output is then known in closed form and compared bit for
bit.
"""
from dataclasses import dataclass
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_gemm
@dataclass(frozen=True)
class Geometry:
name: str
hidden_size: int
intermediate_size: int
num_experts: int
top_k: int
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
GEOMETRIES = (
Geometry("dsv4-pro", 7168, 3072, 384, 6),
Geometry("dsv4-flash", 4096, 2048, 256, 6),
Geometry("mistral-small-4", 4096, 2048, 128, 4),
Geometry("mistral-large-3", 7168, 4096, 128, 4),
)
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
NUM_TOKENS_PER_RANK = 4
MAX_TOKENS_PER_RANK = 128
# Every block scale below is either 1 or a power of two, so the FP4 codes
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
#
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
# it; every weight column the sentinel meets is zero.
X_SIGNAL = 2.0
X_SENTINEL = 6.0
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
W13_DIAG = 6.0
H_SIGNAL = 144.0
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
# sentinel sits next to a zero intermediate channel and pins the block scale.
W2_SENTINEL = 6.0
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
def _expert_out(expert: int) -> float:
return E2M1_CODES[expert % len(E2M1_CODES)]
def _topk_weights(top_k: int) -> tuple[float, ...]:
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
# weight * 144 must requantize exactly too: powers of two do.
return tuple(2.0 ** -(k + 1) for k in range(top_k))
def _routing_method(backend: str) -> RoutingMethodType:
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
return RoutingMethodType.DeepseekV4
return RoutingMethodType.Default
def _routing(
geometry: Geometry, rank: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor]:
"""top_k distinct experts per token, strided across ranks so dispatch goes
remote."""
base = (
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
) % geometry.num_experts
stride = geometry.num_experts // geometry.top_k
offsets = torch.arange(geometry.top_k, device=device) * stride
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
torch.int32
)
topk_weights = torch.tensor(
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
dtype=torch.float32,
device=device,
)
return topk_ids, topk_weights
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
x = torch.zeros(
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
)
for t in range(NUM_TOKENS_PER_RANK):
x[t, 16 * t] = X_SIGNAL
x[t, 16 * t + 1] = X_SENTINEL
return x
def _local_expert_weights(
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
) -> FlashInferMoeEpWeights:
w13 = torch.zeros(
num_local_experts,
2 * geometry.intermediate_size,
geometry.hidden_size,
dtype=torch.bfloat16,
device=device,
)
w2 = torch.zeros(
num_local_experts,
geometry.hidden_size,
geometry.intermediate_size,
dtype=torch.bfloat16,
device=device,
)
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
for local in range(num_local_experts):
expert = rank * num_local_experts + local
w13[local, 0, token_columns] = W13_DIAG
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
w2[local, expert, 0] = _expert_out(expert)
w2[local, expert, 2] = W2_SENTINEL
return FlashInferMoeEpWeights(w13=w13, w2=w2)
def _expected_output(
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
) -> torch.Tensor:
expected = torch.zeros(
NUM_TOKENS_PER_RANK,
geometry.hidden_size,
dtype=torch.float32,
device=topk_ids.device,
)
for t in range(NUM_TOKENS_PER_RANK):
for k in range(geometry.top_k):
expert = int(topk_ids[t, k])
weight = float(topk_weights[t, k])
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
return expected.to(torch.bfloat16)
def _moe_config(
vllm_config: VllmConfig,
geometry: Geometry,
world_size: int,
device: torch.device,
backend: str,
) -> FusedMoEConfig:
parallel = FusedMoEParallelConfig.make(
tp_size_=1,
pcp_size_=1,
dp_size_=world_size,
sp_size_=1,
vllm_parallel_config=vllm_config.parallel_config,
)
return FusedMoEConfig(
num_experts=geometry.num_experts,
experts_per_token=geometry.top_k,
hidden_dim=geometry.hidden_size,
intermediate_size=geometry.intermediate_size,
num_local_experts=geometry.num_experts // world_size,
num_logical_experts=geometry.num_experts,
activation=MoEActivation.SILU,
device=device,
routing_method=_routing_method(backend),
moe_parallel_config=parallel,
in_dtype=torch.bfloat16,
moe_backend=backend,
max_num_tokens=MAX_TOKENS_PER_RANK,
)
def _check_expert_views_alias_kernel_memory(
adapter: FlashInferMoeEp,
geometry: Geometry,
rank: int,
num_local_experts: int,
device: torch.device,
) -> None:
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
Swap local experts 0 and 1 in place on every rank, then route through the
swapped physical slots: the output must be the one the unswapped logical
experts would have produced.
"""
for view in adapter.expert_weight_views():
first = view[0].clone()
view[0].copy_(view[1])
view[1].copy_(first)
torch.distributed.barrier()
topk_ids, topk_weights = _routing(geometry, rank, device)
slot = topk_ids % num_local_experts
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
torch.accelerator.synchronize()
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
def _init_warmup_forward(
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = backend
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
)
rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{rank}")
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
adapter = FlashInferMoeEp(
moe,
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
apply_topk_in_fc1=apply_topk_in_fc1(moe),
)
try:
adapter.warmup()
topk_ids, topk_weights = _routing(geometry, rank, device)
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
torch.accelerator.synchronize()
expected = _expected_output(geometry, topk_ids, topk_weights)
assert output.dtype == expected.dtype and output.shape == expected.shape
assert torch.equal(output, expected), (
f"rank {rank}: max abs diff "
f"{(output.float() - expected.float()).abs().max().item()}, "
f"nonzero output channels per token "
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
)
if backend == FLASHINFER_MOE_EP_CUTEDSL:
_check_expert_views_alias_kernel_memory(
adapter, geometry, rank, moe.num_local_experts, device
)
finally:
adapter.destroy()
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="The FlashInfer MoE-EP megakernels require SM100",
)
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
@pytest.mark.parametrize("world_size", [2, 4])
def test_init_warmup_forward_is_exact_on_grid_inputs(
world_size: int, backend: str, geometry: Geometry
):
pytest.importorskip("flashinfer.moe_ep")
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
pytest.skip("DeepGEMM is not available")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs")
distributed_run(_init_warmup_forward, world_size, geometry, backend)
@@ -42,7 +42,7 @@ from .modular_kernel_tools.parallel_utils import (
ProcessGroupInfo,
parallel_launch_with_config,
)
from .utils import check_accuracy
from .utils import check_accuracy, make_test_weights
has_any_multi_gpu_package = (
has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe()
@@ -691,6 +691,303 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
)
# --- 4b: hidden_dim_unpadded/intermediate_size_per_partition_unpadded matrix -
# K/N intentionally unaligned to AITER's 64/128 granularity (unlike the
# shared mk_objects.py defaults) so hidden_pad/intermediate_pad are forced.
_PADDING_E = 8
_PADDING_M = 16
_PADDING_TOPK = 2
_PADDING_K_UNPADDED = 896 # 7 * 128
_PADDING_K_PADDED = 1024 # +128
_PADDING_N_UNPADDED = 384 # 3 * 128
_PADDING_N_PADDED = 512 # +128
# hidden_pad = (K_padded - hidden_dim_unpadded) // 128 * 128 (rocm_aiter_moe.py)
_PADDING_HIDDEN_PAD_EXPECTED = 128
# intermediate_pad = (N_padded - intermediate_size_per_partition_unpadded)
# // 64 * 64 * (2 if tp_size == 1 else 1) (rocm_aiter_moe.py); tp_size==1 here.
_PADDING_INTERMEDIATE_PAD_EXPECTED = 256
# {no padding, hidden-only, intermediate-only, both} -> (pad_hidden, pad_intermediate)
_PADDING_MODES: dict[str, tuple[bool, bool]] = {
"none": (False, False),
"hidden": (True, False),
"intermediate": (False, True),
"both": (True, True),
}
# unquantized + the 4 fp8 configs AiterExperts supports (excludes MXFP4,
# stubbed below pending MI350/gfx950, and MK_QUANT_CONFIGS[1], unsupported).
_PADDING_QUANT_CONFIGS = [
MK_QUANT_CONFIGS[0], # unquantized
MK_QUANT_CONFIGS[2], # fp8 channel weights / per-token activations
MK_QUANT_CONFIGS[3], # fp8 per-tensor weights / per-tensor activations
MK_QUANT_CONFIGS[4], # fp8 per-tensor weights / per-token activations
MK_QUANT_CONFIGS[5], # fp8 128x128-block weights / 128-block activations
]
_PADDING_QUANT_IDS = [
"unquantized",
"fp8_channel_token",
"fp8_tensor_tensor",
"fp8_tensor_token",
"fp8_block_token",
]
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
gate/up halves of `real_per_half` size each."""
padded_per_half = t.shape[1] // 2
return torch.cat(
[
t[:, :real_per_half],
t[:, padded_per_half : padded_per_half + real_per_half],
],
dim=1,
)
def _slice_unpadded_weights(
weights: WeightTensors,
quant_config: TestMoEQuantConfig | None,
k_unpadded: int,
n_unpadded: int,
) -> WeightTensors:
"""Derive the unpadded sub-block of a padded WeightTensors: same values,
restricted to the region hidden_pad/intermediate_pad keeps.
Scale-shape choice (per_out_ch) must match _make_padding_matrix_weights()."""
block_shape = quant_config.block_shape if quant_config is not None else None
per_out_ch = quant_config is not None and quant_config.per_out_ch_quant
w1 = _slice_gate_up_rows(weights.w1[:, :, :k_unpadded], n_unpadded)
w2 = weights.w2[:, :k_unpadded, :n_unpadded]
if weights.w1_scale is None:
w1_scale = w2_scale = None
elif weights.w2_scale is None:
raise AssertionError("w1_scale and w2_scale must both be set or both None")
elif block_shape is not None:
block_n, block_k = block_shape
assert n_unpadded % block_n == 0 and k_unpadded % block_k == 0
n_scale, k_scale = n_unpadded // block_n, k_unpadded // block_k
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_scale)[..., :k_scale]
w2_scale = weights.w2_scale[:, :k_scale, :n_scale]
elif per_out_ch:
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_unpadded)
w2_scale = weights.w2_scale[:, :k_unpadded, :]
else:
# Per-tensor scale: shape (E, 1, 1), independent of K/N -- reuse as-is.
w1_scale = weights.w1_scale
w2_scale = weights.w2_scale
return WeightTensors(w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale)
def _make_padding_matrix_weights(config: Config) -> WeightTensors:
"""Like WeightTensors.make(), but scales weights by
config.is_per_out_ch_quant instead of is_per_act_token_quant.
WeightTensors.make() ties weight-scale shape to activation quant, so
fp8_tensor_token would get per-channel scales despite being a per-tensor
weight scheme. Scoped here rather than fixing WeightTensors.make(),
which other tests depend on."""
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
e=config.E,
n=config.N,
k=config.K,
in_dtype=config.dtype,
quant_dtype=config.quant_dtype,
block_shape=config.quant_block_shape,
per_out_ch_quant=config.is_per_out_ch_quant,
)
return WeightTensors(
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
)
def _slice_unpadded_rank_tensors(
rank_tensors: RankTensors, k_unpadded: int
) -> RankTensors:
return RankTensors(
hidden_states=rank_tensors.hidden_states[:, :k_unpadded].contiguous(),
hidden_states_scale=rank_tensors.hidden_states_scale,
topk_weights=rank_tensors.topk_weights,
topk_ids=rank_tensors.topk_ids,
expert_map=rank_tensors.expert_map,
)
def _aiter_padding_matrix_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
padded_config: Config,
padded_weights: WeightTensors,
verbose: bool,
hidden_pad_expected: int,
intermediate_pad_expected: int,
):
device = torch.device(f"cuda:{pgi.local_rank}")
init_workspace_manager(device)
set_random_seed(pgi.rank)
weights = copy.deepcopy(padded_weights)
weights.to_current_device()
rank_tensors = RankTensors.make(padded_config, pgi)
with mock.patch.object(
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
) as fused_moe_mock:
mk_out = run_modular_kernel(
pgi, vllm_config, padded_config, weights, rank_tensors
)
assert fused_moe_mock.call_count > 0, (
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
)
for call in fused_moe_mock.call_args_list:
assert call.kwargs["hidden_pad"] == hidden_pad_expected, (
f"AiterExperts.apply() forwarded hidden_pad="
f"{call.kwargs['hidden_pad']}, expected {hidden_pad_expected}."
)
assert call.kwargs["intermediate_pad"] == intermediate_pad_expected, (
f"AiterExperts.apply() forwarded intermediate_pad="
f"{call.kwargs['intermediate_pad']}, expected "
f"{intermediate_pad_expected}."
)
# AiterExperts.apply() never slices its own output -- the buffer stays
# raw (padded) width; only the caller-sliced unpadded prefix is checked.
assert mk_out.shape[-1] == padded_config.K, (
f"AiterExperts output width {mk_out.shape[-1]} != raw hidden_dim "
f"{padded_config.K}."
)
mk_out = mk_out[..., :_PADDING_K_UNPADDED]
# Only the real slice is checked: beyond it is unspecified/reused memory.
assert torch.isfinite(mk_out).all(), (
"AiterExperts output must not contain NaN/Inf in the real "
"hidden_dim_unpadded-wide output slice."
)
unpadded_weights = _slice_unpadded_weights(
weights,
padded_config.quant_config,
_PADDING_K_UNPADDED,
_PADDING_N_UNPADDED,
)
unpadded_rank_tensors = _slice_unpadded_rank_tensors(
rank_tensors, _PADDING_K_UNPADDED
)
with set_current_vllm_config(vllm_config):
ref_out = reference_moe_impl(
padded_config, unpadded_weights, unpadded_rank_tensors
)
# ref_out's magnitude here (~1e-3-1e-2) is below atol=3e-2, so a zeroed-out
# mk_out would still pass check_accuracy below -- guard against that.
ref_scale = ref_out.abs().mean()
mk_scale = mk_out.abs().mean()
assert mk_scale > 0.5 * ref_scale, (
f"AiterExperts output magnitude (mean |mk_out|={mk_scale:.6f}) looks "
f"degenerate/zeroed vs. reference (mean |ref_out|={ref_scale:.6f})."
)
# Lenient check for every scheme, including unquantized: a strict
# assert_close(atol=rtol=3e-2) spuriously fails ~0.1-0.2% of elements
# on MI300 even for unquantized AiterExperts (confirmed on hardware).
check_accuracy(ref_out, mk_out, atol=3e-2, rtol=3e-2, percent=0.9)
@require_aiter_moe
@pytest.mark.parametrize("quant_config", _PADDING_QUANT_CONFIGS, ids=_PADDING_QUANT_IDS)
@pytest.mark.parametrize("mode", list(_PADDING_MODES))
def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig | None):
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
Exercises AiterExperts's hidden_pad/intermediate_pad (rocm_aiter_moe.py)
across {no padding, hidden-only, intermediate-only, both} x the
unquantized + 4 fp8 quant schemes it supports. Compares against an
unpadded reference and verifies the hidden_pad/intermediate_pad
AiterExperts.apply() forwards to rocm_aiter_ops.fused_moe.
"""
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
pad_hidden, pad_intermediate = _PADDING_MODES[mode]
if (
quant_config is not None
and quant_config.block_shape is not None
and not pad_hidden
):
pytest.skip(
"AITER's block-quantized (128x128) CK GEMM kernel does not "
f"support the raw (unpadded) hidden_dim={_PADDING_K_UNPADDED} "
"shape on this hardware -- it raises 'wrong! device_gemm with "
"the specified compilation parameters does not support this "
"GEMM problem'. Only hidden_pad-forcing modes (raw "
f"hidden_dim={_PADDING_K_PADDED}) are exercised for this quant "
"scheme."
)
k = _PADDING_K_PADDED if pad_hidden else _PADDING_K_UNPADDED
n = _PADDING_N_PADDED if pad_intermediate else _PADDING_N_UNPADDED
config = Config(
Ms=_PADDING_M,
K=k,
N=n,
E=_PADDING_E,
topks=_PADDING_TOPK,
dtype=torch.bfloat16,
quant_config=quant_config,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
world_size=1,
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
intermediate_size_per_partition_unpadded=(
_PADDING_N_UNPADDED if pad_intermediate else None
),
)
assert config.is_valid()[0]
assert config.fe_supports_quant_scheme(), (
f"AiterExperts does not support quant scheme {quant_config}."
)
weights = _make_padding_matrix_weights(config)
vllm_config, env_dict = config.make_env_data()
hidden_pad_expected = _PADDING_HIDDEN_PAD_EXPECTED if pad_hidden else 0
intermediate_pad_expected = (
_PADDING_INTERMEDIATE_PAD_EXPECTED if pad_intermediate else 0
)
parallel_launch_with_config(
config.world_size,
_aiter_padding_matrix_worker,
vllm_config,
env_dict,
config,
weights,
False,
hidden_pad_expected,
intermediate_pad_expected,
)
@pytest.mark.skip(
reason="MXFP4 AiterExperts padding requires MI350/gfx950 hardware. "
'See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").'
)
def test_aiter_moe_padding_matrix_mxfp4():
"""Stub for the {no padding, hidden-only, intermediate-only, both} MXFP4
padding matrix once MI350/gfx950 hardware is available in CI."""
if __name__ == "__main__":
# Ability to test individual PrepareAndFinalize and FusedExperts combination
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
+9 -7
View File
@@ -16,10 +16,8 @@ from vllm.utils.torch_utils import set_random_seed
NUM_TOKENS = [1, 17, 256]
TOP_KS = [2, 8]
HIDDEN_SIZE = 512
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
HIDDEN_SIZES = [512, 1000, 3072, 7168]
# Include partial tiles and hidden sizes above the CUDA 4096-element tile cap.
HIDDEN_SIZES = [512, 1000, 3072, 4097, 7168]
DTYPES = [torch.bfloat16, torch.float32]
NUM_EXPERTS = 16
@@ -233,7 +231,7 @@ def test_nonlocal_and_padding_adjacent():
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("dtype", [*DTYPES, torch.float16])
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_weighted_sum_matches_reference(
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
@@ -268,7 +266,11 @@ def test_weighted_sum_matches_reference(
assert not out.isnan().any(), "invalid slots leaked into the reduction"
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
atol, rtol = {
torch.bfloat16: (1e-2, 1e-2),
torch.float16: (1e-3, 1e-3),
torch.float32: (1e-4, 1e-4),
}[dtype]
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
@@ -360,7 +362,7 @@ def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
"""
set_random_seed(0)
device = "cuda"
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 7168
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
+2
View File
@@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize(
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=None,
),
)
return
@@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle(
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
routing_sink=None,
)
else:
# Modular impl uses topk_weights and topk_ids
File diff suppressed because it is too large Load Diff
+30
View File
@@ -91,6 +91,12 @@ QWEN4_EXP_SM100_CASES = [
for num_tokens, config in plans.items()
]
QWEN4_EXP_SM121_CASES = [
(n, k, num_tokens, config)
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS.items()
for num_tokens, config in plans.items()
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
@@ -725,6 +731,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS),
((8, 0), {}),
],
)
@@ -939,6 +946,29 @@ def test_qwen4_exp_sm100_selected_shapes(
assert cosine > 0.999
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM121_CASES)
def test_qwen4_exp_sm121_selected_shapes(
n: int,
k: int,
num_tokens: int,
config: SkinnyGemmConfig,
) -> None:
_require_capability_and_cute((12, 1))
torch.manual_seed(42 + num_tokens)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
selected = qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS[(n, k)][num_tokens]
assert selected == config
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
_require_sm103_and_cute()
spec = glm52_gemm.GLM52_Q_B_PROJECTION
+225 -118
View File
@@ -636,8 +636,8 @@ def test_top_k_per_row_decode_gfx950_k512_masked_graph_replay() -> None:
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
@torch.inference_mode()
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_get_aiter_top_k_kernel,
_launch_aiter_top_k_per_row_prefill,
)
@@ -647,17 +647,14 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
top_k = 4
indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda")
aiter_topk_kernel = _get_aiter_top_k_kernel(
if not rocm_aiter_ops.is_indexer_top_k_supported(
is_prefill=True,
compress_ratio=4,
num_rows=logits.shape[0],
on_gfx950=True,
)
if aiter_topk_kernel is None:
):
pytest.skip("AITER top-k is unavailable")
assert (
_launch_aiter_top_k_per_row_prefill(
aiter_topk_kernel,
logits,
row_starts,
row_ends,
@@ -683,10 +680,7 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
num_speculative_tokens: int,
) -> None:
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_get_aiter_top_k_kernel,
_launch_aiter_top_k_per_row_decode,
)
from vllm._aiter_ops import rocm_aiter_ops
query_tokens = num_speculative_tokens + 1
offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
@@ -701,20 +695,18 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
top_k = 4
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
aiter_topk_kernel = _get_aiter_top_k_kernel(
if not rocm_aiter_ops.is_indexer_top_k_supported(
is_prefill=False,
compress_ratio=4,
num_rows=num_rows,
max_valid_seq_len=seq_lens.max().item(),
on_gfx950=True,
)
if aiter_topk_kernel is None:
):
pytest.skip("AITER top-k is unavailable")
assert (
_launch_aiter_top_k_per_row_decode(
aiter_topk_kernel,
rocm_aiter_ops.indexer_top_k_decode(
logits,
seq_lens,
1,
seq_lens.reshape(-1),
indices,
top_k,
)
@@ -729,130 +721,107 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
assert torch.all(indices[row_idx, num_valid:] == -1)
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
from vllm.v1.attention.ops import rocm_aiter_mla_sparse
@requires_gfx950
@torch.inference_mode()
def test_aiter_decode_topk_matches_per_row_under_mtp() -> None:
"""The aiter backend expands (B, 1) seq_lens into per-row ends before
launching, so next_n > 1 must agree with the in-tree kernel, including
the rows that clamp to a zero-length prefix.
"""
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
def prefill_kernel(*args, **kwargs) -> None:
pass
next_n = 4
top_k = 512
num_columns = 1024
seq_lens = torch.tensor([[2], [513], [700]], dtype=torch.int32, device="cuda")
num_rows = seq_lens.shape[0] * next_n
def decode_kernel(*args, **kwargs) -> None:
pass
torch.manual_seed(0)
logits = torch.randn(num_rows, num_columns, dtype=torch.float32, device="cuda")
outputs = {}
for backend in ("aiter", "per_row"):
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
get_indexer_topk(backend)(logits, seq_lens, next_n, indices, top_k, num_columns)
outputs[backend] = indices
row_ends = (
(seq_lens - next_n + 1 + torch.arange(next_n, device="cuda"))
.clamp_(min=0)
.reshape(-1)
.tolist()
)
assert row_ends[:next_n] == [0, 0, 1, 2]
for row_idx, row_end in enumerate(row_ends):
num_valid = min(top_k, row_end)
aiter_row = outputs["aiter"][row_idx]
assert set(aiter_row[:num_valid].tolist()) == set(
outputs["per_row"][row_idx][:num_valid].tolist()
)
assert torch.all(aiter_row[num_valid:] == -1)
def _enable_aiter_topk(monkeypatch, enabled: bool = True) -> None:
from vllm._aiter_ops import rocm_aiter_ops
monkeypatch.setattr(
rocm_aiter_mla_sparse,
"_get_aiter_topk_ops",
lambda: (prefill_kernel, decode_kernel),
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: enabled, raising=True
)
get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
from vllm._aiter_ops import rocm_aiter_ops
_enable_aiter_topk(monkeypatch)
is_supported = rocm_aiter_ops.is_indexer_top_k_supported
eligible_cases = [
(
dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True),
prefill_kernel,
dict(is_prefill=True, compress_ratio=4, num_rows=1),
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_536),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=257,
max_valid_seq_len=250_000,
),
(
dict(
is_prefill=False,
compress_ratio=4,
num_rows=1,
max_valid_seq_len=65_536,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=4,
num_rows=257,
max_valid_seq_len=250_000,
num_columns=524_288,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=2,
num_rows=385,
max_valid_seq_len=250_000,
num_columns=524_288,
topk_tokens=512,
on_gfx950=True,
),
decode_kernel,
),
(
dict(
is_prefill=False,
compress_ratio=2,
num_rows=384,
max_valid_seq_len=50_000,
num_columns=1_048_577,
topk_tokens=512,
on_gfx950=True,
),
decode_kernel,
),
]
for kwargs, expected_kernel in eligible_cases:
assert get_kernel(**kwargs) is expected_kernel
def fail_if_imported() -> None:
pytest.fail("ineligible shapes must not import AITER top-k")
monkeypatch.setattr(
rocm_aiter_mla_sparse,
"_get_aiter_topk_ops",
fail_if_imported,
)
ineligible_cases = [
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
dict(
is_prefill=False,
compress_ratio=2,
num_rows=384,
max_valid_seq_len=1_048_576,
num_columns=1_048_576,
topk_tokens=512,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=1,
max_valid_seq_len=65_537,
on_gfx950=True,
),
]
for kwargs in eligible_cases:
assert is_supported(**kwargs) is True
ineligible_cases = [
dict(is_prefill=True, compress_ratio=1, num_rows=1),
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_537),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=256,
max_valid_seq_len=125_000,
on_gfx950=True,
),
dict(
is_prefill=False,
compress_ratio=4,
num_rows=320,
max_valid_seq_len=2_500,
on_gfx950=False,
),
]
for kwargs in ineligible_cases:
assert get_kernel(**kwargs) is None
assert is_supported(**kwargs) is False
monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None)
assert (
get_kernel(
is_prefill=True,
compress_ratio=4,
num_rows=1,
on_gfx950=True,
)
is None
)
_enable_aiter_topk(monkeypatch, enabled=False)
assert is_supported(is_prefill=True, compress_ratio=4, num_rows=1) is False
def test_dsv4_native_top_k_window() -> None:
"""The in-tree decode kernel's measured advantage band over AITER, which
applies to the DSV4 indexer only."""
from vllm._aiter_ops import rocm_aiter_ops
prefers_native = rocm_aiter_ops.dsv4_indexer_prefers_native_top_k
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=512) is True
assert prefers_native(num_rows=385, num_columns=524_288, topk_tokens=512) is False
assert prefers_native(num_rows=128, num_columns=1_048_577, topk_tokens=512) is False
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=1024) is False
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
@@ -1927,3 +1896,141 @@ def test_sparse_indexer_topk_backend_resolution() -> None:
if not is_sm100:
with pytest.raises(RuntimeError, match="SM100a/SM103a"):
resolve("deep_select")
def _patch_aiter_topk(monkeypatch, decode_kernel, enabled: bool = True) -> None:
from vllm._aiter_ops import rocm_aiter_ops
_enable_aiter_topk(monkeypatch, enabled=enabled)
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: enabled, raising=True)
monkeypatch.setattr(rocm_aiter_ops, "indexer_top_k_decode", decode_kernel)
def _make_topk(backend: str):
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
with set_current_vllm_config(cfg):
return SparseIndexerTopk()
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
def test_sparse_indexer_topk_aiter_requires_rocm() -> None:
"""Explicitly requesting the ROCm-only backend must fail fast elsewhere."""
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
with pytest.raises(RuntimeError, match="ROCm"):
_make_topk("aiter").resolve_backend(logits, 2048, 8)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
def test_sparse_indexer_topk_backend_resolution_aiter(monkeypatch) -> None:
"""Aiter is opt-in only: "auto" never selects it (the shape-dependent perf
gate lives at the call site), and an explicit request validates hard
capability only."""
def decode_kernel(*args, **kwargs) -> None:
pass
_patch_aiter_topk(monkeypatch, decode_kernel)
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
assert _make_topk("aiter").resolve_backend(logits, 2048, 8) == "aiter"
# Shapes the call-site perf gate would decline still resolve.
assert _make_topk("aiter").resolve_backend(logits, 512, 8) == "aiter"
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
_make_topk("aiter").resolve_backend(logits, 3000, 8)
_enable_aiter_topk(monkeypatch, enabled=False)
with pytest.raises(RuntimeError, match="no tuned indexer top-k kernels"):
_make_topk("aiter").resolve_backend(logits, 2048, 8)
_patch_aiter_topk(monkeypatch, decode_kernel, enabled=False)
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
with pytest.raises(RuntimeError, match="VLLM_ROCM_USE_AITER"):
_make_topk("aiter").resolve_backend(logits, 2048, 8)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
@torch.inference_mode()
def test_sparse_indexer_topk_aiter_applies_no_shape_gate(monkeypatch) -> None:
"""Shape-dependent perf gating belongs to the call site: once "aiter" is
requested the dispatcher must run it for every supported k, including the
k=512 band the in-tree kernel wins on DSV4's compressed-KV logits."""
from vllm.model_executor.layers import indexer_topk
calls: list[tuple] = []
_patch_aiter_topk(monkeypatch, lambda *args: calls.append(args))
monkeypatch.setattr(
indexer_topk.ops,
"top_k_per_row_decode",
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
)
num_rows = 8
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
indices = torch.empty((num_rows, 512), dtype=torch.int32, device="cuda")
op = _make_topk("aiter")
op(logits, seq_lens, 1, indices, 512, 4096)
# Past the compressed-context boundary the call-site gate declines on.
op(logits, seq_lens, 1, indices, 512, 4 * (64 * 1024 + 1))
assert len(calls) == 2
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
@torch.inference_mode()
def test_sparse_indexer_topk_auto_stays_on_per_row(monkeypatch) -> None:
"""The "auto" chain must not reach AITER on its own: the decode top-k runs
through top_k_per_row_decode unless the caller asks for the aiter backend."""
from vllm.model_executor.layers import indexer_topk
def decode_kernel(*args, **kwargs) -> None:
pytest.fail("AITER is opt-in only")
_patch_aiter_topk(monkeypatch, decode_kernel)
calls: list[tuple] = []
monkeypatch.setattr(
indexer_topk.ops, "top_k_per_row_decode", lambda *args: calls.append(args)
)
num_rows = 8
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
indices = torch.empty((num_rows, 1024), dtype=torch.int32, device="cuda")
_make_topk("auto")(logits, seq_lens, 1, indices, 1024, 4096)
assert len(calls) == 1
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
@torch.inference_mode()
def test_sparse_indexer_topk_aiter_expands_mtp_row_ends(monkeypatch) -> None:
"""AITER's decode kernel only implements the next_n == 1 form, so spec
decode batches must be handed the equivalent expanded per-row ends."""
from vllm.model_executor.layers import indexer_topk
recorded: dict = {}
def decode_kernel(logits, next_n, seq_lens, *args, **kwargs) -> None:
recorded["next_n"] = next_n
recorded["row_ends"] = seq_lens.tolist()
_patch_aiter_topk(monkeypatch, decode_kernel)
monkeypatch.setattr(
indexer_topk.ops,
"top_k_per_row_decode",
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
)
next_n = 2
logits = torch.randn(4, 4096, dtype=torch.float32, device="cuda")
seq_lens = torch.tensor([[100], [200]], dtype=torch.int32, device="cuda")
indices = torch.empty((4, 1024), dtype=torch.int32, device="cuda")
_make_topk("aiter")(logits, seq_lens, next_n, indices, 1024, 800)
assert recorded["next_n"] == 1
assert recorded["row_ends"] == [99, 100, 199, 200]