Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -107,3 +107,37 @@ def test_fp8_index_query_and_cache_prefill_topk_matches_bf16() -> None:
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not FP8_CUDA_SUPPORTED, reason="CUDA FP8 support required")
|
||||
@torch.inference_mode()
|
||||
def test_bf16_query_against_fp8_cache_decode_topk_matches_bf16() -> None:
|
||||
"""A mixed pair must upcast rather than emit a mixed-dtype fp8 dot.
|
||||
|
||||
Production allocates index_q with the index-K cache dtype, so the scorers
|
||||
normally see a matched pair and keep the native FP8 MMA. This pins the
|
||||
other branch: a bf16 query against an e4m3 cache still compiles and still
|
||||
ranks the planted blocks.
|
||||
"""
|
||||
dev = "cuda"
|
||||
q, keys = _make_inputs(dev)
|
||||
block_table = torch.arange(NUM_PAGES, dtype=torch.int32, device=dev).unsqueeze(0)
|
||||
seq_lens = torch.tensor([SEQ_LEN], dtype=torch.int32, device=dev)
|
||||
|
||||
def select(cache: torch.Tensor) -> set[int]:
|
||||
idx = minimax_m3_index_decode(
|
||||
q, # bf16 query, deliberately not cast to the cache dtype
|
||||
cache.view(NUM_PAGES, PAGE, HEAD_DIM).contiguous(),
|
||||
block_table,
|
||||
seq_lens,
|
||||
SEQ_LEN,
|
||||
TOPK,
|
||||
0,
|
||||
0,
|
||||
HEADS,
|
||||
1,
|
||||
1,
|
||||
)
|
||||
return {int(b) for b in idx[0, 0].tolist() if b >= 0}
|
||||
|
||||
_check_selection(select(keys), select(keys.to(torch.float8_e4m3fn)))
|
||||
|
||||
@@ -596,6 +596,114 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(7)
|
||||
head_dim = nope_dim + rope_dim
|
||||
return dict(
|
||||
q=torch.randn(3, 3, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
|
||||
kv=torch.randn(5, head_dim, dtype=torch.bfloat16, device=device) * 0.125,
|
||||
indices=torch.tensor([0, 2, 1, 3, 4], dtype=torch.int32, device=device),
|
||||
indptr=torch.tensor([0, 2, 5, 5], dtype=torch.int32, device=device),
|
||||
scale=head_dim**-0.5,
|
||||
attn_sink=torch.tensor([-0.25, 0.0, 0.25], dtype=torch.float32, device=device),
|
||||
nope_head_dim=nope_dim,
|
||||
rope_head_dim=rope_dim,
|
||||
)
|
||||
|
||||
|
||||
# 448+64 is DeepSeek V4/V4.1, 512+64 is V3.2, and 256+0 stands in for the
|
||||
# NoPE-only layouts where the destination spans the whole head dim.
|
||||
SPARSE_PREFILL_DIMS = [(NOPE_HEAD_DIM, ROPE_HEAD_DIM), (512, 64), (256, 0)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_out_matches_allocated(
|
||||
nope_dim: int, rope_dim: int
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
|
||||
allocated = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)
|
||||
|
||||
q = kwargs["q"]
|
||||
dest = torch.empty(q.shape[0], q.shape[1], nope_dim, dtype=q.dtype, device=q.device)
|
||||
returned = _rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
assert returned is dest
|
||||
torch.testing.assert_close(dest, allocated[..., :nope_dim], atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("nope_dim", "rope_dim"), SPARSE_PREFILL_DIMS)
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_out_view_leaves_neighbors_untouched(
|
||||
nope_dim: int, rope_dim: int
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(nope_dim, rope_dim)
|
||||
q = kwargs["q"]
|
||||
num_queries, num_heads = q.shape[0], q.shape[1]
|
||||
|
||||
buffer = torch.full(
|
||||
(num_queries + 4, num_heads + 2, nope_dim),
|
||||
-7.0,
|
||||
dtype=q.dtype,
|
||||
device=q.device,
|
||||
)
|
||||
dest = buffer[2 : 2 + num_queries, :num_heads]
|
||||
assert not dest.is_contiguous()
|
||||
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :nope_dim]
|
||||
torch.testing.assert_close(dest, expected, atol=0.0, rtol=0.0)
|
||||
|
||||
untouched = torch.ones_like(buffer, dtype=torch.bool)
|
||||
untouched[2 : 2 + num_queries, :num_heads] = False
|
||||
assert torch.all(buffer[untouched] == -7.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("out_dtype", [torch.float32, torch.float16])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_ragged_out_casts_to_dest_dtype(
|
||||
out_dtype: torch.dtype,
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(512, 64)
|
||||
q = kwargs["q"]
|
||||
dest = torch.empty(q.shape[0], q.shape[1], 512, dtype=out_dtype, device=q.device)
|
||||
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(**kwargs)[..., :512]
|
||||
assert dest.dtype == out_dtype
|
||||
torch.testing.assert_close(dest.float(), expected.float(), atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_prefill_head_dim_wide_out_keeps_rope_rows() -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
kwargs = _sparse_prefill_ragged_inputs(NOPE_HEAD_DIM, ROPE_HEAD_DIM)
|
||||
q = kwargs["q"]
|
||||
dest = torch.empty_like(q)
|
||||
_rocm_sparse_attn_prefill_ragged_triton(**kwargs, out=dest)
|
||||
|
||||
expected = _ref_sparse_prefill_ragged(
|
||||
q, kwargs["kv"], [[0, 2], [1, 3, 4], []], kwargs["scale"], kwargs["attn_sink"]
|
||||
)
|
||||
torch.testing.assert_close(dest, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_queries", "on_gfx950", "expected"),
|
||||
[(1023, True, False), (1024, True, True), (1024, False, False)],
|
||||
@@ -718,10 +826,11 @@ def test_sparse_attn_prefill_preserves_dense_triton_fallback(monkeypatch) -> Non
|
||||
output = torch.empty_like(q)
|
||||
dense_fallback_calls = 0
|
||||
|
||||
def fake_dense_fallback(*args, **kwargs):
|
||||
def fake_dense_fallback(*args, out, **kwargs):
|
||||
nonlocal dense_fallback_calls
|
||||
dense_fallback_calls += 1
|
||||
return torch.zeros_like(q)
|
||||
out.zero_()
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(mod, "_can_use_aiter_sparse_prefill_opus", lambda *args: True)
|
||||
monkeypatch.setattr(mod, "_get_aiter_sparse_prefill_opus", lambda: None)
|
||||
|
||||
@@ -194,7 +194,7 @@ def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
ba = torch.randn(num_tokens, 2 * HV, dtype=torch.bfloat16, device=device)
|
||||
layer = types.SimpleNamespace(
|
||||
prefix=PREFIX,
|
||||
enable_fused_gdn_decode=True,
|
||||
enable_fused_gdn_spec_decode=True,
|
||||
norm=types.SimpleNamespace(
|
||||
weight=torch.empty(V, dtype=torch.bfloat16, device=device)
|
||||
),
|
||||
|
||||
@@ -167,7 +167,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
|
||||
draft = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
use_native_mega_moe=False,
|
||||
use_sequence_parallel=False,
|
||||
hc_mult=hc_mult,
|
||||
layers=[make_layer(mix) for mix in mixes],
|
||||
|
||||
@@ -88,6 +88,15 @@ class Config:
|
||||
|
||||
torch_trace_dir_path: str | None = None
|
||||
|
||||
# Force AiterExperts's hidden_pad/intermediate_pad computation
|
||||
# (`experts/rocm_aiter_moe.py`) to diverge from the padded K/N sizes above.
|
||||
# None (default) preserves today's behavior: FusedMoEConfig defaults both
|
||||
# to the (unpadded) K/intermediate_size_per_partition, so hidden_pad and
|
||||
# intermediate_pad come out to 0.
|
||||
# See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
hidden_dim_unpadded: int | None = None
|
||||
intermediate_size_per_partition_unpadded: int | None = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.quant_config is None:
|
||||
self.quant_config = TestMoEQuantConfig(None, False, False, None)
|
||||
@@ -664,6 +673,10 @@ def make_modular_kernel(
|
||||
activation=config.activation,
|
||||
device=vllm_config.device_config.device,
|
||||
routing_method=RoutingMethodType.DeepSeekV3,
|
||||
hidden_dim_unpadded=config.hidden_dim_unpadded,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
config.intermediate_size_per_partition_unpadded
|
||||
),
|
||||
)
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
|
||||
@@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph(
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=True,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_sink=None,
|
||||
)
|
||||
|
||||
check_accuracy(
|
||||
@@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
|
||||
router_logits=logits,
|
||||
e_score_correction_bias=bias,
|
||||
routed_scaling_factor=scale,
|
||||
routing_sink=None,
|
||||
**common,
|
||||
)
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
|
||||
|
||||
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
|
||||
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
|
||||
MoE geometry of a real model, warms it up and runs a forward whose inputs are
|
||||
chosen so that every quantization step in the megakernel is exact, for the
|
||||
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
|
||||
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
|
||||
32). The expected output is then known in closed form and compared bit for
|
||||
bit.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
FlashInferMoeEp,
|
||||
FlashInferMoeEpWeights,
|
||||
apply_topk_in_fc1,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Geometry:
|
||||
name: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_experts: int
|
||||
top_k: int
|
||||
|
||||
|
||||
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
|
||||
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
|
||||
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
|
||||
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
|
||||
GEOMETRIES = (
|
||||
Geometry("dsv4-pro", 7168, 3072, 384, 6),
|
||||
Geometry("dsv4-flash", 4096, 2048, 256, 6),
|
||||
Geometry("mistral-small-4", 4096, 2048, 128, 4),
|
||||
Geometry("mistral-large-3", 7168, 4096, 128, 4),
|
||||
)
|
||||
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
|
||||
NUM_TOKENS_PER_RANK = 4
|
||||
MAX_TOKENS_PER_RANK = 128
|
||||
|
||||
# Every block scale below is either 1 or a power of two, so the FP4 codes
|
||||
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
|
||||
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
|
||||
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
|
||||
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
|
||||
#
|
||||
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
|
||||
# it; every weight column the sentinel meets is zero.
|
||||
X_SIGNAL = 2.0
|
||||
X_SENTINEL = 6.0
|
||||
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
|
||||
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
|
||||
W13_DIAG = 6.0
|
||||
H_SIGNAL = 144.0
|
||||
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
|
||||
# sentinel sits next to a zero intermediate channel and pins the block scale.
|
||||
W2_SENTINEL = 6.0
|
||||
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
|
||||
|
||||
|
||||
def _expert_out(expert: int) -> float:
|
||||
return E2M1_CODES[expert % len(E2M1_CODES)]
|
||||
|
||||
|
||||
def _topk_weights(top_k: int) -> tuple[float, ...]:
|
||||
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
|
||||
# weight * 144 must requantize exactly too: powers of two do.
|
||||
return tuple(2.0 ** -(k + 1) for k in range(top_k))
|
||||
|
||||
|
||||
def _routing_method(backend: str) -> RoutingMethodType:
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
|
||||
return RoutingMethodType.DeepseekV4
|
||||
return RoutingMethodType.Default
|
||||
|
||||
|
||||
def _routing(
|
||||
geometry: Geometry, rank: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""top_k distinct experts per token, strided across ranks so dispatch goes
|
||||
remote."""
|
||||
base = (
|
||||
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
|
||||
) % geometry.num_experts
|
||||
stride = geometry.num_experts // geometry.top_k
|
||||
offsets = torch.arange(geometry.top_k, device=device) * stride
|
||||
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
|
||||
torch.int32
|
||||
)
|
||||
topk_weights = torch.tensor(
|
||||
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
return topk_ids, topk_weights
|
||||
|
||||
|
||||
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
|
||||
x = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
x[t, 16 * t] = X_SIGNAL
|
||||
x[t, 16 * t + 1] = X_SENTINEL
|
||||
return x
|
||||
|
||||
|
||||
def _local_expert_weights(
|
||||
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
|
||||
) -> FlashInferMoeEpWeights:
|
||||
w13 = torch.zeros(
|
||||
num_local_experts,
|
||||
2 * geometry.intermediate_size,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
w2 = torch.zeros(
|
||||
num_local_experts,
|
||||
geometry.hidden_size,
|
||||
geometry.intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
|
||||
for local in range(num_local_experts):
|
||||
expert = rank * num_local_experts + local
|
||||
w13[local, 0, token_columns] = W13_DIAG
|
||||
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
|
||||
w2[local, expert, 0] = _expert_out(expert)
|
||||
w2[local, expert, 2] = W2_SENTINEL
|
||||
return FlashInferMoeEpWeights(w13=w13, w2=w2)
|
||||
|
||||
|
||||
def _expected_output(
|
||||
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.float32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
for k in range(geometry.top_k):
|
||||
expert = int(topk_ids[t, k])
|
||||
weight = float(topk_weights[t, k])
|
||||
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
|
||||
return expected.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _moe_config(
|
||||
vllm_config: VllmConfig,
|
||||
geometry: Geometry,
|
||||
world_size: int,
|
||||
device: torch.device,
|
||||
backend: str,
|
||||
) -> FusedMoEConfig:
|
||||
parallel = FusedMoEParallelConfig.make(
|
||||
tp_size_=1,
|
||||
pcp_size_=1,
|
||||
dp_size_=world_size,
|
||||
sp_size_=1,
|
||||
vllm_parallel_config=vllm_config.parallel_config,
|
||||
)
|
||||
return FusedMoEConfig(
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
num_local_experts=geometry.num_experts // world_size,
|
||||
num_logical_experts=geometry.num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device=device,
|
||||
routing_method=_routing_method(backend),
|
||||
moe_parallel_config=parallel,
|
||||
in_dtype=torch.bfloat16,
|
||||
moe_backend=backend,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
)
|
||||
|
||||
|
||||
def _check_expert_views_alias_kernel_memory(
|
||||
adapter: FlashInferMoeEp,
|
||||
geometry: Geometry,
|
||||
rank: int,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
|
||||
|
||||
Swap local experts 0 and 1 in place on every rank, then route through the
|
||||
swapped physical slots: the output must be the one the unswapped logical
|
||||
experts would have produced.
|
||||
"""
|
||||
for view in adapter.expert_weight_views():
|
||||
first = view[0].clone()
|
||||
view[0].copy_(view[1])
|
||||
view[1].copy_(first)
|
||||
torch.distributed.barrier()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
slot = topk_ids % num_local_experts
|
||||
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
|
||||
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
|
||||
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
|
||||
|
||||
|
||||
def _init_warmup_forward(
|
||||
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
|
||||
) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.data_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.kernel_config.moe_backend = backend
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
|
||||
)
|
||||
rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
adapter = FlashInferMoeEp(
|
||||
moe,
|
||||
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
try:
|
||||
adapter.warmup()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = _expected_output(geometry, topk_ids, topk_weights)
|
||||
assert output.dtype == expected.dtype and output.shape == expected.shape
|
||||
assert torch.equal(output, expected), (
|
||||
f"rank {rank}: max abs diff "
|
||||
f"{(output.float() - expected.float()).abs().max().item()}, "
|
||||
f"nonzero output channels per token "
|
||||
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
|
||||
)
|
||||
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL:
|
||||
_check_expert_views_alias_kernel_memory(
|
||||
adapter, geometry, rank, moe.num_local_experts, device
|
||||
)
|
||||
finally:
|
||||
adapter.destroy()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="The FlashInfer MoE-EP megakernels require SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_init_warmup_forward_is_exact_on_grid_inputs(
|
||||
world_size: int, backend: str, geometry: Geometry
|
||||
):
|
||||
pytest.importorskip("flashinfer.moe_ep")
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
|
||||
pytest.skip("DeepGEMM is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs")
|
||||
distributed_run(_init_warmup_forward, world_size, geometry, backend)
|
||||
@@ -42,7 +42,7 @@ from .modular_kernel_tools.parallel_utils import (
|
||||
ProcessGroupInfo,
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from .utils import check_accuracy
|
||||
from .utils import check_accuracy, make_test_weights
|
||||
|
||||
has_any_multi_gpu_package = (
|
||||
has_deep_ep() or has_deep_gemm() or has_flashinfer_cutlass_fused_moe()
|
||||
@@ -691,6 +691,303 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
|
||||
)
|
||||
|
||||
|
||||
# --- 4b: hidden_dim_unpadded/intermediate_size_per_partition_unpadded matrix -
|
||||
|
||||
# K/N intentionally unaligned to AITER's 64/128 granularity (unlike the
|
||||
# shared mk_objects.py defaults) so hidden_pad/intermediate_pad are forced.
|
||||
_PADDING_E = 8
|
||||
_PADDING_M = 16
|
||||
_PADDING_TOPK = 2
|
||||
_PADDING_K_UNPADDED = 896 # 7 * 128
|
||||
_PADDING_K_PADDED = 1024 # +128
|
||||
_PADDING_N_UNPADDED = 384 # 3 * 128
|
||||
_PADDING_N_PADDED = 512 # +128
|
||||
# hidden_pad = (K_padded - hidden_dim_unpadded) // 128 * 128 (rocm_aiter_moe.py)
|
||||
_PADDING_HIDDEN_PAD_EXPECTED = 128
|
||||
# intermediate_pad = (N_padded - intermediate_size_per_partition_unpadded)
|
||||
# // 64 * 64 * (2 if tp_size == 1 else 1) (rocm_aiter_moe.py); tp_size==1 here.
|
||||
_PADDING_INTERMEDIATE_PAD_EXPECTED = 256
|
||||
|
||||
# {no padding, hidden-only, intermediate-only, both} -> (pad_hidden, pad_intermediate)
|
||||
_PADDING_MODES: dict[str, tuple[bool, bool]] = {
|
||||
"none": (False, False),
|
||||
"hidden": (True, False),
|
||||
"intermediate": (False, True),
|
||||
"both": (True, True),
|
||||
}
|
||||
|
||||
# unquantized + the 4 fp8 configs AiterExperts supports (excludes MXFP4,
|
||||
# stubbed below pending MI350/gfx950, and MK_QUANT_CONFIGS[1], unsupported).
|
||||
_PADDING_QUANT_CONFIGS = [
|
||||
MK_QUANT_CONFIGS[0], # unquantized
|
||||
MK_QUANT_CONFIGS[2], # fp8 channel weights / per-token activations
|
||||
MK_QUANT_CONFIGS[3], # fp8 per-tensor weights / per-tensor activations
|
||||
MK_QUANT_CONFIGS[4], # fp8 per-tensor weights / per-token activations
|
||||
MK_QUANT_CONFIGS[5], # fp8 128x128-block weights / 128-block activations
|
||||
]
|
||||
_PADDING_QUANT_IDS = [
|
||||
"unquantized",
|
||||
"fp8_channel_token",
|
||||
"fp8_tensor_tensor",
|
||||
"fp8_tensor_token",
|
||||
"fp8_block_token",
|
||||
]
|
||||
|
||||
|
||||
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
|
||||
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
|
||||
gate/up halves of `real_per_half` size each."""
|
||||
padded_per_half = t.shape[1] // 2
|
||||
return torch.cat(
|
||||
[
|
||||
t[:, :real_per_half],
|
||||
t[:, padded_per_half : padded_per_half + real_per_half],
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
|
||||
|
||||
def _slice_unpadded_weights(
|
||||
weights: WeightTensors,
|
||||
quant_config: TestMoEQuantConfig | None,
|
||||
k_unpadded: int,
|
||||
n_unpadded: int,
|
||||
) -> WeightTensors:
|
||||
"""Derive the unpadded sub-block of a padded WeightTensors: same values,
|
||||
restricted to the region hidden_pad/intermediate_pad keeps.
|
||||
|
||||
Scale-shape choice (per_out_ch) must match _make_padding_matrix_weights()."""
|
||||
block_shape = quant_config.block_shape if quant_config is not None else None
|
||||
per_out_ch = quant_config is not None and quant_config.per_out_ch_quant
|
||||
|
||||
w1 = _slice_gate_up_rows(weights.w1[:, :, :k_unpadded], n_unpadded)
|
||||
w2 = weights.w2[:, :k_unpadded, :n_unpadded]
|
||||
|
||||
if weights.w1_scale is None:
|
||||
w1_scale = w2_scale = None
|
||||
elif weights.w2_scale is None:
|
||||
raise AssertionError("w1_scale and w2_scale must both be set or both None")
|
||||
elif block_shape is not None:
|
||||
block_n, block_k = block_shape
|
||||
assert n_unpadded % block_n == 0 and k_unpadded % block_k == 0
|
||||
n_scale, k_scale = n_unpadded // block_n, k_unpadded // block_k
|
||||
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_scale)[..., :k_scale]
|
||||
w2_scale = weights.w2_scale[:, :k_scale, :n_scale]
|
||||
elif per_out_ch:
|
||||
w1_scale = _slice_gate_up_rows(weights.w1_scale, n_unpadded)
|
||||
w2_scale = weights.w2_scale[:, :k_unpadded, :]
|
||||
else:
|
||||
# Per-tensor scale: shape (E, 1, 1), independent of K/N -- reuse as-is.
|
||||
w1_scale = weights.w1_scale
|
||||
w2_scale = weights.w2_scale
|
||||
|
||||
return WeightTensors(w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale)
|
||||
|
||||
|
||||
def _make_padding_matrix_weights(config: Config) -> WeightTensors:
|
||||
"""Like WeightTensors.make(), but scales weights by
|
||||
config.is_per_out_ch_quant instead of is_per_act_token_quant.
|
||||
|
||||
WeightTensors.make() ties weight-scale shape to activation quant, so
|
||||
fp8_tensor_token would get per-channel scales despite being a per-tensor
|
||||
weight scheme. Scoped here rather than fixing WeightTensors.make(),
|
||||
which other tests depend on."""
|
||||
(_, w1, w1_scale, w1_gs), (_, w2, w2_scale, w2_gs) = make_test_weights(
|
||||
e=config.E,
|
||||
n=config.N,
|
||||
k=config.K,
|
||||
in_dtype=config.dtype,
|
||||
quant_dtype=config.quant_dtype,
|
||||
block_shape=config.quant_block_shape,
|
||||
per_out_ch_quant=config.is_per_out_ch_quant,
|
||||
)
|
||||
return WeightTensors(
|
||||
w1=w1, w2=w2, w1_scale=w1_scale, w2_scale=w2_scale, w1_gs=w1_gs, w2_gs=w2_gs
|
||||
)
|
||||
|
||||
|
||||
def _slice_unpadded_rank_tensors(
|
||||
rank_tensors: RankTensors, k_unpadded: int
|
||||
) -> RankTensors:
|
||||
return RankTensors(
|
||||
hidden_states=rank_tensors.hidden_states[:, :k_unpadded].contiguous(),
|
||||
hidden_states_scale=rank_tensors.hidden_states_scale,
|
||||
topk_weights=rank_tensors.topk_weights,
|
||||
topk_ids=rank_tensors.topk_ids,
|
||||
expert_map=rank_tensors.expert_map,
|
||||
)
|
||||
|
||||
|
||||
def _aiter_padding_matrix_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
padded_config: Config,
|
||||
padded_weights: WeightTensors,
|
||||
verbose: bool,
|
||||
hidden_pad_expected: int,
|
||||
intermediate_pad_expected: int,
|
||||
):
|
||||
device = torch.device(f"cuda:{pgi.local_rank}")
|
||||
init_workspace_manager(device)
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
weights = copy.deepcopy(padded_weights)
|
||||
weights.to_current_device()
|
||||
|
||||
rank_tensors = RankTensors.make(padded_config, pgi)
|
||||
|
||||
with mock.patch.object(
|
||||
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
|
||||
) as fused_moe_mock:
|
||||
mk_out = run_modular_kernel(
|
||||
pgi, vllm_config, padded_config, weights, rank_tensors
|
||||
)
|
||||
|
||||
assert fused_moe_mock.call_count > 0, (
|
||||
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
|
||||
)
|
||||
for call in fused_moe_mock.call_args_list:
|
||||
assert call.kwargs["hidden_pad"] == hidden_pad_expected, (
|
||||
f"AiterExperts.apply() forwarded hidden_pad="
|
||||
f"{call.kwargs['hidden_pad']}, expected {hidden_pad_expected}."
|
||||
)
|
||||
assert call.kwargs["intermediate_pad"] == intermediate_pad_expected, (
|
||||
f"AiterExperts.apply() forwarded intermediate_pad="
|
||||
f"{call.kwargs['intermediate_pad']}, expected "
|
||||
f"{intermediate_pad_expected}."
|
||||
)
|
||||
|
||||
# AiterExperts.apply() never slices its own output -- the buffer stays
|
||||
# raw (padded) width; only the caller-sliced unpadded prefix is checked.
|
||||
assert mk_out.shape[-1] == padded_config.K, (
|
||||
f"AiterExperts output width {mk_out.shape[-1]} != raw hidden_dim "
|
||||
f"{padded_config.K}."
|
||||
)
|
||||
mk_out = mk_out[..., :_PADDING_K_UNPADDED]
|
||||
# Only the real slice is checked: beyond it is unspecified/reused memory.
|
||||
assert torch.isfinite(mk_out).all(), (
|
||||
"AiterExperts output must not contain NaN/Inf in the real "
|
||||
"hidden_dim_unpadded-wide output slice."
|
||||
)
|
||||
|
||||
unpadded_weights = _slice_unpadded_weights(
|
||||
weights,
|
||||
padded_config.quant_config,
|
||||
_PADDING_K_UNPADDED,
|
||||
_PADDING_N_UNPADDED,
|
||||
)
|
||||
unpadded_rank_tensors = _slice_unpadded_rank_tensors(
|
||||
rank_tensors, _PADDING_K_UNPADDED
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ref_out = reference_moe_impl(
|
||||
padded_config, unpadded_weights, unpadded_rank_tensors
|
||||
)
|
||||
|
||||
# ref_out's magnitude here (~1e-3-1e-2) is below atol=3e-2, so a zeroed-out
|
||||
# mk_out would still pass check_accuracy below -- guard against that.
|
||||
ref_scale = ref_out.abs().mean()
|
||||
mk_scale = mk_out.abs().mean()
|
||||
assert mk_scale > 0.5 * ref_scale, (
|
||||
f"AiterExperts output magnitude (mean |mk_out|={mk_scale:.6f}) looks "
|
||||
f"degenerate/zeroed vs. reference (mean |ref_out|={ref_scale:.6f})."
|
||||
)
|
||||
|
||||
# Lenient check for every scheme, including unquantized: a strict
|
||||
# assert_close(atol=rtol=3e-2) spuriously fails ~0.1-0.2% of elements
|
||||
# on MI300 even for unquantized AiterExperts (confirmed on hardware).
|
||||
check_accuracy(ref_out, mk_out, atol=3e-2, rtol=3e-2, percent=0.9)
|
||||
|
||||
|
||||
@require_aiter_moe
|
||||
@pytest.mark.parametrize("quant_config", _PADDING_QUANT_CONFIGS, ids=_PADDING_QUANT_IDS)
|
||||
@pytest.mark.parametrize("mode", list(_PADDING_MODES))
|
||||
def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig | None):
|
||||
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
|
||||
Exercises AiterExperts's hidden_pad/intermediate_pad (rocm_aiter_moe.py)
|
||||
across {no padding, hidden-only, intermediate-only, both} x the
|
||||
unquantized + 4 fp8 quant schemes it supports. Compares against an
|
||||
unpadded reference and verifies the hidden_pad/intermediate_pad
|
||||
AiterExperts.apply() forwards to rocm_aiter_ops.fused_moe.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
pad_hidden, pad_intermediate = _PADDING_MODES[mode]
|
||||
|
||||
if (
|
||||
quant_config is not None
|
||||
and quant_config.block_shape is not None
|
||||
and not pad_hidden
|
||||
):
|
||||
pytest.skip(
|
||||
"AITER's block-quantized (128x128) CK GEMM kernel does not "
|
||||
f"support the raw (unpadded) hidden_dim={_PADDING_K_UNPADDED} "
|
||||
"shape on this hardware -- it raises 'wrong! device_gemm with "
|
||||
"the specified compilation parameters does not support this "
|
||||
"GEMM problem'. Only hidden_pad-forcing modes (raw "
|
||||
f"hidden_dim={_PADDING_K_PADDED}) are exercised for this quant "
|
||||
"scheme."
|
||||
)
|
||||
|
||||
k = _PADDING_K_PADDED if pad_hidden else _PADDING_K_UNPADDED
|
||||
n = _PADDING_N_PADDED if pad_intermediate else _PADDING_N_UNPADDED
|
||||
|
||||
config = Config(
|
||||
Ms=_PADDING_M,
|
||||
K=k,
|
||||
N=n,
|
||||
E=_PADDING_E,
|
||||
topks=_PADDING_TOPK,
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
world_size=1,
|
||||
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
_PADDING_N_UNPADDED if pad_intermediate else None
|
||||
),
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
assert config.fe_supports_quant_scheme(), (
|
||||
f"AiterExperts does not support quant scheme {quant_config}."
|
||||
)
|
||||
|
||||
weights = _make_padding_matrix_weights(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
|
||||
hidden_pad_expected = _PADDING_HIDDEN_PAD_EXPECTED if pad_hidden else 0
|
||||
intermediate_pad_expected = (
|
||||
_PADDING_INTERMEDIATE_PAD_EXPECTED if pad_intermediate else 0
|
||||
)
|
||||
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
_aiter_padding_matrix_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
False,
|
||||
hidden_pad_expected,
|
||||
intermediate_pad_expected,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="MXFP4 AiterExperts padding requires MI350/gfx950 hardware. "
|
||||
'See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").'
|
||||
)
|
||||
def test_aiter_moe_padding_matrix_mxfp4():
|
||||
"""Stub for the {no padding, hidden-only, intermediate-only, both} MXFP4
|
||||
padding matrix once MI350/gfx950 hardware is available in CI."""
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Ability to test individual PrepareAndFinalize and FusedExperts combination
|
||||
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
|
||||
|
||||
@@ -16,10 +16,8 @@ from vllm.utils.torch_utils import set_random_seed
|
||||
NUM_TOKENS = [1, 17, 256]
|
||||
TOP_KS = [2, 8]
|
||||
HIDDEN_SIZE = 512
|
||||
# One BLOCK_K tile, a partial trailing tile, and Kimi-K3's real dims
|
||||
# (moe_intermediate_size 3072, hidden_size 7168) -- exercises the per-tile loop
|
||||
# and its tail mask (BLOCK_K caps at 512 for <=2-byte dtypes).
|
||||
HIDDEN_SIZES = [512, 1000, 3072, 7168]
|
||||
# Include partial tiles and hidden sizes above the CUDA 4096-element tile cap.
|
||||
HIDDEN_SIZES = [512, 1000, 3072, 4097, 7168]
|
||||
DTYPES = [torch.bfloat16, torch.float32]
|
||||
NUM_EXPERTS = 16
|
||||
|
||||
@@ -233,7 +231,7 @@ def test_nonlocal_and_padding_adjacent():
|
||||
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize("hidden_size", HIDDEN_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("dtype", [*DTYPES, torch.float16])
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_weighted_sum_matches_reference(
|
||||
num_tokens: int, top_k: int, hidden_size: int, dtype: torch.dtype
|
||||
@@ -268,7 +266,11 @@ def test_weighted_sum_matches_reference(
|
||||
|
||||
assert not out.isnan().any(), "invalid slots leaked into the reduction"
|
||||
ref = _reference(inputs.nan_to_num(), topk_weights, topk_ids, expert_map=None)
|
||||
atol, rtol = (1e-2, 1e-2) if dtype == torch.bfloat16 else (1e-4, 1e-4)
|
||||
atol, rtol = {
|
||||
torch.bfloat16: (1e-2, 1e-2),
|
||||
torch.float16: (1e-3, 1e-3),
|
||||
torch.float32: (1e-4, 1e-4),
|
||||
}[dtype]
|
||||
torch.testing.assert_close(out.float(), ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@@ -360,7 +362,7 @@ def test_num_valid_tokens_bounds_stale_padding(dtype: torch.dtype):
|
||||
"""
|
||||
set_random_seed(0)
|
||||
device = "cuda"
|
||||
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 512
|
||||
num_tokens, num_recv, top_k, hidden_size = 17, 10, 8, 7168
|
||||
|
||||
inputs = torch.randn(num_tokens, top_k, hidden_size, dtype=dtype, device=device)
|
||||
topk_weights = torch.rand(num_tokens, top_k, dtype=dtype, device=device)
|
||||
|
||||
@@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize(
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=None,
|
||||
),
|
||||
)
|
||||
return
|
||||
@@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routing_sink=None,
|
||||
)
|
||||
else:
|
||||
# Modular impl uses topk_weights and topk_ids
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -91,6 +91,12 @@ QWEN4_EXP_SM100_CASES = [
|
||||
for num_tokens, config in plans.items()
|
||||
]
|
||||
|
||||
QWEN4_EXP_SM121_CASES = [
|
||||
(n, k, num_tokens, config)
|
||||
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS.items()
|
||||
for num_tokens, config in plans.items()
|
||||
]
|
||||
|
||||
EXPECTED_CUTE_CONFIGS = {
|
||||
(3072, 7168, 1): (224, 3, 4, 8),
|
||||
(3072, 7168, 2): (128, 3, 2, 8),
|
||||
@@ -725,6 +731,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
|
||||
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
|
||||
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
|
||||
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
|
||||
((12, 1), qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS),
|
||||
((8, 0), {}),
|
||||
],
|
||||
)
|
||||
@@ -939,6 +946,29 @@ def test_qwen4_exp_sm100_selected_shapes(
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM121_CASES)
|
||||
def test_qwen4_exp_sm121_selected_shapes(
|
||||
n: int,
|
||||
k: int,
|
||||
num_tokens: int,
|
||||
config: SkinnyGemmConfig,
|
||||
) -> None:
|
||||
_require_capability_and_cute((12, 1))
|
||||
torch.manual_seed(42 + num_tokens)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
selected = qwen4_exp_gemm.QWEN4_EXP_SM121_GEMM_PLANS[(n, k)][num_tokens]
|
||||
assert selected == config
|
||||
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
|
||||
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
|
||||
_require_sm103_and_cute()
|
||||
spec = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||
|
||||
+225
-118
@@ -636,8 +636,8 @@ def test_top_k_per_row_decode_gfx950_k512_masked_graph_replay() -> None:
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_get_aiter_top_k_kernel,
|
||||
_launch_aiter_top_k_per_row_prefill,
|
||||
)
|
||||
|
||||
@@ -647,17 +647,14 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
top_k = 4
|
||||
indices = torch.empty((3, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
if not rocm_aiter_ops.is_indexer_top_k_supported(
|
||||
is_prefill=True,
|
||||
compress_ratio=4,
|
||||
num_rows=logits.shape[0],
|
||||
on_gfx950=True,
|
||||
)
|
||||
if aiter_topk_kernel is None:
|
||||
):
|
||||
pytest.skip("AITER top-k is unavailable")
|
||||
assert (
|
||||
_launch_aiter_top_k_per_row_prefill(
|
||||
aiter_topk_kernel,
|
||||
logits,
|
||||
row_starts,
|
||||
row_ends,
|
||||
@@ -683,10 +680,7 @@ def test_aiter_c4a_prefill_topk_returns_sequence_local_indices() -> None:
|
||||
def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
num_speculative_tokens: int,
|
||||
) -> None:
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_get_aiter_top_k_kernel,
|
||||
_launch_aiter_top_k_per_row_decode,
|
||||
)
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
query_tokens = num_speculative_tokens + 1
|
||||
offsets = torch.arange(query_tokens, dtype=torch.int32, device="cuda")
|
||||
@@ -701,20 +695,18 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
top_k = 4
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
|
||||
aiter_topk_kernel = _get_aiter_top_k_kernel(
|
||||
if not rocm_aiter_ops.is_indexer_top_k_supported(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=num_rows,
|
||||
max_valid_seq_len=seq_lens.max().item(),
|
||||
on_gfx950=True,
|
||||
)
|
||||
if aiter_topk_kernel is None:
|
||||
):
|
||||
pytest.skip("AITER top-k is unavailable")
|
||||
assert (
|
||||
_launch_aiter_top_k_per_row_decode(
|
||||
aiter_topk_kernel,
|
||||
rocm_aiter_ops.indexer_top_k_decode(
|
||||
logits,
|
||||
seq_lens,
|
||||
1,
|
||||
seq_lens.reshape(-1),
|
||||
indices,
|
||||
top_k,
|
||||
)
|
||||
@@ -729,130 +721,107 @@ def test_aiter_c4a_decode_topk_uses_exact_mtp_lengths(
|
||||
assert torch.all(indices[row_idx, num_valid:] == -1)
|
||||
|
||||
|
||||
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_aiter_decode_topk_matches_per_row_under_mtp() -> None:
|
||||
"""The aiter backend expands (B, 1) seq_lens into per-row ends before
|
||||
launching, so next_n > 1 must agree with the in-tree kernel, including
|
||||
the rows that clamp to a zero-length prefix.
|
||||
"""
|
||||
from vllm.model_executor.layers.indexer_topk import get_indexer_topk
|
||||
|
||||
def prefill_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
next_n = 4
|
||||
top_k = 512
|
||||
num_columns = 1024
|
||||
seq_lens = torch.tensor([[2], [513], [700]], dtype=torch.int32, device="cuda")
|
||||
num_rows = seq_lens.shape[0] * next_n
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
torch.manual_seed(0)
|
||||
logits = torch.randn(num_rows, num_columns, dtype=torch.float32, device="cuda")
|
||||
|
||||
outputs = {}
|
||||
for backend in ("aiter", "per_row"):
|
||||
indices = torch.empty((num_rows, top_k), dtype=torch.int32, device="cuda")
|
||||
get_indexer_topk(backend)(logits, seq_lens, next_n, indices, top_k, num_columns)
|
||||
outputs[backend] = indices
|
||||
|
||||
row_ends = (
|
||||
(seq_lens - next_n + 1 + torch.arange(next_n, device="cuda"))
|
||||
.clamp_(min=0)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
)
|
||||
assert row_ends[:next_n] == [0, 0, 1, 2]
|
||||
|
||||
for row_idx, row_end in enumerate(row_ends):
|
||||
num_valid = min(top_k, row_end)
|
||||
aiter_row = outputs["aiter"][row_idx]
|
||||
assert set(aiter_row[:num_valid].tolist()) == set(
|
||||
outputs["per_row"][row_idx][:num_valid].tolist()
|
||||
)
|
||||
assert torch.all(aiter_row[num_valid:] == -1)
|
||||
|
||||
|
||||
def _enable_aiter_topk(monkeypatch, enabled: bool = True) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla_sparse,
|
||||
"_get_aiter_topk_ops",
|
||||
lambda: (prefill_kernel, decode_kernel),
|
||||
rocm_aiter_ops, "is_indexer_top_k_enabled", lambda: enabled, raising=True
|
||||
)
|
||||
get_kernel = rocm_aiter_mla_sparse._get_aiter_top_k_kernel
|
||||
|
||||
|
||||
def test_aiter_c4a_topk_kernel_selection(monkeypatch) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_enable_aiter_topk(monkeypatch)
|
||||
is_supported = rocm_aiter_ops.is_indexer_top_k_supported
|
||||
|
||||
eligible_cases = [
|
||||
(
|
||||
dict(is_prefill=True, compress_ratio=4, num_rows=1, on_gfx950=True),
|
||||
prefill_kernel,
|
||||
dict(is_prefill=True, compress_ratio=4, num_rows=1),
|
||||
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_536),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=257,
|
||||
max_valid_seq_len=250_000,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
max_valid_seq_len=65_536,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=257,
|
||||
max_valid_seq_len=250_000,
|
||||
num_columns=524_288,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=385,
|
||||
max_valid_seq_len=250_000,
|
||||
num_columns=524_288,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
(
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=384,
|
||||
max_valid_seq_len=50_000,
|
||||
num_columns=1_048_577,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
decode_kernel,
|
||||
),
|
||||
]
|
||||
for kwargs, expected_kernel in eligible_cases:
|
||||
assert get_kernel(**kwargs) is expected_kernel
|
||||
|
||||
def fail_if_imported() -> None:
|
||||
pytest.fail("ineligible shapes must not import AITER top-k")
|
||||
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla_sparse,
|
||||
"_get_aiter_topk_ops",
|
||||
fail_if_imported,
|
||||
)
|
||||
ineligible_cases = [
|
||||
dict(is_prefill=True, compress_ratio=1, num_rows=1, on_gfx950=True),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=2,
|
||||
num_rows=384,
|
||||
max_valid_seq_len=1_048_576,
|
||||
num_columns=1_048_576,
|
||||
topk_tokens=512,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
max_valid_seq_len=65_537,
|
||||
on_gfx950=True,
|
||||
),
|
||||
]
|
||||
for kwargs in eligible_cases:
|
||||
assert is_supported(**kwargs) is True
|
||||
|
||||
ineligible_cases = [
|
||||
dict(is_prefill=True, compress_ratio=1, num_rows=1),
|
||||
dict(is_prefill=False, compress_ratio=4, num_rows=1, max_valid_seq_len=65_537),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=256,
|
||||
max_valid_seq_len=125_000,
|
||||
on_gfx950=True,
|
||||
),
|
||||
dict(
|
||||
is_prefill=False,
|
||||
compress_ratio=4,
|
||||
num_rows=320,
|
||||
max_valid_seq_len=2_500,
|
||||
on_gfx950=False,
|
||||
),
|
||||
]
|
||||
for kwargs in ineligible_cases:
|
||||
assert get_kernel(**kwargs) is None
|
||||
assert is_supported(**kwargs) is False
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_mla_sparse, "_get_aiter_topk_ops", lambda: None)
|
||||
assert (
|
||||
get_kernel(
|
||||
is_prefill=True,
|
||||
compress_ratio=4,
|
||||
num_rows=1,
|
||||
on_gfx950=True,
|
||||
)
|
||||
is None
|
||||
)
|
||||
_enable_aiter_topk(monkeypatch, enabled=False)
|
||||
assert is_supported(is_prefill=True, compress_ratio=4, num_rows=1) is False
|
||||
|
||||
|
||||
def test_dsv4_native_top_k_window() -> None:
|
||||
"""The in-tree decode kernel's measured advantage band over AITER, which
|
||||
applies to the DSV4 indexer only."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
prefers_native = rocm_aiter_ops.dsv4_indexer_prefers_native_top_k
|
||||
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=512) is True
|
||||
assert prefers_native(num_rows=385, num_columns=524_288, topk_tokens=512) is False
|
||||
assert prefers_native(num_rows=128, num_columns=1_048_577, topk_tokens=512) is False
|
||||
assert prefers_native(num_rows=128, num_columns=524_288, topk_tokens=1024) is False
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="This test requires CUDA")
|
||||
@@ -1927,3 +1896,141 @@ def test_sparse_indexer_topk_backend_resolution() -> None:
|
||||
if not is_sm100:
|
||||
with pytest.raises(RuntimeError, match="SM100a/SM103a"):
|
||||
resolve("deep_select")
|
||||
|
||||
|
||||
def _patch_aiter_topk(monkeypatch, decode_kernel, enabled: bool = True) -> None:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_enable_aiter_topk(monkeypatch, enabled=enabled)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: enabled, raising=True)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "indexer_top_k_decode", decode_kernel)
|
||||
|
||||
|
||||
def _make_topk(backend: str):
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.indexer_topk import SparseIndexerTopk
|
||||
|
||||
cfg = VllmConfig(kernel_config={"sparse_indexer_topk_backend": backend})
|
||||
with set_current_vllm_config(cfg):
|
||||
return SparseIndexerTopk()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
|
||||
def test_sparse_indexer_topk_aiter_requires_rocm() -> None:
|
||||
"""Explicitly requesting the ROCm-only backend must fail fast elsewhere."""
|
||||
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(RuntimeError, match="ROCm"):
|
||||
_make_topk("aiter").resolve_backend(logits, 2048, 8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
def test_sparse_indexer_topk_backend_resolution_aiter(monkeypatch) -> None:
|
||||
"""Aiter is opt-in only: "auto" never selects it (the shape-dependent perf
|
||||
gate lives at the call site), and an explicit request validates hard
|
||||
capability only."""
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pass
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel)
|
||||
logits = torch.randn(8, 4096, dtype=torch.float32, device="cuda")
|
||||
|
||||
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
|
||||
assert _make_topk("aiter").resolve_backend(logits, 2048, 8) == "aiter"
|
||||
# Shapes the call-site perf gate would decline still resolve.
|
||||
assert _make_topk("aiter").resolve_backend(logits, 512, 8) == "aiter"
|
||||
with pytest.raises(RuntimeError, match="topk_tokens must be in"):
|
||||
_make_topk("aiter").resolve_backend(logits, 3000, 8)
|
||||
|
||||
_enable_aiter_topk(monkeypatch, enabled=False)
|
||||
with pytest.raises(RuntimeError, match="no tuned indexer top-k kernels"):
|
||||
_make_topk("aiter").resolve_backend(logits, 2048, 8)
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel, enabled=False)
|
||||
assert _make_topk("auto").resolve_backend(logits, 2048, 8) == "per_row"
|
||||
with pytest.raises(RuntimeError, match="VLLM_ROCM_USE_AITER"):
|
||||
_make_topk("aiter").resolve_backend(logits, 2048, 8)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_aiter_applies_no_shape_gate(monkeypatch) -> None:
|
||||
"""Shape-dependent perf gating belongs to the call site: once "aiter" is
|
||||
requested the dispatcher must run it for every supported k, including the
|
||||
k=512 band the in-tree kernel wins on DSV4's compressed-KV logits."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
calls: list[tuple] = []
|
||||
_patch_aiter_topk(monkeypatch, lambda *args: calls.append(args))
|
||||
monkeypatch.setattr(
|
||||
indexer_topk.ops,
|
||||
"top_k_per_row_decode",
|
||||
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
|
||||
)
|
||||
|
||||
num_rows = 8
|
||||
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((num_rows, 512), dtype=torch.int32, device="cuda")
|
||||
op = _make_topk("aiter")
|
||||
|
||||
op(logits, seq_lens, 1, indices, 512, 4096)
|
||||
# Past the compressed-context boundary the call-site gate declines on.
|
||||
op(logits, seq_lens, 1, indices, 512, 4 * (64 * 1024 + 1))
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_auto_stays_on_per_row(monkeypatch) -> None:
|
||||
"""The "auto" chain must not reach AITER on its own: the decode top-k runs
|
||||
through top_k_per_row_decode unless the caller asks for the aiter backend."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
def decode_kernel(*args, **kwargs) -> None:
|
||||
pytest.fail("AITER is opt-in only")
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel)
|
||||
calls: list[tuple] = []
|
||||
monkeypatch.setattr(
|
||||
indexer_topk.ops, "top_k_per_row_decode", lambda *args: calls.append(args)
|
||||
)
|
||||
|
||||
num_rows = 8
|
||||
logits = torch.randn(num_rows, 4096, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.full((num_rows, 1), 4096, dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((num_rows, 1024), dtype=torch.int32, device="cuda")
|
||||
|
||||
_make_topk("auto")(logits, seq_lens, 1, indices, 1024, 4096)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="requires ROCm")
|
||||
@torch.inference_mode()
|
||||
def test_sparse_indexer_topk_aiter_expands_mtp_row_ends(monkeypatch) -> None:
|
||||
"""AITER's decode kernel only implements the next_n == 1 form, so spec
|
||||
decode batches must be handed the equivalent expanded per-row ends."""
|
||||
from vllm.model_executor.layers import indexer_topk
|
||||
|
||||
recorded: dict = {}
|
||||
|
||||
def decode_kernel(logits, next_n, seq_lens, *args, **kwargs) -> None:
|
||||
recorded["next_n"] = next_n
|
||||
recorded["row_ends"] = seq_lens.tolist()
|
||||
|
||||
_patch_aiter_topk(monkeypatch, decode_kernel)
|
||||
monkeypatch.setattr(
|
||||
indexer_topk.ops,
|
||||
"top_k_per_row_decode",
|
||||
lambda *args: pytest.fail("an explicit aiter request must not fall back"),
|
||||
)
|
||||
|
||||
next_n = 2
|
||||
logits = torch.randn(4, 4096, dtype=torch.float32, device="cuda")
|
||||
seq_lens = torch.tensor([[100], [200]], dtype=torch.int32, device="cuda")
|
||||
indices = torch.empty((4, 1024), dtype=torch.int32, device="cuda")
|
||||
|
||||
_make_topk("aiter")(logits, seq_lens, next_n, indices, 1024, 800)
|
||||
|
||||
assert recorded["next_n"] == 1
|
||||
assert recorded["row_ends"] == [99, 100, 199, 200]
|
||||
|
||||
Reference in New Issue
Block a user