[ROCm][CI][AITER Coverage] Harden MoE sorting-backend/dispatch env-var test matrix (#58393)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>
This commit is contained in:
Divakar Verma
2026-09-24 21:14:02 -05:00
committed by GitHub
parent 32fc82343f
commit 65c5e24689
4 changed files with 355 additions and 12 deletions
@@ -2,20 +2,24 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import inspect
import textwrap
import traceback
from itertools import product
from typing import Any
from unittest import mock
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
from vllm.utils.import_utils import has_deep_ep, has_deep_gemm
from vllm.utils.import_utils import has_aiter, has_deep_ep, has_deep_gemm
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
@@ -72,7 +76,7 @@ def assert_aiter_quant_scheme_case(config: Config) -> None:
indirectly through the general quant-config sweep.
See https://github.com/vllm-project/vllm/issues/54966."""
fe_cls = config.fused_experts_type
if getattr(fe_cls, "__name__", "") != "AiterExperts":
if fe_cls is not AiterExperts:
return
if config.quant_config is None:
@@ -93,7 +97,7 @@ def assert_aiter_activation_case(config: Config) -> None:
AiterExperts being reached only indirectly through the general
activation sweep. See https://github.com/vllm-project/vllm/issues/54966."""
fe_cls = config.fused_experts_type
if getattr(fe_cls, "__name__", "") != "AiterExperts":
if fe_cls is not AiterExperts:
return
assert fe_cls._supports_activation(config.activation), (
@@ -148,7 +152,7 @@ def rank_worker(
if (
topk == 1
and config.supports_apply_weight_on_input()
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
and config.fused_experts_type is AiterExperts
and config.quant_block_shape is not None
):
print(
@@ -164,7 +168,7 @@ def rank_worker(
# https://github.com/vllm-project/vllm/issues/57029
if (
config.world_size > 1
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
and config.fused_experts_type is AiterExperts
and getattr(config.prepare_finalize_type, "__name__", "")
in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize")
):
@@ -202,7 +206,7 @@ def rank_worker(
is_aiter_fp8 = (
_cp.is_rocm()
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
and config.fused_experts_type is AiterExperts
and config.quant_config is not None
)
if is_aiter_fp8:
@@ -262,7 +266,7 @@ def is_nyi_config(config: Config) -> bool:
return True
if config.activation != MoEActivation.SILU:
if getattr(config.fused_experts_type, "__name__", "") != "AiterExperts":
if config.fused_experts_type is not AiterExperts:
return True # AITER-only for this axis, for now
if config.quant_dtype is not None:
return True # unquantized-only for this axis, for now
@@ -425,6 +429,268 @@ def test_modular_kernel_combinations_singlegpu(
run(config, verbosity > 0)
# AITER sorting-backend dispatch env-var matrix (issue #54966 step 3) -------
#
# AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING are aiter globals
# read once at import time, so each combo needs a fresh child process (see
# parallel_launch_with_config) rather than in-process monkeypatching.
#
# AITER_MOE_SORT_BACKEND is not covered: it only matters when output_aux=True,
# which vLLM only sets on the MXFP4 path -- not yet wired into AiterExperts.
def _aiter_sort_backend_hooks_available() -> bool:
"""Whether the installed aiter build exposes the sort-backend hooks that
AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING select between.
"""
try:
import aiter.fused_moe as aiter_fused_moe
except ImportError:
return False
required = (
"_moe_sorting_impl",
"_flydsl_moe_sorting",
"_USE_CK_MOE_SORTING",
"_USE_FLYDSL_MOE_SORTING",
)
return all(hasattr(aiter_fused_moe, name) for name in required)
# Deliberately does NOT call _aiter_sort_backend_hooks_available() here: this
# marker is evaluated at module-import time, which also happens inside each
# spawned child (to unpickle its worker) -- before that case's env vars are
# applied, permanently freezing aiter's sort-backend globals. The hooks check
# instead runs inside the test body below (parent process only).
require_aiter_moe = pytest.mark.skipif(
not (
current_platform.is_rocm()
and has_aiter()
and rocm_aiter_ops.is_fused_moe_enabled()
),
reason=(
"AITER MoE sorting-backend dispatch needs ROCm + AITER, with "
"VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1 set before "
"the test process starts."
),
)
# (AITER_USE_CK_MOE_SORTING, AITER_USE_FLYDSL_MOE_SORTING, expected_backend)
AITER_SORTING_BACKEND_ENV_CASES = [
(1, 0, "ck"),
(1, 1, "ck"), # CK wins over FlyDSL even when both are requested.
(0, 1, "flydsl"),
(0, 0, "opus"), # default when neither flag is set.
]
def _aiter_sorting_backend_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
config: Config,
weights: WeightTensors,
verbose: bool,
ck: int,
flydsl: int,
expected_backend: str,
):
# Imported lazily in the spawned child, after env vars are applied, so
# aiter's import-time globals pick up this case's env.
import aiter.fused_moe as aiter_fused_moe
# Assert the env vars actually stuck in aiter's import-time globals,
# rather than inferring it indirectly from which sorting function fires.
assert bool(ck) == aiter_fused_moe._USE_CK_MOE_SORTING, (
f"aiter.fused_moe._USE_CK_MOE_SORTING={aiter_fused_moe._USE_CK_MOE_SORTING} "
f"but AITER_USE_CK_MOE_SORTING={ck} was set before this process started -- "
"the env var never reached aiter's import-time globals."
)
assert bool(flydsl) == aiter_fused_moe._USE_FLYDSL_MOE_SORTING, (
"aiter.fused_moe._USE_FLYDSL_MOE_SORTING="
f"{aiter_fused_moe._USE_FLYDSL_MOE_SORTING} but AITER_USE_FLYDSL_MOE_SORTING="
f"{flydsl} was set before this process started -- the env var never "
"reached aiter's import-time globals."
)
# Captured before patching, since the patched attribute would resolve to
# the mock's own (*args, **kwargs) signature instead of the real one.
original_moe_sorting_impl = aiter_fused_moe._moe_sorting_impl
with (
mock.patch.object(
aiter_fused_moe,
"_moe_sorting_impl",
wraps=original_moe_sorting_impl,
) as sorting_impl_mock,
mock.patch.object(
aiter_fused_moe,
"_flydsl_moe_sorting",
wraps=aiter_fused_moe._flydsl_moe_sorting,
) as flydsl_mock,
):
# Reuses the real correctness/accuracy checking rank_worker already
# does for every other AiterExperts combo, instead of duplicating it.
rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose)
if expected_backend == "flydsl":
assert flydsl_mock.call_count > 0, "Expected FlyDSL sorting to fire."
assert sorting_impl_mock.call_count == 0, (
"FlyDSL was requested and eligible, but the opus/CK sorting "
"path fired instead -- a silent fallback."
)
else:
assert flydsl_mock.call_count == 0, (
f"Expected the '{expected_backend}' sorting path, but FlyDSL "
"fired instead."
)
assert sorting_impl_mock.call_count > 0, (
f"Expected the '{expected_backend}' sorting path to fire."
)
# Bind by signature so this works regardless of whether use_opus
# is passed positionally or by keyword.
call = sorting_impl_mock.call_args
bound = inspect.signature(original_moe_sorting_impl).bind(
*call.args, **call.kwargs
)
use_opus = bound.arguments["use_opus"]
assert use_opus == (expected_backend == "opus"), (
f"Expected use_opus={expected_backend == 'opus'} for the "
f"'{expected_backend}' sorting path, got use_opus={use_opus}."
)
@require_aiter_moe
@pytest.mark.parametrize("ck,flydsl,expected_backend", AITER_SORTING_BACKEND_ENV_CASES)
def test_aiter_moe_sorting_backend_dispatch_env_matrix(
ck: int, flydsl: int, expected_backend: str
):
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER
backend and dispatch settings")."""
if not _aiter_sort_backend_hooks_available():
pytest.skip(
"Installed aiter build predates the sort-backend hooks "
"(_moe_sorting_impl, _flydsl_moe_sorting, _USE_CK_MOE_SORTING, "
"_USE_FLYDSL_MOE_SORTING)."
)
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
config = Config(
Ms=[32],
K=2048,
N=1024,
E=32,
topks=[4],
dtype=torch.bfloat16,
quant_config=None,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
world_size=1,
)
assert config.is_valid()[0]
weights = WeightTensors.make(config)
vllm_config, env_dict = config.make_env_data()
env_dict = {
**env_dict,
"AITER_USE_CK_MOE_SORTING": str(ck),
"AITER_USE_FLYDSL_MOE_SORTING": str(flydsl),
}
parallel_launch_with_config(
config.world_size,
_aiter_sorting_backend_worker,
vllm_config,
env_dict,
config,
weights,
False,
ck,
flydsl,
expected_backend,
)
def _aiter_dispatch_policy_worker(
pgi: ProcessGroupInfo,
vllm_config: VllmConfig,
cpu_group,
config: Config,
weights: WeightTensors,
verbose: bool,
dispatch_policy: int,
):
# Drives the real AiterExperts.apply() -> rocm_aiter_fused_experts() call
# through the modular kernel, so it also catches a regression in apply()'s
# own forwarding (unlike test_rocm_aiter_moe.py's version, which supplies
# the policy value directly to rocm_aiter_fused_experts()).
with mock.patch.object(
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
) as fused_moe_mock:
rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose)
assert fused_moe_mock.call_count > 0, (
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
)
for call in fused_moe_mock.call_args_list:
forwarded = call.kwargs["moe_sorting_dispatch_policy"]
assert forwarded == dispatch_policy, (
f"AiterExperts.apply() forwarded moe_sorting_dispatch_policy="
f"{forwarded}, but VLLM_ROCM_AITER_MOE_DISPATCH_POLICY="
f"{dispatch_policy} was set before this process started."
)
@require_aiter_moe
@pytest.mark.parametrize("dispatch_policy", [0, 1, 2])
def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int):
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER
backend and dispatch settings").
Complements test_aiter_moe_dispatch_policy_forwarded_to_fused_moe in
test_rocm_aiter_moe.py by driving the real AiterExperts.apply() call
instead of supplying moe_sorting_dispatch_policy directly.
"""
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
MoEPrepareAndFinalizeNoDPEPModular,
)
config = Config(
Ms=[32],
K=2048,
N=1024,
E=32,
topks=[4],
dtype=torch.bfloat16,
quant_config=None,
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
fused_experts_type=AiterExperts,
world_size=1,
)
assert config.is_valid()[0]
weights = WeightTensors.make(config)
vllm_config, env_dict = config.make_env_data()
env_dict = {
**env_dict,
"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": str(dispatch_policy),
}
parallel_launch_with_config(
config.world_size,
_aiter_dispatch_policy_worker,
vllm_config,
env_dict,
config,
weights,
False,
dispatch_policy,
)
if __name__ == "__main__":
# Ability to test individual PrepareAndFinalize and FusedExperts combination
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
+71
View File
@@ -22,6 +22,7 @@ import importlib
import math
import warnings
from typing import Any, NamedTuple
from unittest import mock
import pytest
import torch
@@ -645,6 +646,76 @@ def test_aiter_moe_padding_env_var(
assert envs.VLLM_ROCM_MOE_PADDING is moe_padding
# Dispatch-policy forwarding test ------------------------------------------
@pytest.mark.parametrize("dispatch_policy", [0, 1, 2])
def test_aiter_moe_dispatch_policy_forwarded_to_fused_moe(
dispatch_policy: int,
monkeypatch: pytest.MonkeyPatch,
):
"""VLLM_ROCM_AITER_MOE_DISPATCH_POLICY should reach rocm_aiter_ops.fused_moe
unchanged, the same forwarding AiterExperts.apply does via
rocm_aiter_fused_experts, for every documented policy value (0=auto,
1=always single-pass, 2=always multi-pass). See vllm-project/vllm#54966.
"""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
rocm_aiter_fused_experts,
)
_assert_aiter_supported()
case = _make_moe_case(
num_tokens=8,
hidden_dim=256,
intermediate_dim=512,
num_experts=4,
topk=2,
seed=5,
)
w1_shuffled, w2_shuffled = _shuffle_moe_weights(case["w1"], case["w2"])
moe_config = make_dummy_moe_config(
num_experts=4,
experts_per_token=2,
hidden_dim=256,
intermediate_size=512,
in_dtype=torch.bfloat16,
)
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", str(dispatch_policy))
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert rocm_aiter_ops.get_moe_dispatch_policy() == dispatch_policy, (
"rocm_aiter_ops cached a stale dispatch policy after the env var changed."
)
with mock.patch.object(
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
) as fused_moe_mock:
rocm_aiter_fused_experts(
hidden_states=case["hidden_states"],
w1=w1_shuffled,
w2=w2_shuffled,
topk_weights=case["topk_weights"],
topk_ids=case["topk_ids"],
moe_config=moe_config,
activation=MoEActivation.SILU,
moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(),
)
assert (
fused_moe_mock.call_args.kwargs["moe_sorting_dispatch_policy"]
== dispatch_policy
), (
"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY was not forwarded to "
"rocm_aiter_ops.fused_moe."
)
# Enum tests --------------------------------------------------------------
+1
View File
@@ -2019,6 +2019,7 @@ class rocm_aiter_ops:
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
_sync_aiter_situv2_moe_env()
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
@staticmethod
def get_aiter_activation_type(activation_str: str) -> "ActivationType | None":
@@ -7,11 +7,6 @@ import json
import os
import torch
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
flydsl_a16w4_gemm2,
)
from vllm.logger import init_logger
from vllm.utils.platform_utils import get_device_name_as_file_name
@@ -81,6 +76,10 @@ def moe_sorting(
model_dim: int,
block_m: int,
):
# Imported lazily so aiter's import-time sort-backend env vars are read
# when this is called, not whenever this module happens to be imported.
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
topk_ids_i32 = topk_ids.to(torch.int32)
topk_w_f32 = topk_weights.to(torch.float32)
sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = (
@@ -185,6 +184,12 @@ def fused_flydsl_moe_impl(
tile_n2: int | None = None,
tile_k2: int | None = None,
) -> torch.Tensor:
# Imported lazily -- see the comment in moe_sorting() above.
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
flydsl_a16w4_gemm2,
)
device = hidden_states.device
tokens = hidden_states.shape[0]
model_dim = hidden_states.shape[1]