diff --git a/tests/kernels/moe/test_modular_kernel_combinations.py b/tests/kernels/moe/test_modular_kernel_combinations.py index 9bfd3b5a2e..253abc5c38 100644 --- a/tests/kernels/moe/test_modular_kernel_combinations.py +++ b/tests/kernels/moe/test_modular_kernel_combinations.py @@ -2,20 +2,24 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import copy +import inspect import textwrap import traceback from itertools import product from typing import Any +from unittest import mock import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe -from vllm.utils.import_utils import has_deep_ep, has_deep_gemm +from vllm.utils.import_utils import has_aiter, has_deep_ep, has_deep_gemm from vllm.utils.torch_utils import set_random_seed from vllm.v1.worker.workspace import init_workspace_manager @@ -72,7 +76,7 @@ def assert_aiter_quant_scheme_case(config: Config) -> None: indirectly through the general quant-config sweep. See https://github.com/vllm-project/vllm/issues/54966.""" fe_cls = config.fused_experts_type - if getattr(fe_cls, "__name__", "") != "AiterExperts": + if fe_cls is not AiterExperts: return if config.quant_config is None: @@ -93,7 +97,7 @@ def assert_aiter_activation_case(config: Config) -> None: AiterExperts being reached only indirectly through the general activation sweep. See https://github.com/vllm-project/vllm/issues/54966.""" fe_cls = config.fused_experts_type - if getattr(fe_cls, "__name__", "") != "AiterExperts": + if fe_cls is not AiterExperts: return assert fe_cls._supports_activation(config.activation), ( @@ -148,7 +152,7 @@ def rank_worker( if ( topk == 1 and config.supports_apply_weight_on_input() - and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and config.fused_experts_type is AiterExperts and config.quant_block_shape is not None ): print( @@ -164,7 +168,7 @@ def rank_worker( # https://github.com/vllm-project/vllm/issues/57029 if ( config.world_size > 1 - and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and config.fused_experts_type is AiterExperts and getattr(config.prepare_finalize_type, "__name__", "") in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize") ): @@ -202,7 +206,7 @@ def rank_worker( is_aiter_fp8 = ( _cp.is_rocm() - and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts" + and config.fused_experts_type is AiterExperts and config.quant_config is not None ) if is_aiter_fp8: @@ -262,7 +266,7 @@ def is_nyi_config(config: Config) -> bool: return True if config.activation != MoEActivation.SILU: - if getattr(config.fused_experts_type, "__name__", "") != "AiterExperts": + if config.fused_experts_type is not AiterExperts: return True # AITER-only for this axis, for now if config.quant_dtype is not None: return True # unquantized-only for this axis, for now @@ -425,6 +429,268 @@ def test_modular_kernel_combinations_singlegpu( run(config, verbosity > 0) +# AITER sorting-backend dispatch env-var matrix (issue #54966 step 3) ------- +# +# AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING are aiter globals +# read once at import time, so each combo needs a fresh child process (see +# parallel_launch_with_config) rather than in-process monkeypatching. +# +# AITER_MOE_SORT_BACKEND is not covered: it only matters when output_aux=True, +# which vLLM only sets on the MXFP4 path -- not yet wired into AiterExperts. + + +def _aiter_sort_backend_hooks_available() -> bool: + """Whether the installed aiter build exposes the sort-backend hooks that + AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING select between. + """ + try: + import aiter.fused_moe as aiter_fused_moe + except ImportError: + return False + + required = ( + "_moe_sorting_impl", + "_flydsl_moe_sorting", + "_USE_CK_MOE_SORTING", + "_USE_FLYDSL_MOE_SORTING", + ) + return all(hasattr(aiter_fused_moe, name) for name in required) + + +# Deliberately does NOT call _aiter_sort_backend_hooks_available() here: this +# marker is evaluated at module-import time, which also happens inside each +# spawned child (to unpickle its worker) -- before that case's env vars are +# applied, permanently freezing aiter's sort-backend globals. The hooks check +# instead runs inside the test body below (parent process only). +require_aiter_moe = pytest.mark.skipif( + not ( + current_platform.is_rocm() + and has_aiter() + and rocm_aiter_ops.is_fused_moe_enabled() + ), + reason=( + "AITER MoE sorting-backend dispatch needs ROCm + AITER, with " + "VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1 set before " + "the test process starts." + ), +) + +# (AITER_USE_CK_MOE_SORTING, AITER_USE_FLYDSL_MOE_SORTING, expected_backend) +AITER_SORTING_BACKEND_ENV_CASES = [ + (1, 0, "ck"), + (1, 1, "ck"), # CK wins over FlyDSL even when both are requested. + (0, 1, "flydsl"), + (0, 0, "opus"), # default when neither flag is set. +] + + +def _aiter_sorting_backend_worker( + pgi: ProcessGroupInfo, + vllm_config: VllmConfig, + cpu_group, + config: Config, + weights: WeightTensors, + verbose: bool, + ck: int, + flydsl: int, + expected_backend: str, +): + # Imported lazily in the spawned child, after env vars are applied, so + # aiter's import-time globals pick up this case's env. + import aiter.fused_moe as aiter_fused_moe + + # Assert the env vars actually stuck in aiter's import-time globals, + # rather than inferring it indirectly from which sorting function fires. + assert bool(ck) == aiter_fused_moe._USE_CK_MOE_SORTING, ( + f"aiter.fused_moe._USE_CK_MOE_SORTING={aiter_fused_moe._USE_CK_MOE_SORTING} " + f"but AITER_USE_CK_MOE_SORTING={ck} was set before this process started -- " + "the env var never reached aiter's import-time globals." + ) + assert bool(flydsl) == aiter_fused_moe._USE_FLYDSL_MOE_SORTING, ( + "aiter.fused_moe._USE_FLYDSL_MOE_SORTING=" + f"{aiter_fused_moe._USE_FLYDSL_MOE_SORTING} but AITER_USE_FLYDSL_MOE_SORTING=" + f"{flydsl} was set before this process started -- the env var never " + "reached aiter's import-time globals." + ) + + # Captured before patching, since the patched attribute would resolve to + # the mock's own (*args, **kwargs) signature instead of the real one. + original_moe_sorting_impl = aiter_fused_moe._moe_sorting_impl + + with ( + mock.patch.object( + aiter_fused_moe, + "_moe_sorting_impl", + wraps=original_moe_sorting_impl, + ) as sorting_impl_mock, + mock.patch.object( + aiter_fused_moe, + "_flydsl_moe_sorting", + wraps=aiter_fused_moe._flydsl_moe_sorting, + ) as flydsl_mock, + ): + # Reuses the real correctness/accuracy checking rank_worker already + # does for every other AiterExperts combo, instead of duplicating it. + rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose) + + if expected_backend == "flydsl": + assert flydsl_mock.call_count > 0, "Expected FlyDSL sorting to fire." + assert sorting_impl_mock.call_count == 0, ( + "FlyDSL was requested and eligible, but the opus/CK sorting " + "path fired instead -- a silent fallback." + ) + else: + assert flydsl_mock.call_count == 0, ( + f"Expected the '{expected_backend}' sorting path, but FlyDSL " + "fired instead." + ) + assert sorting_impl_mock.call_count > 0, ( + f"Expected the '{expected_backend}' sorting path to fire." + ) + # Bind by signature so this works regardless of whether use_opus + # is passed positionally or by keyword. + call = sorting_impl_mock.call_args + bound = inspect.signature(original_moe_sorting_impl).bind( + *call.args, **call.kwargs + ) + use_opus = bound.arguments["use_opus"] + assert use_opus == (expected_backend == "opus"), ( + f"Expected use_opus={expected_backend == 'opus'} for the " + f"'{expected_backend}' sorting path, got use_opus={use_opus}." + ) + + +@require_aiter_moe +@pytest.mark.parametrize("ck,flydsl,expected_backend", AITER_SORTING_BACKEND_ENV_CASES) +def test_aiter_moe_sorting_backend_dispatch_env_matrix( + ck: int, flydsl: int, expected_backend: str +): + """See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER + backend and dispatch settings").""" + if not _aiter_sort_backend_hooks_available(): + pytest.skip( + "Installed aiter build predates the sort-backend hooks " + "(_moe_sorting_impl, _flydsl_moe_sorting, _USE_CK_MOE_SORTING, " + "_USE_FLYDSL_MOE_SORTING)." + ) + + from vllm.model_executor.layers.fused_moe.prepare_finalize import ( + MoEPrepareAndFinalizeNoDPEPModular, + ) + + config = Config( + Ms=[32], + K=2048, + N=1024, + E=32, + topks=[4], + dtype=torch.bfloat16, + quant_config=None, + prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular, + fused_experts_type=AiterExperts, + world_size=1, + ) + assert config.is_valid()[0] + + weights = WeightTensors.make(config) + vllm_config, env_dict = config.make_env_data() + env_dict = { + **env_dict, + "AITER_USE_CK_MOE_SORTING": str(ck), + "AITER_USE_FLYDSL_MOE_SORTING": str(flydsl), + } + + parallel_launch_with_config( + config.world_size, + _aiter_sorting_backend_worker, + vllm_config, + env_dict, + config, + weights, + False, + ck, + flydsl, + expected_backend, + ) + + +def _aiter_dispatch_policy_worker( + pgi: ProcessGroupInfo, + vllm_config: VllmConfig, + cpu_group, + config: Config, + weights: WeightTensors, + verbose: bool, + dispatch_policy: int, +): + # Drives the real AiterExperts.apply() -> rocm_aiter_fused_experts() call + # through the modular kernel, so it also catches a regression in apply()'s + # own forwarding (unlike test_rocm_aiter_moe.py's version, which supplies + # the policy value directly to rocm_aiter_fused_experts()). + with mock.patch.object( + rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe + ) as fused_moe_mock: + rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose) + + assert fused_moe_mock.call_count > 0, ( + "Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe." + ) + for call in fused_moe_mock.call_args_list: + forwarded = call.kwargs["moe_sorting_dispatch_policy"] + assert forwarded == dispatch_policy, ( + f"AiterExperts.apply() forwarded moe_sorting_dispatch_policy=" + f"{forwarded}, but VLLM_ROCM_AITER_MOE_DISPATCH_POLICY=" + f"{dispatch_policy} was set before this process started." + ) + + +@require_aiter_moe +@pytest.mark.parametrize("dispatch_policy", [0, 1, 2]) +def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int): + """See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER + backend and dispatch settings"). + + Complements test_aiter_moe_dispatch_policy_forwarded_to_fused_moe in + test_rocm_aiter_moe.py by driving the real AiterExperts.apply() call + instead of supplying moe_sorting_dispatch_policy directly. + """ + from vllm.model_executor.layers.fused_moe.prepare_finalize import ( + MoEPrepareAndFinalizeNoDPEPModular, + ) + + config = Config( + Ms=[32], + K=2048, + N=1024, + E=32, + topks=[4], + dtype=torch.bfloat16, + quant_config=None, + prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular, + fused_experts_type=AiterExperts, + world_size=1, + ) + assert config.is_valid()[0] + + weights = WeightTensors.make(config) + vllm_config, env_dict = config.make_env_data() + env_dict = { + **env_dict, + "VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": str(dispatch_policy), + } + + parallel_launch_with_config( + config.world_size, + _aiter_dispatch_policy_worker, + vllm_config, + env_dict, + config, + weights, + False, + dispatch_policy, + ) + + if __name__ == "__main__": # Ability to test individual PrepareAndFinalize and FusedExperts combination from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser diff --git a/tests/kernels/moe/test_rocm_aiter_moe.py b/tests/kernels/moe/test_rocm_aiter_moe.py index 0967f9499e..8fa1706f18 100644 --- a/tests/kernels/moe/test_rocm_aiter_moe.py +++ b/tests/kernels/moe/test_rocm_aiter_moe.py @@ -22,6 +22,7 @@ import importlib import math import warnings from typing import Any, NamedTuple +from unittest import mock import pytest import torch @@ -645,6 +646,76 @@ def test_aiter_moe_padding_env_var( assert envs.VLLM_ROCM_MOE_PADDING is moe_padding +# Dispatch-policy forwarding test ------------------------------------------ + + +@pytest.mark.parametrize("dispatch_policy", [0, 1, 2]) +def test_aiter_moe_dispatch_policy_forwarded_to_fused_moe( + dispatch_policy: int, + monkeypatch: pytest.MonkeyPatch, +): + """VLLM_ROCM_AITER_MOE_DISPATCH_POLICY should reach rocm_aiter_ops.fused_moe + unchanged, the same forwarding AiterExperts.apply does via + rocm_aiter_fused_experts, for every documented policy value (0=auto, + 1=always single-pass, 2=always multi-pass). See vllm-project/vllm#54966. + """ + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm._aiter_ops import rocm_aiter_ops + from vllm.model_executor.layers.fused_moe.activation import MoEActivation + from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import ( + rocm_aiter_fused_experts, + ) + + _assert_aiter_supported() + case = _make_moe_case( + num_tokens=8, + hidden_dim=256, + intermediate_dim=512, + num_experts=4, + topk=2, + seed=5, + ) + w1_shuffled, w2_shuffled = _shuffle_moe_weights(case["w1"], case["w2"]) + moe_config = make_dummy_moe_config( + num_experts=4, + experts_per_token=2, + hidden_dim=256, + intermediate_size=512, + in_dtype=torch.bfloat16, + ) + + with monkeypatch.context() as mp: + mp.setenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", str(dispatch_policy)) + _reload_envs() + rocm_aiter_ops.refresh_env_variables() + + assert rocm_aiter_ops.get_moe_dispatch_policy() == dispatch_policy, ( + "rocm_aiter_ops cached a stale dispatch policy after the env var changed." + ) + + with mock.patch.object( + rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe + ) as fused_moe_mock: + rocm_aiter_fused_experts( + hidden_states=case["hidden_states"], + w1=w1_shuffled, + w2=w2_shuffled, + topk_weights=case["topk_weights"], + topk_ids=case["topk_ids"], + moe_config=moe_config, + activation=MoEActivation.SILU, + moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(), + ) + + assert ( + fused_moe_mock.call_args.kwargs["moe_sorting_dispatch_policy"] + == dispatch_policy + ), ( + "VLLM_ROCM_AITER_MOE_DISPATCH_POLICY was not forwarded to " + "rocm_aiter_ops.fused_moe." + ) + + # Enum tests -------------------------------------------------------------- diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 0a28c06ee7..af0213361e 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -2019,6 +2019,7 @@ class rocm_aiter_ops: cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2 _sync_aiter_situv2_moe_env() cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM + cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY @staticmethod def get_aiter_activation_type(activation_str: str) -> "ActivationType | None": diff --git a/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py b/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py index 2533c3a895..98a2577967 100644 --- a/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_flydsl_moe.py @@ -7,11 +7,6 @@ import json import os import torch -from aiter.fused_moe import moe_sorting as aiter_moe_sorting -from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import ( - flydsl_a16w4_gemm1, - flydsl_a16w4_gemm2, -) from vllm.logger import init_logger from vllm.utils.platform_utils import get_device_name_as_file_name @@ -81,6 +76,10 @@ def moe_sorting( model_dim: int, block_m: int, ): + # Imported lazily so aiter's import-time sort-backend env vars are read + # when this is called, not whenever this module happens to be imported. + from aiter.fused_moe import moe_sorting as aiter_moe_sorting + topk_ids_i32 = topk_ids.to(torch.int32) topk_w_f32 = topk_weights.to(torch.float32) sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = ( @@ -185,6 +184,12 @@ def fused_flydsl_moe_impl( tile_n2: int | None = None, tile_k2: int | None = None, ) -> torch.Tensor: + # Imported lazily -- see the comment in moe_sorting() above. + from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import ( + flydsl_a16w4_gemm1, + flydsl_a16w4_gemm2, + ) + device = hidden_states.device tokens = hidden_states.shape[0] model_dim = hidden_states.shape[1]