[ROCm][CI][AITER Coverage] Harden MoE sorting-backend/dispatch env-var test matrix (#58393)
Signed-off-by: Divakar Verma <divakar.verma@amd.com>
This commit is contained in:
@@ -2,20 +2,24 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
import inspect
|
||||
import textwrap
|
||||
import traceback
|
||||
from itertools import product
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_gemm
|
||||
from vllm.utils.import_utils import has_aiter, has_deep_ep, has_deep_gemm
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@@ -72,7 +76,7 @@ def assert_aiter_quant_scheme_case(config: Config) -> None:
|
||||
indirectly through the general quant-config sweep.
|
||||
See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if getattr(fe_cls, "__name__", "") != "AiterExperts":
|
||||
if fe_cls is not AiterExperts:
|
||||
return
|
||||
|
||||
if config.quant_config is None:
|
||||
@@ -93,7 +97,7 @@ def assert_aiter_activation_case(config: Config) -> None:
|
||||
AiterExperts being reached only indirectly through the general
|
||||
activation sweep. See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if getattr(fe_cls, "__name__", "") != "AiterExperts":
|
||||
if fe_cls is not AiterExperts:
|
||||
return
|
||||
|
||||
assert fe_cls._supports_activation(config.activation), (
|
||||
@@ -148,7 +152,7 @@ def rank_worker(
|
||||
if (
|
||||
topk == 1
|
||||
and config.supports_apply_weight_on_input()
|
||||
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and config.quant_block_shape is not None
|
||||
):
|
||||
print(
|
||||
@@ -164,7 +168,7 @@ def rank_worker(
|
||||
# https://github.com/vllm-project/vllm/issues/57029
|
||||
if (
|
||||
config.world_size > 1
|
||||
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and getattr(config.prepare_finalize_type, "__name__", "")
|
||||
in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize")
|
||||
):
|
||||
@@ -202,7 +206,7 @@ def rank_worker(
|
||||
|
||||
is_aiter_fp8 = (
|
||||
_cp.is_rocm()
|
||||
and getattr(config.fused_experts_type, "__name__", "") == "AiterExperts"
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and config.quant_config is not None
|
||||
)
|
||||
if is_aiter_fp8:
|
||||
@@ -262,7 +266,7 @@ def is_nyi_config(config: Config) -> bool:
|
||||
return True
|
||||
|
||||
if config.activation != MoEActivation.SILU:
|
||||
if getattr(config.fused_experts_type, "__name__", "") != "AiterExperts":
|
||||
if config.fused_experts_type is not AiterExperts:
|
||||
return True # AITER-only for this axis, for now
|
||||
if config.quant_dtype is not None:
|
||||
return True # unquantized-only for this axis, for now
|
||||
@@ -425,6 +429,268 @@ def test_modular_kernel_combinations_singlegpu(
|
||||
run(config, verbosity > 0)
|
||||
|
||||
|
||||
# AITER sorting-backend dispatch env-var matrix (issue #54966 step 3) -------
|
||||
#
|
||||
# AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING are aiter globals
|
||||
# read once at import time, so each combo needs a fresh child process (see
|
||||
# parallel_launch_with_config) rather than in-process monkeypatching.
|
||||
#
|
||||
# AITER_MOE_SORT_BACKEND is not covered: it only matters when output_aux=True,
|
||||
# which vLLM only sets on the MXFP4 path -- not yet wired into AiterExperts.
|
||||
|
||||
|
||||
def _aiter_sort_backend_hooks_available() -> bool:
|
||||
"""Whether the installed aiter build exposes the sort-backend hooks that
|
||||
AITER_USE_CK_MOE_SORTING / AITER_USE_FLYDSL_MOE_SORTING select between.
|
||||
"""
|
||||
try:
|
||||
import aiter.fused_moe as aiter_fused_moe
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
required = (
|
||||
"_moe_sorting_impl",
|
||||
"_flydsl_moe_sorting",
|
||||
"_USE_CK_MOE_SORTING",
|
||||
"_USE_FLYDSL_MOE_SORTING",
|
||||
)
|
||||
return all(hasattr(aiter_fused_moe, name) for name in required)
|
||||
|
||||
|
||||
# Deliberately does NOT call _aiter_sort_backend_hooks_available() here: this
|
||||
# marker is evaluated at module-import time, which also happens inside each
|
||||
# spawned child (to unpickle its worker) -- before that case's env vars are
|
||||
# applied, permanently freezing aiter's sort-backend globals. The hooks check
|
||||
# instead runs inside the test body below (parent process only).
|
||||
require_aiter_moe = pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_rocm()
|
||||
and has_aiter()
|
||||
and rocm_aiter_ops.is_fused_moe_enabled()
|
||||
),
|
||||
reason=(
|
||||
"AITER MoE sorting-backend dispatch needs ROCm + AITER, with "
|
||||
"VLLM_ROCM_USE_AITER=1 and VLLM_ROCM_USE_AITER_MOE=1 set before "
|
||||
"the test process starts."
|
||||
),
|
||||
)
|
||||
|
||||
# (AITER_USE_CK_MOE_SORTING, AITER_USE_FLYDSL_MOE_SORTING, expected_backend)
|
||||
AITER_SORTING_BACKEND_ENV_CASES = [
|
||||
(1, 0, "ck"),
|
||||
(1, 1, "ck"), # CK wins over FlyDSL even when both are requested.
|
||||
(0, 1, "flydsl"),
|
||||
(0, 0, "opus"), # default when neither flag is set.
|
||||
]
|
||||
|
||||
|
||||
def _aiter_sorting_backend_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
verbose: bool,
|
||||
ck: int,
|
||||
flydsl: int,
|
||||
expected_backend: str,
|
||||
):
|
||||
# Imported lazily in the spawned child, after env vars are applied, so
|
||||
# aiter's import-time globals pick up this case's env.
|
||||
import aiter.fused_moe as aiter_fused_moe
|
||||
|
||||
# Assert the env vars actually stuck in aiter's import-time globals,
|
||||
# rather than inferring it indirectly from which sorting function fires.
|
||||
assert bool(ck) == aiter_fused_moe._USE_CK_MOE_SORTING, (
|
||||
f"aiter.fused_moe._USE_CK_MOE_SORTING={aiter_fused_moe._USE_CK_MOE_SORTING} "
|
||||
f"but AITER_USE_CK_MOE_SORTING={ck} was set before this process started -- "
|
||||
"the env var never reached aiter's import-time globals."
|
||||
)
|
||||
assert bool(flydsl) == aiter_fused_moe._USE_FLYDSL_MOE_SORTING, (
|
||||
"aiter.fused_moe._USE_FLYDSL_MOE_SORTING="
|
||||
f"{aiter_fused_moe._USE_FLYDSL_MOE_SORTING} but AITER_USE_FLYDSL_MOE_SORTING="
|
||||
f"{flydsl} was set before this process started -- the env var never "
|
||||
"reached aiter's import-time globals."
|
||||
)
|
||||
|
||||
# Captured before patching, since the patched attribute would resolve to
|
||||
# the mock's own (*args, **kwargs) signature instead of the real one.
|
||||
original_moe_sorting_impl = aiter_fused_moe._moe_sorting_impl
|
||||
|
||||
with (
|
||||
mock.patch.object(
|
||||
aiter_fused_moe,
|
||||
"_moe_sorting_impl",
|
||||
wraps=original_moe_sorting_impl,
|
||||
) as sorting_impl_mock,
|
||||
mock.patch.object(
|
||||
aiter_fused_moe,
|
||||
"_flydsl_moe_sorting",
|
||||
wraps=aiter_fused_moe._flydsl_moe_sorting,
|
||||
) as flydsl_mock,
|
||||
):
|
||||
# Reuses the real correctness/accuracy checking rank_worker already
|
||||
# does for every other AiterExperts combo, instead of duplicating it.
|
||||
rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose)
|
||||
|
||||
if expected_backend == "flydsl":
|
||||
assert flydsl_mock.call_count > 0, "Expected FlyDSL sorting to fire."
|
||||
assert sorting_impl_mock.call_count == 0, (
|
||||
"FlyDSL was requested and eligible, but the opus/CK sorting "
|
||||
"path fired instead -- a silent fallback."
|
||||
)
|
||||
else:
|
||||
assert flydsl_mock.call_count == 0, (
|
||||
f"Expected the '{expected_backend}' sorting path, but FlyDSL "
|
||||
"fired instead."
|
||||
)
|
||||
assert sorting_impl_mock.call_count > 0, (
|
||||
f"Expected the '{expected_backend}' sorting path to fire."
|
||||
)
|
||||
# Bind by signature so this works regardless of whether use_opus
|
||||
# is passed positionally or by keyword.
|
||||
call = sorting_impl_mock.call_args
|
||||
bound = inspect.signature(original_moe_sorting_impl).bind(
|
||||
*call.args, **call.kwargs
|
||||
)
|
||||
use_opus = bound.arguments["use_opus"]
|
||||
assert use_opus == (expected_backend == "opus"), (
|
||||
f"Expected use_opus={expected_backend == 'opus'} for the "
|
||||
f"'{expected_backend}' sorting path, got use_opus={use_opus}."
|
||||
)
|
||||
|
||||
|
||||
@require_aiter_moe
|
||||
@pytest.mark.parametrize("ck,flydsl,expected_backend", AITER_SORTING_BACKEND_ENV_CASES)
|
||||
def test_aiter_moe_sorting_backend_dispatch_env_matrix(
|
||||
ck: int, flydsl: int, expected_backend: str
|
||||
):
|
||||
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER
|
||||
backend and dispatch settings")."""
|
||||
if not _aiter_sort_backend_hooks_available():
|
||||
pytest.skip(
|
||||
"Installed aiter build predates the sort-backend hooks "
|
||||
"(_moe_sorting_impl, _flydsl_moe_sorting, _USE_CK_MOE_SORTING, "
|
||||
"_USE_FLYDSL_MOE_SORTING)."
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
config = Config(
|
||||
Ms=[32],
|
||||
K=2048,
|
||||
N=1024,
|
||||
E=32,
|
||||
topks=[4],
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
|
||||
weights = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
env_dict = {
|
||||
**env_dict,
|
||||
"AITER_USE_CK_MOE_SORTING": str(ck),
|
||||
"AITER_USE_FLYDSL_MOE_SORTING": str(flydsl),
|
||||
}
|
||||
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
_aiter_sorting_backend_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
False,
|
||||
ck,
|
||||
flydsl,
|
||||
expected_backend,
|
||||
)
|
||||
|
||||
|
||||
def _aiter_dispatch_policy_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
verbose: bool,
|
||||
dispatch_policy: int,
|
||||
):
|
||||
# Drives the real AiterExperts.apply() -> rocm_aiter_fused_experts() call
|
||||
# through the modular kernel, so it also catches a regression in apply()'s
|
||||
# own forwarding (unlike test_rocm_aiter_moe.py's version, which supplies
|
||||
# the policy value directly to rocm_aiter_fused_experts()).
|
||||
with mock.patch.object(
|
||||
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
|
||||
) as fused_moe_mock:
|
||||
rank_worker(pgi, vllm_config, cpu_group, config, weights, verbose)
|
||||
|
||||
assert fused_moe_mock.call_count > 0, (
|
||||
"Expected AiterExperts.apply() to call rocm_aiter_ops.fused_moe."
|
||||
)
|
||||
for call in fused_moe_mock.call_args_list:
|
||||
forwarded = call.kwargs["moe_sorting_dispatch_policy"]
|
||||
assert forwarded == dispatch_policy, (
|
||||
f"AiterExperts.apply() forwarded moe_sorting_dispatch_policy="
|
||||
f"{forwarded}, but VLLM_ROCM_AITER_MOE_DISPATCH_POLICY="
|
||||
f"{dispatch_policy} was set before this process started."
|
||||
)
|
||||
|
||||
|
||||
@require_aiter_moe
|
||||
@pytest.mark.parametrize("dispatch_policy", [0, 1, 2])
|
||||
def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int):
|
||||
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test AITER
|
||||
backend and dispatch settings").
|
||||
|
||||
Complements test_aiter_moe_dispatch_policy_forwarded_to_fused_moe in
|
||||
test_rocm_aiter_moe.py by driving the real AiterExperts.apply() call
|
||||
instead of supplying moe_sorting_dispatch_policy directly.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
config = Config(
|
||||
Ms=[32],
|
||||
K=2048,
|
||||
N=1024,
|
||||
E=32,
|
||||
topks=[4],
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
|
||||
weights = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
env_dict = {
|
||||
**env_dict,
|
||||
"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": str(dispatch_policy),
|
||||
}
|
||||
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
_aiter_dispatch_policy_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
False,
|
||||
dispatch_policy,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Ability to test individual PrepareAndFinalize and FusedExperts combination
|
||||
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
|
||||
|
||||
@@ -22,6 +22,7 @@ import importlib
|
||||
import math
|
||||
import warnings
|
||||
from typing import Any, NamedTuple
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -645,6 +646,76 @@ def test_aiter_moe_padding_env_var(
|
||||
assert envs.VLLM_ROCM_MOE_PADDING is moe_padding
|
||||
|
||||
|
||||
# Dispatch-policy forwarding test ------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dispatch_policy", [0, 1, 2])
|
||||
def test_aiter_moe_dispatch_policy_forwarded_to_fused_moe(
|
||||
dispatch_policy: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""VLLM_ROCM_AITER_MOE_DISPATCH_POLICY should reach rocm_aiter_ops.fused_moe
|
||||
unchanged, the same forwarding AiterExperts.apply does via
|
||||
rocm_aiter_fused_experts, for every documented policy value (0=auto,
|
||||
1=always single-pass, 2=always multi-pass). See vllm-project/vllm#54966.
|
||||
"""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
rocm_aiter_fused_experts,
|
||||
)
|
||||
|
||||
_assert_aiter_supported()
|
||||
case = _make_moe_case(
|
||||
num_tokens=8,
|
||||
hidden_dim=256,
|
||||
intermediate_dim=512,
|
||||
num_experts=4,
|
||||
topk=2,
|
||||
seed=5,
|
||||
)
|
||||
w1_shuffled, w2_shuffled = _shuffle_moe_weights(case["w1"], case["w2"])
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=4,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", str(dispatch_policy))
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
assert rocm_aiter_ops.get_moe_dispatch_policy() == dispatch_policy, (
|
||||
"rocm_aiter_ops cached a stale dispatch policy after the env var changed."
|
||||
)
|
||||
|
||||
with mock.patch.object(
|
||||
rocm_aiter_ops, "fused_moe", wraps=rocm_aiter_ops.fused_moe
|
||||
) as fused_moe_mock:
|
||||
rocm_aiter_fused_experts(
|
||||
hidden_states=case["hidden_states"],
|
||||
w1=w1_shuffled,
|
||||
w2=w2_shuffled,
|
||||
topk_weights=case["topk_weights"],
|
||||
topk_ids=case["topk_ids"],
|
||||
moe_config=moe_config,
|
||||
activation=MoEActivation.SILU,
|
||||
moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(),
|
||||
)
|
||||
|
||||
assert (
|
||||
fused_moe_mock.call_args.kwargs["moe_sorting_dispatch_policy"]
|
||||
== dispatch_policy
|
||||
), (
|
||||
"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY was not forwarded to "
|
||||
"rocm_aiter_ops.fused_moe."
|
||||
)
|
||||
|
||||
|
||||
# Enum tests --------------------------------------------------------------
|
||||
|
||||
|
||||
|
||||
@@ -2019,6 +2019,7 @@ class rocm_aiter_ops:
|
||||
cls._MOE_SITUV2 = envs.VLLM_ROCM_USE_AITER_MOE_SITUV2
|
||||
_sync_aiter_situv2_moe_env()
|
||||
cls._TRITON_UNQUANT_GEMM = envs.VLLM_ROCM_USE_AITER_TRITON_GEMM
|
||||
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
|
||||
|
||||
@staticmethod
|
||||
def get_aiter_activation_type(activation_str: str) -> "ActivationType | None":
|
||||
|
||||
@@ -7,11 +7,6 @@ import json
|
||||
import os
|
||||
|
||||
import torch
|
||||
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
|
||||
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
|
||||
flydsl_a16w4_gemm1,
|
||||
flydsl_a16w4_gemm2,
|
||||
)
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.platform_utils import get_device_name_as_file_name
|
||||
@@ -81,6 +76,10 @@ def moe_sorting(
|
||||
model_dim: int,
|
||||
block_m: int,
|
||||
):
|
||||
# Imported lazily so aiter's import-time sort-backend env vars are read
|
||||
# when this is called, not whenever this module happens to be imported.
|
||||
from aiter.fused_moe import moe_sorting as aiter_moe_sorting
|
||||
|
||||
topk_ids_i32 = topk_ids.to(torch.int32)
|
||||
topk_w_f32 = topk_weights.to(torch.float32)
|
||||
sorted_ids, sorted_w, sorted_expert_ids, num_valid_ids, _moe_buf = (
|
||||
@@ -185,6 +184,12 @@ def fused_flydsl_moe_impl(
|
||||
tile_n2: int | None = None,
|
||||
tile_k2: int | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Imported lazily -- see the comment in moe_sorting() above.
|
||||
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
|
||||
flydsl_a16w4_gemm1,
|
||||
flydsl_a16w4_gemm2,
|
||||
)
|
||||
|
||||
device = hidden_states.device
|
||||
tokens = hidden_states.shape[0]
|
||||
model_dim = hidden_states.shape[1]
|
||||
|
||||
Reference in New Issue
Block a user