[ROCm][Perf] Expose AITER MoE sorting dispatch policy via env var (#39177)

Signed-off-by: nholmber <nholmber@users.noreply.github.com>
This commit is contained in:
Nico Holmberg
2026-05-27 13:11:02 +08:00
committed by GitHub
parent 1fc2cee50a
commit 7b54690244
3 changed files with 32 additions and 0 deletions
+17
View File
@@ -144,6 +144,7 @@ def _rocm_aiter_fused_moe_impl(
intermediate_pad: int = 0,
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
) -> torch.Tensor:
from aiter import ActivationType, QuantType
from aiter.fused_moe import fused_moe
@@ -171,6 +172,7 @@ def _rocm_aiter_fused_moe_impl(
intermediate_pad=intermediate_pad,
bias1=bias1,
bias2=bias2,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
)
@@ -194,6 +196,7 @@ def _rocm_aiter_fused_moe_fake(
intermediate_pad: int = 0,
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
) -> torch.Tensor:
if output_dtype is not None:
return torch.empty_like(hidden_states, dtype=output_dtype)
@@ -1282,6 +1285,18 @@ class rocm_aiter_ops:
- Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale
"""
_MOE_DISPATCH_POLICY: int | None = None
@classmethod
@if_aiter_supported
def get_moe_dispatch_policy(cls) -> int:
"""Cached MoE sorting dispatch policy."""
if cls._MOE_DISPATCH_POLICY is None:
import vllm.envs as envs
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
return cls._MOE_DISPATCH_POLICY
# Check if the env variable is set
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
@@ -1890,6 +1905,7 @@ class rocm_aiter_ops:
intermediate_pad: int = 0,
bias1: torch.Tensor | None = None,
bias2: torch.Tensor | None = None,
moe_sorting_dispatch_policy: int = 0,
) -> torch.Tensor:
return torch.ops.vllm.rocm_aiter_fused_moe(
hidden_states,
@@ -1911,6 +1927,7 @@ class rocm_aiter_ops:
intermediate_pad,
bias1,
bias2,
moe_sorting_dispatch_policy,
)
@staticmethod
+12
View File
@@ -116,6 +116,7 @@ if TYPE_CHECKING:
VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False
VLLM_ROCM_USE_AITER_LINEAR: bool = True
VLLM_ROCM_USE_AITER_MOE: bool = True
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
VLLM_ROCM_USE_AITER_MLA: bool = True
VLLM_ROCM_USE_AITER_MHA: bool = True
@@ -1103,6 +1104,17 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER_MOE": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
),
# MoE sorting dispatch policy for AITER fused MoE kernels.
# 0 = auto (default): single-pass for small batches, multi-pass
# for large batches
# 1 = always single-pass: one kernel launch, no workspace,
# may be preferred for low-concurrency decode workloads
# 2 = always multi-pass: can be faster for MoE-heavy models
# (e.g., +2-5% on Qwen3-Next, +1.5% on DeepSeek-V3 at TP4,
# see PR #39177 for benchmarks)
"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": lambda: int(
os.getenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", "0")
),
# use aiter rms norm op if aiter ops are enabled.
"VLLM_ROCM_USE_AITER_RMSNORM": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in ("true", "1")
@@ -246,6 +246,7 @@ def rocm_aiter_fused_experts(
a1q_scale: torch.Tensor | None = None,
num_local_tokens: torch.Tensor | None = None,
output_dtype: torch.dtype | None = None,
moe_sorting_dispatch_policy: int = 0,
) -> torch.Tensor:
"""ROCm AITER fused MoE expert computation."""
if quant_config is None:
@@ -361,6 +362,7 @@ def rocm_aiter_fused_experts(
intermediate_pad=intermediate_pad // 64 * 64 * 2,
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
)
@@ -504,6 +506,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
a1q_scale=a1q_scale,
num_local_tokens=num_local_tokens,
output_dtype=output.dtype,
moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(),
)
# avoid redundant copy when output is a view of the result
if (