diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index ce4fc3cfba..2c9b939fa2 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -144,6 +144,7 @@ def _rocm_aiter_fused_moe_impl( intermediate_pad: int = 0, bias1: torch.Tensor | None = None, bias2: torch.Tensor | None = None, + moe_sorting_dispatch_policy: int = 0, ) -> torch.Tensor: from aiter import ActivationType, QuantType from aiter.fused_moe import fused_moe @@ -171,6 +172,7 @@ def _rocm_aiter_fused_moe_impl( intermediate_pad=intermediate_pad, bias1=bias1, bias2=bias2, + moe_sorting_dispatch_policy=moe_sorting_dispatch_policy, ) @@ -194,6 +196,7 @@ def _rocm_aiter_fused_moe_fake( intermediate_pad: int = 0, bias1: torch.Tensor | None = None, bias2: torch.Tensor | None = None, + moe_sorting_dispatch_policy: int = 0, ) -> torch.Tensor: if output_dtype is not None: return torch.empty_like(hidden_states, dtype=output_dtype) @@ -1282,6 +1285,18 @@ class rocm_aiter_ops: - Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale """ + _MOE_DISPATCH_POLICY: int | None = None + + @classmethod + @if_aiter_supported + def get_moe_dispatch_policy(cls) -> int: + """Cached MoE sorting dispatch policy.""" + if cls._MOE_DISPATCH_POLICY is None: + import vllm.envs as envs + + cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY + return cls._MOE_DISPATCH_POLICY + # Check if the env variable is set _AITER_ENABLED = envs.VLLM_ROCM_USE_AITER _LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR @@ -1890,6 +1905,7 @@ class rocm_aiter_ops: intermediate_pad: int = 0, bias1: torch.Tensor | None = None, bias2: torch.Tensor | None = None, + moe_sorting_dispatch_policy: int = 0, ) -> torch.Tensor: return torch.ops.vllm.rocm_aiter_fused_moe( hidden_states, @@ -1911,6 +1927,7 @@ class rocm_aiter_ops: intermediate_pad, bias1, bias2, + moe_sorting_dispatch_policy, ) @staticmethod diff --git a/vllm/envs.py b/vllm/envs.py index 9b4a6790b6..3a3934f3cd 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -116,6 +116,7 @@ if TYPE_CHECKING: VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False VLLM_ROCM_USE_AITER_LINEAR: bool = True VLLM_ROCM_USE_AITER_MOE: bool = True + VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0 VLLM_ROCM_USE_AITER_RMSNORM: bool = True VLLM_ROCM_USE_AITER_MLA: bool = True VLLM_ROCM_USE_AITER_MHA: bool = True @@ -1103,6 +1104,17 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_ROCM_USE_AITER_MOE": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1") ), + # MoE sorting dispatch policy for AITER fused MoE kernels. + # 0 = auto (default): single-pass for small batches, multi-pass + # for large batches + # 1 = always single-pass: one kernel launch, no workspace, + # may be preferred for low-concurrency decode workloads + # 2 = always multi-pass: can be faster for MoE-heavy models + # (e.g., +2-5% on Qwen3-Next, +1.5% on DeepSeek-V3 at TP4, + # see PR #39177 for benchmarks) + "VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": lambda: int( + os.getenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", "0") + ), # use aiter rms norm op if aiter ops are enabled. "VLLM_ROCM_USE_AITER_RMSNORM": lambda: ( os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in ("true", "1") diff --git a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py index 2b8abbfc50..64c8604acd 100644 --- a/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/rocm_aiter_moe.py @@ -246,6 +246,7 @@ def rocm_aiter_fused_experts( a1q_scale: torch.Tensor | None = None, num_local_tokens: torch.Tensor | None = None, output_dtype: torch.dtype | None = None, + moe_sorting_dispatch_policy: int = 0, ) -> torch.Tensor: """ROCm AITER fused MoE expert computation.""" if quant_config is None: @@ -361,6 +362,7 @@ def rocm_aiter_fused_experts( intermediate_pad=intermediate_pad // 64 * 64 * 2, bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None, bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None, + moe_sorting_dispatch_policy=moe_sorting_dispatch_policy, ) @@ -504,6 +506,7 @@ class AiterExperts(mk.FusedMoEExpertsModular): a1q_scale=a1q_scale, num_local_tokens=num_local_tokens, output_dtype=output.dtype, + moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(), ) # avoid redundant copy when output is a view of the result if (