[ROCm][Perf] Expose AITER MoE sorting dispatch policy via env var (#39177)
Signed-off-by: nholmber <nholmber@users.noreply.github.com>
This commit is contained in:
@@ -144,6 +144,7 @@ def _rocm_aiter_fused_moe_impl(
|
||||
intermediate_pad: int = 0,
|
||||
bias1: torch.Tensor | None = None,
|
||||
bias2: torch.Tensor | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
) -> torch.Tensor:
|
||||
from aiter import ActivationType, QuantType
|
||||
from aiter.fused_moe import fused_moe
|
||||
@@ -171,6 +172,7 @@ def _rocm_aiter_fused_moe_impl(
|
||||
intermediate_pad=intermediate_pad,
|
||||
bias1=bias1,
|
||||
bias2=bias2,
|
||||
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
|
||||
)
|
||||
|
||||
|
||||
@@ -194,6 +196,7 @@ def _rocm_aiter_fused_moe_fake(
|
||||
intermediate_pad: int = 0,
|
||||
bias1: torch.Tensor | None = None,
|
||||
bias2: torch.Tensor | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
) -> torch.Tensor:
|
||||
if output_dtype is not None:
|
||||
return torch.empty_like(hidden_states, dtype=output_dtype)
|
||||
@@ -1282,6 +1285,18 @@ class rocm_aiter_ops:
|
||||
- Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale
|
||||
"""
|
||||
|
||||
_MOE_DISPATCH_POLICY: int | None = None
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def get_moe_dispatch_policy(cls) -> int:
|
||||
"""Cached MoE sorting dispatch policy."""
|
||||
if cls._MOE_DISPATCH_POLICY is None:
|
||||
import vllm.envs as envs
|
||||
|
||||
cls._MOE_DISPATCH_POLICY = envs.VLLM_ROCM_AITER_MOE_DISPATCH_POLICY
|
||||
return cls._MOE_DISPATCH_POLICY
|
||||
|
||||
# Check if the env variable is set
|
||||
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
|
||||
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
|
||||
@@ -1890,6 +1905,7 @@ class rocm_aiter_ops:
|
||||
intermediate_pad: int = 0,
|
||||
bias1: torch.Tensor | None = None,
|
||||
bias2: torch.Tensor | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
) -> torch.Tensor:
|
||||
return torch.ops.vllm.rocm_aiter_fused_moe(
|
||||
hidden_states,
|
||||
@@ -1911,6 +1927,7 @@ class rocm_aiter_ops:
|
||||
intermediate_pad,
|
||||
bias1,
|
||||
bias2,
|
||||
moe_sorting_dispatch_policy,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -116,6 +116,7 @@ if TYPE_CHECKING:
|
||||
VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False
|
||||
VLLM_ROCM_USE_AITER_LINEAR: bool = True
|
||||
VLLM_ROCM_USE_AITER_MOE: bool = True
|
||||
VLLM_ROCM_AITER_MOE_DISPATCH_POLICY: int = 0
|
||||
VLLM_ROCM_USE_AITER_RMSNORM: bool = True
|
||||
VLLM_ROCM_USE_AITER_MLA: bool = True
|
||||
VLLM_ROCM_USE_AITER_MHA: bool = True
|
||||
@@ -1103,6 +1104,17 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ROCM_USE_AITER_MOE": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_MOE", "True").lower() in ("true", "1")
|
||||
),
|
||||
# MoE sorting dispatch policy for AITER fused MoE kernels.
|
||||
# 0 = auto (default): single-pass for small batches, multi-pass
|
||||
# for large batches
|
||||
# 1 = always single-pass: one kernel launch, no workspace,
|
||||
# may be preferred for low-concurrency decode workloads
|
||||
# 2 = always multi-pass: can be faster for MoE-heavy models
|
||||
# (e.g., +2-5% on Qwen3-Next, +1.5% on DeepSeek-V3 at TP4,
|
||||
# see PR #39177 for benchmarks)
|
||||
"VLLM_ROCM_AITER_MOE_DISPATCH_POLICY": lambda: int(
|
||||
os.getenv("VLLM_ROCM_AITER_MOE_DISPATCH_POLICY", "0")
|
||||
),
|
||||
# use aiter rms norm op if aiter ops are enabled.
|
||||
"VLLM_ROCM_USE_AITER_RMSNORM": lambda: (
|
||||
os.getenv("VLLM_ROCM_USE_AITER_RMSNORM", "True").lower() in ("true", "1")
|
||||
|
||||
@@ -246,6 +246,7 @@ def rocm_aiter_fused_experts(
|
||||
a1q_scale: torch.Tensor | None = None,
|
||||
num_local_tokens: torch.Tensor | None = None,
|
||||
output_dtype: torch.dtype | None = None,
|
||||
moe_sorting_dispatch_policy: int = 0,
|
||||
) -> torch.Tensor:
|
||||
"""ROCm AITER fused MoE expert computation."""
|
||||
if quant_config is None:
|
||||
@@ -361,6 +362,7 @@ def rocm_aiter_fused_experts(
|
||||
intermediate_pad=intermediate_pad // 64 * 64 * 2,
|
||||
bias1=quant_config.w1_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
bias2=quant_config.w2_bias if quant_config.use_mxfp4_w4a16 else None,
|
||||
moe_sorting_dispatch_policy=moe_sorting_dispatch_policy,
|
||||
)
|
||||
|
||||
|
||||
@@ -504,6 +506,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
a1q_scale=a1q_scale,
|
||||
num_local_tokens=num_local_tokens,
|
||||
output_dtype=output.dtype,
|
||||
moe_sorting_dispatch_policy=rocm_aiter_ops.get_moe_dispatch_policy(),
|
||||
)
|
||||
# avoid redundant copy when output is a view of the result
|
||||
if (
|
||||
|
||||
Reference in New Issue
Block a user