[ROCm][Perf] Route softmax MoE top-k, including fused shared-expert sigmoid, through AITER topk_gating (#56226)

Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-15 17:53:18 +00:00
committed by GitHub
co-authored by Cursor
parent 4f49b55c1f
commit 47fd76e4ce
6 changed files with 469 additions and 8 deletions
+82 -1
View File
@@ -526,6 +526,46 @@ def _rocm_aiter_topk_softmax_impl(
)
def _rocm_aiter_topk_gating_impl(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
token_expert_indices: torch.Tensor,
gating_output: torch.Tensor,
renormalize: bool,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> None:
# Callers pick this op only for launches AITER's topk_gating supports (see
# dispatch_topk_softmax_func); everything else goes to topk_softmax.
from aiter.ops.topk import topk_gating
# AITER renormalizes the selected softmax mass inside this launch,
# preserving vLLM's `renormalize=True` semantics without a second
# pointwise kernel, and sigmoids the trailing shared-expert columns.
topk_gating(
topk_weights,
topk_indices,
gating_output,
correction_bias=None,
need_renorm=renormalize,
routed_scaling_factor=1.0,
score_func="softmax",
num_shared_experts=num_shared_experts,
)
def _rocm_aiter_topk_gating_fake(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
token_expert_indices: torch.Tensor,
gating_output: torch.Tensor,
renormalize: bool,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> None:
pass
def _rocm_aiter_topk_sigmoid_impl(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
@@ -2020,7 +2060,7 @@ class rocm_aiter_ops:
Operations:
- GEMM operations: gemm_a8w8, gemm_a8w8_blockscale
- Fused MoE: fused_moe, asm_moe_tkw1
- Routing: topk_softmax, biased_grouped_topk, grouped_topk
- Routing: topk_softmax, topk_gating, biased_grouped_topk, grouped_topk
- MLA decode: mla_decode_fwd
- Quantization: per_tensor_quant, per_token_quant, group_fp8_quant
- Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale
@@ -2198,6 +2238,18 @@ class rocm_aiter_ops:
def is_fused_moe_enabled(cls) -> bool:
return cls._AITER_ENABLED and cls._FMOE_ENABLED
@classmethod
@if_aiter_supported
def is_topk_gating_enabled(cls) -> bool:
"""Use AITER ``topk_gating`` for softmax routing.
CDNA3 and CDNA4 only (gfx942, gfx950): those are the targets it was
measured on. Other AITER targets keep the legacy ``topk_softmax`` launcher.
"""
from vllm.platforms.rocm import get_cdna_version
return cls.is_fused_moe_enabled() and get_cdna_version() in (3, 4)
@classmethod
@if_aiter_supported
def is_fusion_moe_shared_experts_enabled(cls) -> bool:
@@ -2851,6 +2903,14 @@ class rocm_aiter_ops:
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="rocm_aiter_topk_gating",
op_func=_rocm_aiter_topk_gating_impl,
mutates_args=["topk_weights", "topk_indices"],
fake_impl=_rocm_aiter_topk_gating_fake,
dispatch_key=current_platform.dispatch_key,
)
direct_register_custom_op(
op_name="rocm_aiter_topk_sigmoid",
op_func=_rocm_aiter_topk_sigmoid_impl,
@@ -3382,6 +3442,27 @@ class rocm_aiter_ops:
)
return topk_weights, topk_indices
@staticmethod
def topk_gating(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
token_expert_indices: torch.Tensor,
gating_output: torch.Tensor,
renormalize: bool,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> tuple[torch.Tensor, ...]:
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_indices,
token_expert_indices,
gating_output,
renormalize,
num_shared_experts,
shared_expert_scoring_func,
)
return topk_weights, topk_indices
@staticmethod
def topk_sigmoid(
topk_weights: torch.Tensor,