[ROCm][Perf] Route softmax MoE top-k, including fused shared-expert sigmoid, through AITER topk_gating (#56226)
Signed-off-by: Mikko Tukiainen <Mikko.Tukiainen@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
4f49b55c1f
commit
47fd76e4ce
+82
-1
@@ -526,6 +526,46 @@ def _rocm_aiter_topk_softmax_impl(
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_topk_gating_impl(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> None:
|
||||
# Callers pick this op only for launches AITER's topk_gating supports (see
|
||||
# dispatch_topk_softmax_func); everything else goes to topk_softmax.
|
||||
from aiter.ops.topk import topk_gating
|
||||
|
||||
# AITER renormalizes the selected softmax mass inside this launch,
|
||||
# preserving vLLM's `renormalize=True` semantics without a second
|
||||
# pointwise kernel, and sigmoids the trailing shared-expert columns.
|
||||
topk_gating(
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
gating_output,
|
||||
correction_bias=None,
|
||||
need_renorm=renormalize,
|
||||
routed_scaling_factor=1.0,
|
||||
score_func="softmax",
|
||||
num_shared_experts=num_shared_experts,
|
||||
)
|
||||
|
||||
|
||||
def _rocm_aiter_topk_gating_fake(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _rocm_aiter_topk_sigmoid_impl(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
@@ -2020,7 +2060,7 @@ class rocm_aiter_ops:
|
||||
Operations:
|
||||
- GEMM operations: gemm_a8w8, gemm_a8w8_blockscale
|
||||
- Fused MoE: fused_moe, asm_moe_tkw1
|
||||
- Routing: topk_softmax, biased_grouped_topk, grouped_topk
|
||||
- Routing: topk_softmax, topk_gating, biased_grouped_topk, grouped_topk
|
||||
- MLA decode: mla_decode_fwd
|
||||
- Quantization: per_tensor_quant, per_token_quant, group_fp8_quant
|
||||
- Triton ops: triton_rotary_embed, triton_fp8_bmm, triton_gemm_a8w8_blockscale
|
||||
@@ -2198,6 +2238,18 @@ class rocm_aiter_ops:
|
||||
def is_fused_moe_enabled(cls) -> bool:
|
||||
return cls._AITER_ENABLED and cls._FMOE_ENABLED
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_topk_gating_enabled(cls) -> bool:
|
||||
"""Use AITER ``topk_gating`` for softmax routing.
|
||||
|
||||
CDNA3 and CDNA4 only (gfx942, gfx950): those are the targets it was
|
||||
measured on. Other AITER targets keep the legacy ``topk_softmax`` launcher.
|
||||
"""
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
|
||||
return cls.is_fused_moe_enabled() and get_cdna_version() in (3, 4)
|
||||
|
||||
@classmethod
|
||||
@if_aiter_supported
|
||||
def is_fusion_moe_shared_experts_enabled(cls) -> bool:
|
||||
@@ -2851,6 +2903,14 @@ class rocm_aiter_ops:
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_topk_gating",
|
||||
op_func=_rocm_aiter_topk_gating_impl,
|
||||
mutates_args=["topk_weights", "topk_indices"],
|
||||
fake_impl=_rocm_aiter_topk_gating_fake,
|
||||
dispatch_key=current_platform.dispatch_key,
|
||||
)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="rocm_aiter_topk_sigmoid",
|
||||
op_func=_rocm_aiter_topk_sigmoid_impl,
|
||||
@@ -3382,6 +3442,27 @@ class rocm_aiter_ops:
|
||||
)
|
||||
return topk_weights, topk_indices
|
||||
|
||||
@staticmethod
|
||||
def topk_gating(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
token_expert_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
renormalize: bool,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
num_shared_experts,
|
||||
shared_expert_scoring_func,
|
||||
)
|
||||
return topk_weights, topk_indices
|
||||
|
||||
@staticmethod
|
||||
def topk_sigmoid(
|
||||
topk_weights: torch.Tensor,
|
||||
|
||||
+12
-2
@@ -84,7 +84,14 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
total_topk_weights_slice = total_topk_weights[:M]
|
||||
topk_ids_slice = total_topk_ids[:M, :topk]
|
||||
|
||||
topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True)
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=True,
|
||||
topk_weights=total_topk_weights_slice,
|
||||
topk_indices=topk_ids_slice,
|
||||
gating_output=router_logits,
|
||||
num_shared_experts=num_fse,
|
||||
shared_expert_scoring_func="sigmoid",
|
||||
)
|
||||
topk_func(
|
||||
total_topk_weights_slice,
|
||||
topk_ids_slice,
|
||||
@@ -110,7 +117,10 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
|
||||
)
|
||||
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
|
||||
topk_weights=topk_weights,
|
||||
topk_indices=topk_ids,
|
||||
gating_output=routing_logits,
|
||||
)
|
||||
topk_weights, topk_ids = topk_func(
|
||||
topk_weights,
|
||||
|
||||
@@ -61,10 +61,73 @@ def vllm_topk_sigmoid(
|
||||
return topk_weights, topk_indices
|
||||
|
||||
|
||||
# At E=512 / top-10, AITER's topk_gating is faster than the legacy launcher
|
||||
# through this many tokens; its one-row generic prefill path regresses above it.
|
||||
_AITER_TOPK_GATING_MAX_TOKENS = 4096
|
||||
|
||||
|
||||
def _aiter_topk_gating_supported(
|
||||
topk_weights: torch.Tensor,
|
||||
topk_indices: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
num_shared_experts: int,
|
||||
shared_expert_scoring_func: str,
|
||||
) -> bool:
|
||||
"""Whether AITER's ``topk_gating`` can serve this softmax top-k launch.
|
||||
|
||||
It needs contiguous gating rows and at most ``_AITER_TOPK_GATING_MAX_TOKENS``
|
||||
tokens. Fused shared experts are scored with sigmoid after the routed
|
||||
top-k: 1, 2, 4 or 8 of them, with ``topk_weights`` and ``topk_indices``
|
||||
sharing a row stride and ``topk_weights`` wide enough for the shared
|
||||
columns. Shared expert ids stay as the caller prefilled them.
|
||||
"""
|
||||
if (
|
||||
not gating_output.is_contiguous()
|
||||
or gating_output.shape[0] > _AITER_TOPK_GATING_MAX_TOKENS
|
||||
):
|
||||
return False
|
||||
if num_shared_experts == 0:
|
||||
return not shared_expert_scoring_func
|
||||
return (
|
||||
shared_expert_scoring_func == "sigmoid"
|
||||
and num_shared_experts in (1, 2, 4, 8)
|
||||
and topk_weights.stride(0) == topk_indices.stride(0)
|
||||
and topk_weights.shape[-1] >= topk_indices.shape[-1] + num_shared_experts
|
||||
)
|
||||
|
||||
|
||||
def dispatch_topk_softmax_func(
|
||||
use_rocm_aiter: bool = False,
|
||||
*,
|
||||
topk_weights: torch.Tensor | None = None,
|
||||
topk_indices: torch.Tensor | None = None,
|
||||
gating_output: torch.Tensor | None = None,
|
||||
num_shared_experts: int = 0,
|
||||
shared_expert_scoring_func: str = "",
|
||||
) -> Callable[..., tuple[torch.Tensor, ...]]:
|
||||
"""Pick the softmax top-k implementation for one launch.
|
||||
|
||||
On ROCm with AITER, ``topk_gating`` is chosen on gfx942 and gfx950 when the
|
||||
launch described by the tensors and shared-expert arguments is one it supports;
|
||||
every other AITER launch uses the legacy ``topk_softmax``. The tensors are
|
||||
optional: without them the launch cannot be judged and the legacy launcher
|
||||
is used.
|
||||
"""
|
||||
if use_rocm_aiter:
|
||||
if (
|
||||
rocm_aiter_ops.is_topk_gating_enabled()
|
||||
and topk_weights is not None
|
||||
and topk_indices is not None
|
||||
and gating_output is not None
|
||||
and _aiter_topk_gating_supported(
|
||||
topk_weights,
|
||||
topk_indices,
|
||||
gating_output,
|
||||
num_shared_experts,
|
||||
shared_expert_scoring_func,
|
||||
)
|
||||
):
|
||||
return rocm_aiter_ops.topk_gating
|
||||
return rocm_aiter_ops.topk_softmax
|
||||
return vllm_topk_softmax
|
||||
|
||||
@@ -104,7 +167,10 @@ def fused_topk(
|
||||
|
||||
if scoring_func == "softmax":
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
|
||||
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
|
||||
topk_weights=topk_weights,
|
||||
topk_indices=topk_ids,
|
||||
gating_output=gating_output,
|
||||
)
|
||||
topk_weights, topk_ids = topk_func(
|
||||
topk_weights, topk_ids, token_expert_indices, gating_output, renormalize
|
||||
|
||||
Reference in New Issue
Block a user