[ROCm][MoE] Fix expert_map vs AITER expert_mask for non-AITER experts under EP (#49758)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rohan Potdar
2026-08-11 15:33:02 -05:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 36f4630d89
commit 0f0cb918b7
6 changed files with 70 additions and 59 deletions
@@ -62,6 +62,8 @@ def is_aiter_mxfp8_moe_available() -> bool:
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
consumes_expert_mask = True
@property
def quant_dtype(self) -> torch.dtype | str | None:
return self.quant_config.quant_dtype
@@ -148,21 +150,9 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
limit = self.quant_config.gemm1_clamp_limit
swiglu_limit = 0.0 if limit is None else float(limit)
# Under EP, aiter expects ``expert_mask``: a 0/1 *local-expert* mask over
# global ids with a trailing fake-expert sentinel slot (shape
# ``[global_num_experts + 1]``), from which it derives the global->local
# compaction. What ``RoutedExperts.expert_map`` hands us depends on the
# aiter master switch (``rocm_aiter_fmoe_enabled``).
# Branching on the (static) master flag — not the tensor contents —
# keeps this HIP-graph/torch.compile safe (no data-dependent sync).
# ``None`` under pure TP.
if expert_map is None:
expert_mask = None
elif self.moe_config.rocm_aiter_fmoe_enabled:
expert_mask = expert_map
else:
local_mask = (expert_map >= 0).to(torch.int32)
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
# RoutedExperts.expert_map hands AITER experts the precomputed 0/1
# expert_mask (with trailing sentinel) instead of the vLLM expert_map.
expert_mask = expert_map
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
# call is captured under HIP graphs / torch.compile (a direct
@@ -240,7 +240,7 @@ def rocm_aiter_fused_experts(
moe_config: FusedMoEConfig,
activation: MoEActivation = MoEActivation.SILU,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
expert_mask: torch.Tensor | None = None,
quant_config: FusedMoEQuantConfig | None = None,
a1q_scale: torch.Tensor | None = None,
num_local_tokens: torch.Tensor | None = None,
@@ -271,8 +271,6 @@ def rocm_aiter_fused_experts(
topk_weights = topk_weights.to(torch.float32)
topk_ids = topk_ids.to(torch.int32)
expert_mask = expert_map if expert_map is not None else None
# w8a8 per-channel quantization
if (
quant_config.per_act_token_quant
@@ -420,6 +418,8 @@ def rocm_aiter_fused_experts(
class AiterExperts(mk.FusedMoEExpertsModular):
consumes_expert_mask = True
@property
def expects_unquantized_inputs(self) -> bool:
# When paired with MoRI, the prepare/finalize handles FP8
@@ -550,7 +550,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
topk_ids=topk_ids,
activation=activation,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
expert_mask=expert_map,
quant_config=self.quant_config,
moe_config=self.moe_config,
a1q_scale=a1q_scale,
@@ -470,6 +470,12 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
# TODO: add supported activations method (return string)
class FusedMoEExperts(ABC):
# ROCm AITER kernels consume a 0/1 local-expert mask (with a trailing
# sentinel slot); every other backend consumes the canonical -1/local-slot
# expert_map. RoutedExperts.expert_map reads this flag to pick which to hand
# the active experts kernel.
consumes_expert_mask: bool = False
def __init__(
self,
moe_config: FusedMoEConfig,
@@ -226,9 +226,15 @@ class RoutedExperts(PluggableLayer):
@property
def expert_map(self) -> torch.Tensor | None:
return (
self._expert_map if not self.rocm_aiter_fmoe_enabled else self.expert_mask
)
# AITER fused-MoE kernels consume the 0/1 expert_mask; every other
# backend consumes the canonical -1/local-slot map. Ask the active
# experts kernel which it wants (only AITER sets consumes_expert_mask)
# rather than keying on the global VLLM_ROCM_USE_AITER switch, which
# does not reflect the per-layer kernel actually selected.
moe_kernel = getattr(self.quant_method, "moe_kernel", None)
if moe_kernel is not None and moe_kernel.fused_experts.consumes_expert_mask:
return self.expert_mask
return self._expert_map
def update_expert_map_info(self):
# Update local attributes from ExpertMapManager
@@ -1026,7 +1026,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
apply_router_weight_on_input=layer.apply_router_weight_on_input,
quant_config=self.moe_quant_config,
moe_config=layer.moe_config,
expert_map=layer.expert_map,
expert_mask=layer.expert_mask,
)