[ROCm][MoE] Fix expert_map vs AITER expert_mask for non-AITER experts under EP (#49758)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
36f4630d89
commit
0f0cb918b7
@@ -62,6 +62,8 @@ def is_aiter_mxfp8_moe_available() -> bool:
|
||||
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
|
||||
|
||||
consumes_expert_mask = True
|
||||
|
||||
@property
|
||||
def quant_dtype(self) -> torch.dtype | str | None:
|
||||
return self.quant_config.quant_dtype
|
||||
@@ -148,21 +150,9 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||
limit = self.quant_config.gemm1_clamp_limit
|
||||
swiglu_limit = 0.0 if limit is None else float(limit)
|
||||
|
||||
# Under EP, aiter expects ``expert_mask``: a 0/1 *local-expert* mask over
|
||||
# global ids with a trailing fake-expert sentinel slot (shape
|
||||
# ``[global_num_experts + 1]``), from which it derives the global->local
|
||||
# compaction. What ``RoutedExperts.expert_map`` hands us depends on the
|
||||
# aiter master switch (``rocm_aiter_fmoe_enabled``).
|
||||
# Branching on the (static) master flag — not the tensor contents —
|
||||
# keeps this HIP-graph/torch.compile safe (no data-dependent sync).
|
||||
# ``None`` under pure TP.
|
||||
if expert_map is None:
|
||||
expert_mask = None
|
||||
elif self.moe_config.rocm_aiter_fmoe_enabled:
|
||||
expert_mask = expert_map
|
||||
else:
|
||||
local_mask = (expert_map >= 0).to(torch.int32)
|
||||
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
|
||||
# RoutedExperts.expert_map hands AITER experts the precomputed 0/1
|
||||
# expert_mask (with trailing sentinel) instead of the vLLM expert_map.
|
||||
expert_mask = expert_map
|
||||
|
||||
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
|
||||
# call is captured under HIP graphs / torch.compile (a direct
|
||||
|
||||
@@ -240,7 +240,7 @@ def rocm_aiter_fused_experts(
|
||||
moe_config: FusedMoEConfig,
|
||||
activation: MoEActivation = MoEActivation.SILU,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
expert_mask: torch.Tensor | None = None,
|
||||
quant_config: FusedMoEQuantConfig | None = None,
|
||||
a1q_scale: torch.Tensor | None = None,
|
||||
num_local_tokens: torch.Tensor | None = None,
|
||||
@@ -271,8 +271,6 @@ def rocm_aiter_fused_experts(
|
||||
topk_weights = topk_weights.to(torch.float32)
|
||||
topk_ids = topk_ids.to(torch.int32)
|
||||
|
||||
expert_mask = expert_map if expert_map is not None else None
|
||||
|
||||
# w8a8 per-channel quantization
|
||||
if (
|
||||
quant_config.per_act_token_quant
|
||||
@@ -420,6 +418,8 @@ def rocm_aiter_fused_experts(
|
||||
|
||||
|
||||
class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
consumes_expert_mask = True
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
# When paired with MoRI, the prepare/finalize handles FP8
|
||||
@@ -550,7 +550,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
expert_map=expert_map,
|
||||
expert_mask=expert_map,
|
||||
quant_config=self.quant_config,
|
||||
moe_config=self.moe_config,
|
||||
a1q_scale=a1q_scale,
|
||||
|
||||
@@ -470,6 +470,12 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
|
||||
|
||||
# TODO: add supported activations method (return string)
|
||||
class FusedMoEExperts(ABC):
|
||||
# ROCm AITER kernels consume a 0/1 local-expert mask (with a trailing
|
||||
# sentinel slot); every other backend consumes the canonical -1/local-slot
|
||||
# expert_map. RoutedExperts.expert_map reads this flag to pick which to hand
|
||||
# the active experts kernel.
|
||||
consumes_expert_mask: bool = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
|
||||
@@ -226,9 +226,15 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
@property
|
||||
def expert_map(self) -> torch.Tensor | None:
|
||||
return (
|
||||
self._expert_map if not self.rocm_aiter_fmoe_enabled else self.expert_mask
|
||||
)
|
||||
# AITER fused-MoE kernels consume the 0/1 expert_mask; every other
|
||||
# backend consumes the canonical -1/local-slot map. Ask the active
|
||||
# experts kernel which it wants (only AITER sets consumes_expert_mask)
|
||||
# rather than keying on the global VLLM_ROCM_USE_AITER switch, which
|
||||
# does not reflect the per-layer kernel actually selected.
|
||||
moe_kernel = getattr(self.quant_method, "moe_kernel", None)
|
||||
if moe_kernel is not None and moe_kernel.fused_experts.consumes_expert_mask:
|
||||
return self.expert_mask
|
||||
return self._expert_map
|
||||
|
||||
def update_expert_map_info(self):
|
||||
# Update local attributes from ExpertMapManager
|
||||
|
||||
@@ -1026,7 +1026,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
quant_config=self.moe_quant_config,
|
||||
moe_config=layer.moe_config,
|
||||
expert_map=layer.expert_map,
|
||||
expert_mask=layer.expert_mask,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user