[ROCm][MoE] Fix expert_map vs AITER expert_mask for non-AITER experts under EP (#49758)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
36f4630d89
commit
0f0cb918b7
@@ -425,14 +425,14 @@ def test_mxfp8_linear_emulation_bf16_at_load(
|
|||||||
|
|
||||||
|
|
||||||
# ── EP expert_mask handling for the FlyDSL (AITER_MXFP8) MoE ────────────────
|
# ── EP expert_mask handling for the FlyDSL (AITER_MXFP8) MoE ────────────────
|
||||||
# Regression for the EP + aiter-master-switch interaction: under expert
|
# The map/mask choice lives in ``RoutedExperts.expert_map``: it hands AITER
|
||||||
# parallelism ``RoutedExperts.expert_map`` hands the experts either the 0/1
|
# experts (``consumes_expert_mask``) the precomputed 0/1 ``expert_mask`` and
|
||||||
# ``expert_mask`` (aiter master ON, ``rocm_aiter_fmoe_enabled``) or vLLM's -1
|
# everyone else the canonical -1 index map, keyed on the resolved experts kernel
|
||||||
# index map (master OFF). ``AiterMxfp8Experts.apply`` must forward the right 0/1
|
# rather than the global VLLM_ROCM_USE_AITER switch. ``AiterMxfp8Experts.apply``
|
||||||
# mask to aiter in BOTH cases. The old code always rebuilt the mask via
|
# forwards that mask to aiter unchanged (it no longer rebuilds it via
|
||||||
# ``(expert_map >= 0)``; on the already-0/1 mask that collapses to all-ones (no
|
# ``(expert_map >= 0)``, which collapsed an already-0/1 mask to all-ones and
|
||||||
# experts masked out) and EP output becomes garbage (no accuracy).
|
# produced EP garbage).
|
||||||
def _capture_expert_mask(expert_map, *, rocm_aiter_fmoe_enabled, global_num_experts):
|
def _capture_expert_mask(expert_mask, *, global_num_experts):
|
||||||
"""Drive the real ``AiterMxfp8Experts.apply`` mask branch and capture the
|
"""Drive the real ``AiterMxfp8Experts.apply`` mask branch and capture the
|
||||||
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``."""
|
``expert_mask`` it forwards to ``rocm_aiter_ops.fused_moe``."""
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
@@ -444,9 +444,6 @@ def _capture_expert_mask(expert_map, *, rocm_aiter_fmoe_enabled, global_num_expe
|
|||||||
)
|
)
|
||||||
|
|
||||||
experts = object.__new__(AiterMxfp8Experts) # bypass heavy __init__
|
experts = object.__new__(AiterMxfp8Experts) # bypass heavy __init__
|
||||||
experts.moe_config = SimpleNamespace(
|
|
||||||
rocm_aiter_fmoe_enabled=rocm_aiter_fmoe_enabled
|
|
||||||
)
|
|
||||||
experts.quant_config = SimpleNamespace(gemm1_clamp_limit=None)
|
experts.quant_config = SimpleNamespace(gemm1_clamp_limit=None)
|
||||||
experts.w1_scale_val = None
|
experts.w1_scale_val = None
|
||||||
experts.w2_scale_val = None
|
experts.w2_scale_val = None
|
||||||
@@ -474,7 +471,7 @@ def _capture_expert_mask(expert_map, *, rocm_aiter_fmoe_enabled, global_num_expe
|
|||||||
topk_ids=ti,
|
topk_ids=ti,
|
||||||
activation=None,
|
activation=None,
|
||||||
global_num_experts=global_num_experts,
|
global_num_experts=global_num_experts,
|
||||||
expert_map=expert_map,
|
expert_map=expert_mask,
|
||||||
a1q_scale=None,
|
a1q_scale=None,
|
||||||
a2_scale=None,
|
a2_scale=None,
|
||||||
workspace13=None,
|
workspace13=None,
|
||||||
@@ -486,33 +483,45 @@ def _capture_expert_mask(expert_map, *, rocm_aiter_fmoe_enabled, global_num_expe
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||||
def test_aiter_mxfp8_ep_expert_mask_both_master_modes():
|
def test_aiter_mxfp8_apply_forwards_expert_mask_unchanged():
|
||||||
"""Both aiter-master forms must yield the SAME correct 0/1 aiter mask;
|
"""AiterMxfp8Experts.apply forwards the precomputed 0/1 mask to aiter as-is
|
||||||
guards the EP+master regression (mask must not collapse to all-ones)."""
|
(no re-derivation that would collapse an already-0/1 mask to all-ones)."""
|
||||||
from vllm.model_executor.layers.fused_moe.expert_map_manager import (
|
# 0/1 local-expert mask over global ids + trailing sentinel (rank owns 0..3).
|
||||||
determine_expert_map,
|
ep_mask = torch.tensor(
|
||||||
|
[1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32, device=DEVICE
|
||||||
)
|
)
|
||||||
|
got = _capture_expert_mask(ep_mask, global_num_experts=8)
|
||||||
|
assert torch.equal(got.cpu().to(torch.int32), ep_mask.cpu().to(torch.int32))
|
||||||
|
assert got.sum().item() == 4 # the 4 local experts, not all 9
|
||||||
|
|
||||||
E, ep_size, ep_rank = 8, 2, 0 # rank owns global experts 0..3
|
|
||||||
# master OFF: vLLM's -1 index map
|
|
||||||
_, idx_map, _ = determine_expert_map(ep_size, ep_rank, E, return_expert_mask=False)
|
|
||||||
# master ON: 0/1 mask (+ trailing sentinel) that RoutedExperts forwards
|
|
||||||
_, _, ep_mask = determine_expert_map(ep_size, ep_rank, E, return_expert_mask=True)
|
|
||||||
|
|
||||||
idx_map = idx_map.to(DEVICE)
|
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||||
ep_mask = ep_mask.to(DEVICE)
|
def test_routed_experts_expert_map_delegates_to_kernel():
|
||||||
|
"""RoutedExperts.expert_map returns the 0/1 mask only for kernels that set
|
||||||
|
``consumes_expert_mask`` (AITER), and the canonical -1 map otherwise -- keyed
|
||||||
|
on the resolved kernel, not the global aiter switch."""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
# Expected aiter expert_mask: 0/1 over global ids + trailing sentinel slot.
|
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||||
expected = torch.tensor([1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32)
|
|
||||||
|
|
||||||
got_off = _capture_expert_mask(
|
canonical = torch.tensor([0, 1, 2, 3, -1, -1, -1, -1], dtype=torch.int32)
|
||||||
idx_map, rocm_aiter_fmoe_enabled=False, global_num_experts=E
|
mask = torch.tensor([1, 1, 1, 1, 0, 0, 0, 0, 0], dtype=torch.int32)
|
||||||
)
|
|
||||||
got_on = _capture_expert_mask(
|
|
||||||
ep_mask, rocm_aiter_fmoe_enabled=True, global_num_experts=E
|
|
||||||
)
|
|
||||||
|
|
||||||
assert torch.equal(got_off.cpu().to(torch.int32), expected)
|
def resolve(consumes_mask, *, has_moe_kernel=True):
|
||||||
# master ON forwards the prebuilt mask unchanged (NOT collapsed to all-ones)
|
moe_kernel = (
|
||||||
assert torch.equal(got_on.cpu().to(torch.int32), ep_mask.cpu().to(torch.int32))
|
SimpleNamespace(
|
||||||
assert got_on.sum().item() == 4 # exactly the 4 local experts, not all 9
|
fused_experts=SimpleNamespace(consumes_expert_mask=consumes_mask)
|
||||||
|
)
|
||||||
|
if has_moe_kernel
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
layer = SimpleNamespace(
|
||||||
|
_expert_map=canonical,
|
||||||
|
expert_mask=mask,
|
||||||
|
quant_method=SimpleNamespace(moe_kernel=moe_kernel),
|
||||||
|
)
|
||||||
|
return RoutedExperts.expert_map.fget(layer)
|
||||||
|
|
||||||
|
assert torch.equal(resolve(True), mask) # AITER kernel -> 0/1 mask
|
||||||
|
assert torch.equal(resolve(False), canonical) # non-AITER -> canonical map
|
||||||
|
assert torch.equal(resolve(False, has_moe_kernel=False), canonical) # non-modular
|
||||||
|
|||||||
@@ -62,6 +62,8 @@ def is_aiter_mxfp8_moe_available() -> bool:
|
|||||||
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
||||||
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
|
"""MXFP8 MoE through AITER's FlyDSL two-stage grouped GEMM (gfx950)."""
|
||||||
|
|
||||||
|
consumes_expert_mask = True
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def quant_dtype(self) -> torch.dtype | str | None:
|
def quant_dtype(self) -> torch.dtype | str | None:
|
||||||
return self.quant_config.quant_dtype
|
return self.quant_config.quant_dtype
|
||||||
@@ -148,21 +150,9 @@ class AiterMxfp8Experts(Mxfp8TritonExpertsBase):
|
|||||||
limit = self.quant_config.gemm1_clamp_limit
|
limit = self.quant_config.gemm1_clamp_limit
|
||||||
swiglu_limit = 0.0 if limit is None else float(limit)
|
swiglu_limit = 0.0 if limit is None else float(limit)
|
||||||
|
|
||||||
# Under EP, aiter expects ``expert_mask``: a 0/1 *local-expert* mask over
|
# RoutedExperts.expert_map hands AITER experts the precomputed 0/1
|
||||||
# global ids with a trailing fake-expert sentinel slot (shape
|
# expert_mask (with trailing sentinel) instead of the vLLM expert_map.
|
||||||
# ``[global_num_experts + 1]``), from which it derives the global->local
|
expert_mask = expert_map
|
||||||
# compaction. What ``RoutedExperts.expert_map`` hands us depends on the
|
|
||||||
# aiter master switch (``rocm_aiter_fmoe_enabled``).
|
|
||||||
# Branching on the (static) master flag — not the tensor contents —
|
|
||||||
# keeps this HIP-graph/torch.compile safe (no data-dependent sync).
|
|
||||||
# ``None`` under pure TP.
|
|
||||||
if expert_map is None:
|
|
||||||
expert_mask = None
|
|
||||||
elif self.moe_config.rocm_aiter_fmoe_enabled:
|
|
||||||
expert_mask = expert_map
|
|
||||||
else:
|
|
||||||
local_mask = (expert_map >= 0).to(torch.int32)
|
|
||||||
expert_mask = torch.cat([local_mask, local_mask.new_zeros(1)])
|
|
||||||
|
|
||||||
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
|
# Route through the graph-safe ``rocm_aiter_fused_moe`` custom op so the
|
||||||
# call is captured under HIP graphs / torch.compile (a direct
|
# call is captured under HIP graphs / torch.compile (a direct
|
||||||
|
|||||||
@@ -240,7 +240,7 @@ def rocm_aiter_fused_experts(
|
|||||||
moe_config: FusedMoEConfig,
|
moe_config: FusedMoEConfig,
|
||||||
activation: MoEActivation = MoEActivation.SILU,
|
activation: MoEActivation = MoEActivation.SILU,
|
||||||
apply_router_weight_on_input: bool = False,
|
apply_router_weight_on_input: bool = False,
|
||||||
expert_map: torch.Tensor | None = None,
|
expert_mask: torch.Tensor | None = None,
|
||||||
quant_config: FusedMoEQuantConfig | None = None,
|
quant_config: FusedMoEQuantConfig | None = None,
|
||||||
a1q_scale: torch.Tensor | None = None,
|
a1q_scale: torch.Tensor | None = None,
|
||||||
num_local_tokens: torch.Tensor | None = None,
|
num_local_tokens: torch.Tensor | None = None,
|
||||||
@@ -271,8 +271,6 @@ def rocm_aiter_fused_experts(
|
|||||||
topk_weights = topk_weights.to(torch.float32)
|
topk_weights = topk_weights.to(torch.float32)
|
||||||
topk_ids = topk_ids.to(torch.int32)
|
topk_ids = topk_ids.to(torch.int32)
|
||||||
|
|
||||||
expert_mask = expert_map if expert_map is not None else None
|
|
||||||
|
|
||||||
# w8a8 per-channel quantization
|
# w8a8 per-channel quantization
|
||||||
if (
|
if (
|
||||||
quant_config.per_act_token_quant
|
quant_config.per_act_token_quant
|
||||||
@@ -420,6 +418,8 @@ def rocm_aiter_fused_experts(
|
|||||||
|
|
||||||
|
|
||||||
class AiterExperts(mk.FusedMoEExpertsModular):
|
class AiterExperts(mk.FusedMoEExpertsModular):
|
||||||
|
consumes_expert_mask = True
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def expects_unquantized_inputs(self) -> bool:
|
def expects_unquantized_inputs(self) -> bool:
|
||||||
# When paired with MoRI, the prepare/finalize handles FP8
|
# When paired with MoRI, the prepare/finalize handles FP8
|
||||||
@@ -550,7 +550,7 @@ class AiterExperts(mk.FusedMoEExpertsModular):
|
|||||||
topk_ids=topk_ids,
|
topk_ids=topk_ids,
|
||||||
activation=activation,
|
activation=activation,
|
||||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||||
expert_map=expert_map,
|
expert_mask=expert_map,
|
||||||
quant_config=self.quant_config,
|
quant_config=self.quant_config,
|
||||||
moe_config=self.moe_config,
|
moe_config=self.moe_config,
|
||||||
a1q_scale=a1q_scale,
|
a1q_scale=a1q_scale,
|
||||||
|
|||||||
@@ -470,6 +470,12 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
|
|||||||
|
|
||||||
# TODO: add supported activations method (return string)
|
# TODO: add supported activations method (return string)
|
||||||
class FusedMoEExperts(ABC):
|
class FusedMoEExperts(ABC):
|
||||||
|
# ROCm AITER kernels consume a 0/1 local-expert mask (with a trailing
|
||||||
|
# sentinel slot); every other backend consumes the canonical -1/local-slot
|
||||||
|
# expert_map. RoutedExperts.expert_map reads this flag to pick which to hand
|
||||||
|
# the active experts kernel.
|
||||||
|
consumes_expert_mask: bool = False
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
moe_config: FusedMoEConfig,
|
moe_config: FusedMoEConfig,
|
||||||
|
|||||||
@@ -226,9 +226,15 @@ class RoutedExperts(PluggableLayer):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def expert_map(self) -> torch.Tensor | None:
|
def expert_map(self) -> torch.Tensor | None:
|
||||||
return (
|
# AITER fused-MoE kernels consume the 0/1 expert_mask; every other
|
||||||
self._expert_map if not self.rocm_aiter_fmoe_enabled else self.expert_mask
|
# backend consumes the canonical -1/local-slot map. Ask the active
|
||||||
)
|
# experts kernel which it wants (only AITER sets consumes_expert_mask)
|
||||||
|
# rather than keying on the global VLLM_ROCM_USE_AITER switch, which
|
||||||
|
# does not reflect the per-layer kernel actually selected.
|
||||||
|
moe_kernel = getattr(self.quant_method, "moe_kernel", None)
|
||||||
|
if moe_kernel is not None and moe_kernel.fused_experts.consumes_expert_mask:
|
||||||
|
return self.expert_mask
|
||||||
|
return self._expert_map
|
||||||
|
|
||||||
def update_expert_map_info(self):
|
def update_expert_map_info(self):
|
||||||
# Update local attributes from ExpertMapManager
|
# Update local attributes from ExpertMapManager
|
||||||
|
|||||||
@@ -1026,7 +1026,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
|||||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||||
quant_config=self.moe_quant_config,
|
quant_config=self.moe_quant_config,
|
||||||
moe_config=layer.moe_config,
|
moe_config=layer.moe_config,
|
||||||
expert_map=layer.expert_map,
|
expert_mask=layer.expert_mask,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user