[Bugfix][MoE] Apply expert_map remapping for CUTLASS FP4 MOE expert parallel (#59042)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:02:56 -04:00
committed by GitHub
parent f022975964
commit b3ee560ae8
6 changed files with 470 additions and 59 deletions
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
)
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
@@ -412,10 +413,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# CutlassExpertsFp8 does not support expert map, which is
# needed for STANDARD activation format kernels in DP/EP mode.
# Note that the BATCHED activation format does not use
# the expert map for identifying experts.
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
@@ -491,6 +491,15 @@ FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
def _prepare_topk_ids_for_cutlass_fp4_moe(
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> torch.Tensor:
if expert_map is None:
return topk_ids
return remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
def run_cutlass_moe_fp4(
output: torch.Tensor,
a: torch.Tensor,
@@ -513,6 +522,7 @@ def run_cutlass_moe_fp4(
e: int,
device: torch.device,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
@@ -543,6 +553,8 @@ def run_cutlass_moe_fp4(
assumes that topk < k < n to satisfy - up/down projection expectations.
"""
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
is_gated = activation.is_gated
# For gated activations (e.g. SiLU), w1 output is 2*n (gate + up).
# For non-gated activations (e.g. SiLU_NO_MUL), w1 output is n (up only).
@@ -729,9 +741,14 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
# CutlassExpertsFp4 does not support expert map, which is
# needed for STANDARD activation format kernels in EP mode.
return moe_parallel_config.ep_size == 1
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
@staticmethod
def _supports_batch_invariance() -> bool:
@@ -806,6 +823,7 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
e=e,
device=hidden_states.device,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
activation_config=self.activation_config,
)
@@ -828,10 +846,13 @@ def run_cutlass_moe_mxfp4(
e: int,
device: torch.device,
apply_router_weight_on_input: bool = False,
expert_map: torch.Tensor | None = None,
*,
activation_config: ApplyMoEActivationConfig | None = None,
) -> None:
"""MXFP4 x MXFP4 MoE implementation using CUTLASS grouped GEMM."""
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
is_gated = activation.is_gated
w1_n = n * 2 if is_gated else n
@@ -1036,7 +1057,14 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return moe_parallel_config.ep_size == 1
# EP is supported only with allgather_reducescatter (expert_map remapping).
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
# post-dispatch layouts for Standard-format grouped GEMM.
return not (
moe_parallel_config.use_fi_nvl_two_sided_kernels
or moe_parallel_config.use_deepep_ht_kernels
or moe_parallel_config.use_fi_nvl_one_sided_kernels
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
@@ -1103,6 +1131,7 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
e=e,
device=hidden_states.device,
apply_router_weight_on_input=apply_router_weight_on_input,
expert_map=expert_map,
activation_config=self.activation_config,
)
@@ -21,7 +21,10 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
@@ -994,49 +997,6 @@ def masked_moe_sum(
)
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] int64 local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor, expert_map: torch.Tensor
) -> torch.Tensor:
"""Fused global->local expert-id mapping over a topk_ids tensor, preserving -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW int64 tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
(Distinct from ``deep_gemm_utils.apply_expert_map``, which is a scalar
``@triton.jit`` device helper called from within other kernels.)
"""
out = torch.empty_like(topk_ids, dtype=torch.int64)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](topk_ids, expert_map, out, n, BLOCK=BLOCK)
return out
class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
@property
def expects_unquantized_inputs(self) -> bool:
@@ -2133,6 +2133,59 @@ def make_mxfp4_moe_quant_config(
)
_MXFP4_SCHEME_CANDIDATES: list[tuple[QuantKey, QuantKey | None]] = [
(kMxfp4Static, kMxfp4Dynamic),
(kMxfp4Static, kMxfp8Dynamic),
(kMxfp4Static, kFp8Dynamic128Sym),
(kMxfp4Static, kFp8StaticTensorSym),
(kMxfp4Static, None),
]
def _mxfp4_quant_keys_for_experts(
experts_cls: type[mk.FusedMoEExperts],
mxfp4_backend: Mxfp4MoeBackend,
) -> tuple[QuantKey | None, QuantKey | None]:
for weight_key, activation_key in _MXFP4_SCHEME_CANDIDATES:
if experts_cls._supports_quant_scheme(weight_key, activation_key):
return weight_key, activation_key
return kMxfp4Static, _backend_activation_key(mxfp4_backend)
def _activation_format_for_config(
moe_config: FusedMoEConfig,
) -> mk.FusedMoEActivationFormat:
return (
mk.FusedMoEActivationFormat.BatchedExperts
if moe_config.moe_parallel_config.use_batched_activation_format
else mk.FusedMoEActivationFormat.Standard
)
def _validate_mxfp4_experts_config(
experts_cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
mxfp4_backend: Mxfp4MoeBackend,
) -> None:
"""Fail closed when a caller bypasses the oracle and picks experts directly."""
activation_format = _activation_format_for_config(moe_config)
weight_key, activation_key = _mxfp4_quant_keys_for_experts(
experts_cls, mxfp4_backend
)
supported, reason = experts_cls.is_supported_config(
experts_cls,
moe_config,
weight_key,
activation_key,
activation_format,
)
if not supported:
raise ValueError(
f"{experts_cls.__name__} does not support this MoE deployment"
f"{f' since {reason}' if reason else ''}."
)
def make_mxfp4_moe_kernel(
moe_quant_config: FusedMoEQuantConfig,
moe_config: FusedMoEConfig,
@@ -2141,6 +2194,8 @@ def make_mxfp4_moe_kernel(
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
) -> mk.FusedMoEKernel:
"""Create a FusedMoEKernel for the given MXFP4 backend."""
_validate_mxfp4_experts_config(experts_cls, moe_config, mxfp4_backend)
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
prepare_finalize = maybe_make_prepare_finalize(
@@ -235,6 +235,73 @@ def count_expert_num_tokens(
return expert_num_tokens
@triton.jit
def _remap_topk_to_local_kernel(
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
out_ptr, # [n] local expert IDs (-1 for invalid/non-local)
n_elements,
BLOCK: tl.constexpr,
OUT_INT32: tl.constexpr,
):
pid = tl.program_id(0)
offs = pid * BLOCK + tl.arange(0, BLOCK)
mask = offs < n_elements
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
# rows don't read OOB, then select -1 for them. Matches
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
# plain expert_map[-1] would wrap to a valid local id and misroute).
valid = tid >= 0
idx = tl.where(valid, tid, 0)
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
if OUT_INT32:
out = tl.where(valid, local, -1)
else:
out = tl.where(valid, local.to(tl.int64), -1)
tl.store(out_ptr + offs, out, mask=mask)
def remap_topk_to_local(
topk_ids: torch.Tensor,
expert_map: torch.Tensor,
*,
out_dtype: torch.dtype = torch.int64,
) -> torch.Tensor:
"""Map global expert ids to local ids; non-local or invalid routes become -1.
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
with one kernel. Returns a NEW tensor -- the caller keeps the original
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
Args:
topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous;
the Triton kernel indexes by flat ``numel()`` offsets.
expert_map: Global-to-local mapping; -1 marks non-local experts.
out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``.
Returns:
Remapped local expert ids with the same shape as ``topk_ids``.
"""
if out_dtype not in (torch.int32, torch.int64):
raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}")
assert topk_ids.is_contiguous()
out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device)
n = topk_ids.numel()
BLOCK = 1024
grid = (triton.cdiv(n, BLOCK),)
_remap_topk_to_local_kernel[grid](
topk_ids,
expert_map,
out,
n,
BLOCK=BLOCK,
OUT_INT32=out_dtype == torch.int32,
)
return out
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
"""Shrink the given tensor and apply the given view to it. This is
used to resize the intermediate fused_moe caches.