[Bugfix][MoE] Apply expert_map remapping for CUTLASS FP4 MOE expert parallel (#59042)
Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
remap_topk_to_local,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -412,10 +413,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# CutlassExpertsFp8 does not support expert map, which is
|
||||
# needed for STANDARD activation format kernels in DP/EP mode.
|
||||
# Note that the BATCHED activation format does not use
|
||||
# the expert map for identifying experts.
|
||||
# EP is supported only with allgather_reducescatter (expert_map remapping).
|
||||
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
|
||||
# post-dispatch layouts for Standard-format grouped GEMM.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
@@ -491,6 +491,15 @@ FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max()
|
||||
FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max
|
||||
|
||||
|
||||
def _prepare_topk_ids_for_cutlass_fp4_moe(
|
||||
topk_ids: torch.Tensor,
|
||||
expert_map: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
if expert_map is None:
|
||||
return topk_ids
|
||||
return remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
|
||||
|
||||
|
||||
def run_cutlass_moe_fp4(
|
||||
output: torch.Tensor,
|
||||
a: torch.Tensor,
|
||||
@@ -513,6 +522,7 @@ def run_cutlass_moe_fp4(
|
||||
e: int,
|
||||
device: torch.device,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
*,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> None:
|
||||
@@ -543,6 +553,8 @@ def run_cutlass_moe_fp4(
|
||||
|
||||
assumes that topk < k < n to satisfy - up/down projection expectations.
|
||||
"""
|
||||
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
|
||||
|
||||
is_gated = activation.is_gated
|
||||
# For gated activations (e.g. SiLU), w1 output is 2*n (gate + up).
|
||||
# For non-gated activations (e.g. SiLU_NO_MUL), w1 output is n (up only).
|
||||
@@ -729,9 +741,14 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
# CutlassExpertsFp4 does not support expert map, which is
|
||||
# needed for STANDARD activation format kernels in EP mode.
|
||||
return moe_parallel_config.ep_size == 1
|
||||
# EP is supported only with allgather_reducescatter (expert_map remapping).
|
||||
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
|
||||
# post-dispatch layouts for Standard-format grouped GEMM.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_batch_invariance() -> bool:
|
||||
@@ -806,6 +823,7 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular):
|
||||
e=e,
|
||||
device=hidden_states.device,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
expert_map=expert_map,
|
||||
activation_config=self.activation_config,
|
||||
)
|
||||
|
||||
@@ -828,10 +846,13 @@ def run_cutlass_moe_mxfp4(
|
||||
e: int,
|
||||
device: torch.device,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
*,
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
) -> None:
|
||||
"""MXFP4 x MXFP4 MoE implementation using CUTLASS grouped GEMM."""
|
||||
topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map)
|
||||
|
||||
is_gated = activation.is_gated
|
||||
w1_n = n * 2 if is_gated else n
|
||||
|
||||
@@ -1036,7 +1057,14 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> bool:
|
||||
return moe_parallel_config.ep_size == 1
|
||||
# EP is supported only with allgather_reducescatter (expert_map remapping).
|
||||
# DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible
|
||||
# post-dispatch layouts for Standard-format grouped GEMM.
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_deepep_ht_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
@@ -1103,6 +1131,7 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular):
|
||||
e=e,
|
||||
device=hidden_states.device,
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
expert_map=expert_map,
|
||||
activation_config=self.activation_config,
|
||||
)
|
||||
|
||||
|
||||
@@ -21,7 +21,10 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import (
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
remap_topk_to_local,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
@@ -994,49 +997,6 @@ def masked_moe_sum(
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _remap_topk_to_local_kernel(
|
||||
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
|
||||
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
|
||||
out_ptr, # [n] int64 local expert IDs (-1 for invalid/non-local)
|
||||
n_elements,
|
||||
BLOCK: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < n_elements
|
||||
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
|
||||
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
|
||||
# rows don't read OOB, then select -1 for them. Matches
|
||||
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
|
||||
# plain expert_map[-1] would wrap to a valid local id and misroute).
|
||||
valid = tid >= 0
|
||||
idx = tl.where(valid, tid, 0)
|
||||
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
|
||||
out = tl.where(valid, local.to(tl.int64), -1)
|
||||
tl.store(out_ptr + offs, out, mask=mask)
|
||||
|
||||
|
||||
def remap_topk_to_local(
|
||||
topk_ids: torch.Tensor, expert_map: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""Fused global->local expert-id mapping over a topk_ids tensor, preserving -1.
|
||||
|
||||
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
|
||||
with one kernel. Returns a NEW int64 tensor -- the caller keeps the original
|
||||
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
|
||||
|
||||
(Distinct from ``deep_gemm_utils.apply_expert_map``, which is a scalar
|
||||
``@triton.jit`` device helper called from within other kernels.)
|
||||
"""
|
||||
out = torch.empty_like(topk_ids, dtype=torch.int64)
|
||||
n = topk_ids.numel()
|
||||
BLOCK = 1024
|
||||
grid = (triton.cdiv(n, BLOCK),)
|
||||
_remap_topk_to_local_kernel[grid](topk_ids, expert_map, out, n, BLOCK=BLOCK)
|
||||
return out
|
||||
|
||||
|
||||
class BaseOAITritonExperts(mk.FusedMoEExpertsModular):
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
|
||||
@@ -2133,6 +2133,59 @@ def make_mxfp4_moe_quant_config(
|
||||
)
|
||||
|
||||
|
||||
_MXFP4_SCHEME_CANDIDATES: list[tuple[QuantKey, QuantKey | None]] = [
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
(kMxfp4Static, kMxfp8Dynamic),
|
||||
(kMxfp4Static, kFp8Dynamic128Sym),
|
||||
(kMxfp4Static, kFp8StaticTensorSym),
|
||||
(kMxfp4Static, None),
|
||||
]
|
||||
|
||||
|
||||
def _mxfp4_quant_keys_for_experts(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
) -> tuple[QuantKey | None, QuantKey | None]:
|
||||
for weight_key, activation_key in _MXFP4_SCHEME_CANDIDATES:
|
||||
if experts_cls._supports_quant_scheme(weight_key, activation_key):
|
||||
return weight_key, activation_key
|
||||
return kMxfp4Static, _backend_activation_key(mxfp4_backend)
|
||||
|
||||
|
||||
def _activation_format_for_config(
|
||||
moe_config: FusedMoEConfig,
|
||||
) -> mk.FusedMoEActivationFormat:
|
||||
return (
|
||||
mk.FusedMoEActivationFormat.BatchedExperts
|
||||
if moe_config.moe_parallel_config.use_batched_activation_format
|
||||
else mk.FusedMoEActivationFormat.Standard
|
||||
)
|
||||
|
||||
|
||||
def _validate_mxfp4_experts_config(
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
mxfp4_backend: Mxfp4MoeBackend,
|
||||
) -> None:
|
||||
"""Fail closed when a caller bypasses the oracle and picks experts directly."""
|
||||
activation_format = _activation_format_for_config(moe_config)
|
||||
weight_key, activation_key = _mxfp4_quant_keys_for_experts(
|
||||
experts_cls, mxfp4_backend
|
||||
)
|
||||
supported, reason = experts_cls.is_supported_config(
|
||||
experts_cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
if not supported:
|
||||
raise ValueError(
|
||||
f"{experts_cls.__name__} does not support this MoE deployment"
|
||||
f"{f' since {reason}' if reason else ''}."
|
||||
)
|
||||
|
||||
|
||||
def make_mxfp4_moe_kernel(
|
||||
moe_quant_config: FusedMoEQuantConfig,
|
||||
moe_config: FusedMoEConfig,
|
||||
@@ -2141,6 +2194,8 @@ def make_mxfp4_moe_kernel(
|
||||
routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None,
|
||||
) -> mk.FusedMoEKernel:
|
||||
"""Create a FusedMoEKernel for the given MXFP4 backend."""
|
||||
_validate_mxfp4_experts_config(experts_cls, moe_config, mxfp4_backend)
|
||||
|
||||
is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic)
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
|
||||
@@ -235,6 +235,73 @@ def count_expert_num_tokens(
|
||||
return expert_num_tokens
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _remap_topk_to_local_kernel(
|
||||
topk_ids_ptr, # [n] global expert IDs (-1 = invalid)
|
||||
expert_map_ptr, # [num_experts] global->local (-1 for non-local)
|
||||
out_ptr, # [n] local expert IDs (-1 for invalid/non-local)
|
||||
n_elements,
|
||||
BLOCK: tl.constexpr,
|
||||
OUT_INT32: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(0)
|
||||
offs = pid * BLOCK + tl.arange(0, BLOCK)
|
||||
mask = offs < n_elements
|
||||
tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1)
|
||||
# Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid
|
||||
# rows don't read OOB, then select -1 for them. Matches
|
||||
# torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a
|
||||
# plain expert_map[-1] would wrap to a valid local id and misroute).
|
||||
valid = tid >= 0
|
||||
idx = tl.where(valid, tid, 0)
|
||||
local = tl.load(expert_map_ptr + idx, mask=mask, other=-1)
|
||||
if OUT_INT32:
|
||||
out = tl.where(valid, local, -1)
|
||||
else:
|
||||
out = tl.where(valid, local.to(tl.int64), -1)
|
||||
tl.store(out_ptr + offs, out, mask=mask)
|
||||
|
||||
|
||||
def remap_topk_to_local(
|
||||
topk_ids: torch.Tensor,
|
||||
expert_map: torch.Tensor,
|
||||
*,
|
||||
out_dtype: torch.dtype = torch.int64,
|
||||
) -> torch.Tensor:
|
||||
"""Map global expert ids to local ids; non-local or invalid routes become -1.
|
||||
|
||||
Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)``
|
||||
with one kernel. Returns a NEW tensor -- the caller keeps the original
|
||||
``topk_ids`` as ``global_topk_ids``, so this must not write in place.
|
||||
|
||||
Args:
|
||||
topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous;
|
||||
the Triton kernel indexes by flat ``numel()`` offsets.
|
||||
expert_map: Global-to-local mapping; -1 marks non-local experts.
|
||||
out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``.
|
||||
|
||||
Returns:
|
||||
Remapped local expert ids with the same shape as ``topk_ids``.
|
||||
|
||||
"""
|
||||
if out_dtype not in (torch.int32, torch.int64):
|
||||
raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}")
|
||||
assert topk_ids.is_contiguous()
|
||||
out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device)
|
||||
n = topk_ids.numel()
|
||||
BLOCK = 1024
|
||||
grid = (triton.cdiv(n, BLOCK),)
|
||||
_remap_topk_to_local_kernel[grid](
|
||||
topk_ids,
|
||||
expert_map,
|
||||
out,
|
||||
n,
|
||||
BLOCK=BLOCK,
|
||||
OUT_INT32=out_dtype == torch.int32,
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor:
|
||||
"""Shrink the given tensor and apply the given view to it. This is
|
||||
used to resize the intermediate fused_moe caches.
|
||||
|
||||
Reference in New Issue
Block a user