diff --git a/tests/kernels/moe/test_cutlass_moe.py b/tests/kernels/moe/test_cutlass_moe.py index 050e1bc104..a06abfbc1f 100644 --- a/tests/kernels/moe/test_cutlass_moe.py +++ b/tests/kernels/moe/test_cutlass_moe.py @@ -9,7 +9,7 @@ import pytest import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk -from tests.kernels.moe.utils import make_dummy_moe_config +from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_config from vllm import _custom_ops as ops from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config from vllm.model_executor.layers.fused_moe import ( @@ -26,8 +26,10 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( ) from vllm.model_executor.layers.fused_moe.config import ( FUSED_MOE_UNQUANTIZED_CONFIG, + FusedMoEParallelConfig, FusedMoEQuantConfig, fp8_w8a8_moe_quant_config, + mxfp4_moe_quant_config, ) from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsFp4, @@ -41,7 +43,10 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( ) from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle -from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input +from vllm.model_executor.layers.fused_moe.utils import ( + moe_kernel_quantize_input, + remap_topk_to_local, +) from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed @@ -183,6 +188,189 @@ def test_cutlass_moe_permutation_maps_padding_to_zero(): torch.testing.assert_close(restored[3], torch.zeros_like(restored[3])) +def test_remap_topk_ids_to_local(): + device = "cuda" + expert_map = torch.tensor([-1, 0, -1, 1, -1, 2], dtype=torch.int32, device=device) + topk_ids = torch.tensor([[1, 3], [0, 5], [-1, 4]], dtype=torch.int32, device=device) + expected = torch.tensor( + [[0, 1], [-1, 2], [-1, -1]], dtype=torch.int32, device=device + ) + actual = remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32) + torch.testing.assert_close(actual, expected) + + +def test_remap_topk_ids_to_local_requires_contiguous_topk_ids(): + expert_map = torch.tensor([0, 1], dtype=torch.int32) + topk_ids = torch.tensor([[0, 1], [1, 0]], dtype=torch.int32).t() + assert not topk_ids.is_contiguous() + with pytest.raises(AssertionError): + remap_topk_to_local(topk_ids, expert_map) + + +def test_cutlass_fp4_moe_supports_expert_parallel_config(): + ep_config = FusedMoEParallelConfig( + tp_size=1, + tp_rank=0, + pcp_size=1, + pcp_rank=0, + dp_size=8, + dp_rank=0, + ep_size=8, + ep_rank=0, + sp_size=1, + use_ep=True, + all2all_backend="allgather_reducescatter", + enable_eplb=False, + ) + assert CutlassExpertsMxfp4._supports_parallel_config(ep_config) + assert CutlassExpertsFp4._supports_parallel_config(ep_config) + + +def _slice_moe_quant_config( + quant_config: FusedMoEQuantConfig, start: int, end: int +) -> FusedMoEQuantConfig: + num_experts = quant_config.w1_scale.shape[0] + sliced_config = copy.deepcopy(quant_config) + for desc_name in ("_a1", "_a2", "_w1", "_w2"): + desc = getattr(sliced_config, desc_name) + for field_name in ("scale", "alpha_or_gscale", "bias", "zp"): + tensor = getattr(desc, field_name) + if ( + isinstance(tensor, torch.Tensor) + and tensor.ndim > 0 + and tensor.shape[0] == num_experts + ): + setattr(desc, field_name, tensor[start:end]) + return sliced_config + + +def _swizzle_mxfp4_moe_scales( + w1_scale: torch.Tensor, + w2_scale: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: + e, w1_n, w1_scale_k = w1_scale.shape + w1_k = w1_scale_k * 32 + w2_m, w2_scale_n = w2_scale.shape[1], w2_scale.shape[2] + w2_n = w2_scale_n * 32 + + swizzled_w1 = [] + swizzled_w2 = [] + for expert in range(e): + swizzled_w1.append( + swizzle_mxfp4_scales(w1_scale[expert], w1_n, w1_k).reshape(w1_n, w1_scale_k) + ) + swizzled_w2.append( + swizzle_mxfp4_scales(w2_scale[expert], w2_m, w2_n).reshape(w2_m, w2_scale_n) + ) + return torch.stack(swizzled_w1), torch.stack(swizzled_w2) + + +def _make_cutlass_fp4_moe_weights( + quantization: str, + num_experts: int, + n: int, + k: int, + dtype: torch.dtype, + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]: + if quantization == "nvfp4": + return make_test_quant_config( + num_experts, + n, + k, + in_dtype=dtype, + quant_dtype="nvfp4", + block_shape=None, + per_act_token_quant=False, + make_gate=True, + ) + + from vllm.model_executor.layers.quantization.utils.mxfp4_utils import ( + mxfp4_quantize, + ) + + w1, w1_scale = mxfp4_quantize( + torch.randn((num_experts, 2 * n, k), device=device, dtype=dtype) / 8 + ) + w2, w2_scale = mxfp4_quantize( + torch.randn((num_experts, k, n), device=device, dtype=dtype) / 8 + ) + w1_scale, w2_scale = _swizzle_mxfp4_moe_scales(w1_scale, w2_scale) + return ( + w1.contiguous(), + w2.contiguous(), + mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale), + ) + + +@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"]) +@torch.inference_mode() +def test_cutlass_fp4_moe_EP( + quantization: str, + workspace_init, +): + """Sharded CUTLASS FP4/MXFP4 MoE with expert_map must match full weights.""" + experts_cls = CutlassExpertsFp4 if quantization == "nvfp4" else CutlassExpertsMxfp4 + if not experts_cls._supports_current_device(): + pytest.skip(f"CUTLASS {quantization} MoE is not supported on this GPU") + + set_random_seed(7) + m, n, k = 8, 128, 128 + num_experts, num_local_experts, topk = 8, 2, 2 + dtype = torch.bfloat16 + device = torch.device("cuda") + + w1, w2, quant_config = _make_cutlass_fp4_moe_weights( + quantization, num_experts, n, k, dtype, device + ) + hidden_states = torch.randn((m, k), device=device, dtype=dtype) / 10 + score = torch.randn((m, num_experts), device=device, dtype=dtype) + topk_weights, topk_ids, _ = fused_topk( + hidden_states, score, topk, renormalize=False + ) + + moe_config = make_dummy_moe_config( + num_experts=num_experts, + num_local_experts=num_experts, + experts_per_token=topk, + hidden_dim=k, + intermediate_size=n, + in_dtype=dtype, + max_num_tokens=m, + ) + kernel = mk.FusedMoEKernel( + maybe_make_prepare_finalize( + moe=moe_config, + quant_config=quant_config, + allow_new_interface=True, + use_monolithic=False, + ), + experts_cls(moe_config=moe_config, quant_config=quant_config), + ) + apply_kwargs = { + "hidden_states": hidden_states, + "w1": w1, + "w2": w2, + "topk_weights": topk_weights, + "topk_ids": topk_ids, + "global_num_experts": num_experts, + "activation": MoEActivation.SILU, + "expert_map": None, + "apply_router_weight_on_input": False, + } + reference = kernel.apply(**apply_kwargs) + sharded = run_with_expert_maps( + num_experts, + num_local_experts, + quant_config, + experts_cls, + **apply_kwargs, + ) + + assert reference.abs().max() > 0 + torch.testing.assert_close(sharded, reference, atol=1e-2, rtol=1e-2) + + @pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"]) @torch.inference_mode() def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization: str): @@ -486,6 +674,7 @@ def run_with_expert_maps( num_experts: int, num_local_experts: int, quant_config: FusedMoEQuantConfig, + experts_cls: type[mk.FusedMoEExperts], **cutlass_moe_kwargs, ): def slice_experts(): @@ -513,9 +702,7 @@ def run_with_expert_maps( for k, t in full_tensors.items(): cutlass_moe_kwargs[k] = t[s:e] - new_quant_config = copy.deepcopy(quant_config) - new_quant_config._w1.scale = quant_config.w1_scale[s:e] - new_quant_config._w2.scale = quant_config.w2_scale[s:e] + new_quant_config = _slice_moe_quant_config(quant_config, s, e) yield cutlass_moe_kwargs, new_quant_config @@ -523,13 +710,14 @@ def run_with_expert_maps( for kwargs, new_quant_config in slice_experts(): w2 = kwargs["w2"] a = kwargs["hidden_states"] + w1 = kwargs["w1"] moe_config = make_dummy_moe_config( max_num_tokens=kwargs.get("hidden_states").shape[0], experts_per_token=kwargs.get("topk_ids").shape[1], num_experts=num_experts, num_local_experts=num_local_experts, hidden_dim=w2.shape[1], - intermediate_size=w2.shape[2], + intermediate_size=w1.shape[1] // 2, in_dtype=a.dtype, ) kernel = mk.FusedMoEKernel( @@ -539,7 +727,7 @@ def run_with_expert_maps( allow_new_interface=True, use_monolithic=False, ), - CutlassExpertsFp8( + experts_cls( moe_config=moe_config, quant_config=new_quant_config, ), @@ -624,6 +812,7 @@ def run_8_bit( num_experts, num_local_experts, quant_config, + CutlassExpertsFp8, **kwargs, ) diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index 13f29087d9..0a8f5c29d0 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -2114,6 +2114,117 @@ def test_aiter_mxfp4_monolithic_rejects_expert_parallel( assert "parallel config" in reason +@pytest.mark.parametrize( + "all2all_backend", + [ + "deepep_high_throughput", + "flashinfer_nvlink_two_sided", + "flashinfer_nvlink_one_sided", + ], +) +def test_make_mxfp4_moe_kernel_rejects_cutlass_unsupported_expert_parallel( + monkeypatch: pytest.MonkeyPatch, + all2all_backend: str, +): + """Compressed-tensors CUTLASS W4A4 bypasses the oracle; guard at assembly.""" + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + mxfp4_moe_quant_config, + ) + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + CutlassExpertsMxfp4, + ) + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + make_mxfp4_moe_kernel, + ) + + monkeypatch.setattr( + CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True) + ) + + moe_config = replace( + make_dummy_moe_config( + num_experts=256, + num_local_experts=32, + hidden_dim=4096, + intermediate_size=2048, + ), + moe_parallel_config=FusedMoEParallelConfig( + tp_size=1, + tp_rank=0, + pcp_size=1, + pcp_rank=0, + dp_size=8, + dp_rank=0, + ep_size=8, + ep_rank=0, + sp_size=1, + use_ep=True, + all2all_backend=all2all_backend, + enable_eplb=False, + ), + ) + w1_scale = torch.empty(32, 64, 128, dtype=torch.uint8) + w2_scale = torch.empty(32, 4096, 64, dtype=torch.uint8) + quant_config = mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale) + + with pytest.raises(ValueError, match="parallel config"): + make_mxfp4_moe_kernel( + quant_config, + moe_config, + CutlassExpertsMxfp4, + Mxfp4MoeBackend.MARLIN, + ) + + +def test_make_mxfp4_moe_kernel_validates_cutlass_expert_parallel( + monkeypatch: pytest.MonkeyPatch, +): + """CUTLASS MXFP4 supports EP with allgather_reducescatter after expert_map.""" + from tests.kernels.moe.utils import make_dummy_moe_config + from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + CutlassExpertsMxfp4, + ) + from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( + Mxfp4MoeBackend, + _validate_mxfp4_experts_config, + ) + + monkeypatch.setattr( + CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True) + ) + + moe_config = replace( + make_dummy_moe_config( + num_experts=256, + num_local_experts=32, + hidden_dim=4096, + intermediate_size=2048, + ), + moe_parallel_config=FusedMoEParallelConfig( + tp_size=1, + tp_rank=0, + pcp_size=1, + pcp_rank=0, + dp_size=8, + dp_rank=0, + ep_size=8, + ep_rank=0, + sp_size=1, + use_ep=True, + all2all_backend="allgather_reducescatter", + enable_eplb=False, + ), + ) + + _validate_mxfp4_experts_config( + CutlassExpertsMxfp4, moe_config, Mxfp4MoeBackend.MARLIN + ) + + # Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that # `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns # the activation untouched, so a name it does not know (e.g. "mxfp6" instead of diff --git a/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py index 6f9a62f282..5c6a96f26d 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py @@ -31,6 +31,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( ) from vllm.model_executor.layers.fused_moe.utils import ( _resize_cache, + remap_topk_to_local, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -412,10 +413,9 @@ class CutlassExpertsFp8(CutlassExpertsFp8Base): @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: - # CutlassExpertsFp8 does not support expert map, which is - # needed for STANDARD activation format kernels in DP/EP mode. - # Note that the BATCHED activation format does not use - # the expert map for identifying experts. + # EP is supported only with allgather_reducescatter (expert_map remapping). + # DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible + # post-dispatch layouts for Standard-format grouped GEMM. return not ( moe_parallel_config.use_fi_nvl_two_sided_kernels or moe_parallel_config.use_deepep_ht_kernels @@ -491,6 +491,15 @@ FLOAT4_E2M1_MAX = scalar_types.float4_e2m1f.max() FLOAT8_E4M3_MAX = torch.finfo(torch.float8_e4m3fn).max +def _prepare_topk_ids_for_cutlass_fp4_moe( + topk_ids: torch.Tensor, + expert_map: torch.Tensor | None, +) -> torch.Tensor: + if expert_map is None: + return topk_ids + return remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32) + + def run_cutlass_moe_fp4( output: torch.Tensor, a: torch.Tensor, @@ -513,6 +522,7 @@ def run_cutlass_moe_fp4( e: int, device: torch.device, apply_router_weight_on_input: bool = False, + expert_map: torch.Tensor | None = None, *, activation_config: ApplyMoEActivationConfig | None = None, ) -> None: @@ -543,6 +553,8 @@ def run_cutlass_moe_fp4( assumes that topk < k < n to satisfy - up/down projection expectations. """ + topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map) + is_gated = activation.is_gated # For gated activations (e.g. SiLU), w1 output is 2*n (gate + up). # For non-gated activations (e.g. SiLU_NO_MUL), w1 output is n (up only). @@ -729,9 +741,14 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular): @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: - # CutlassExpertsFp4 does not support expert map, which is - # needed for STANDARD activation format kernels in EP mode. - return moe_parallel_config.ep_size == 1 + # EP is supported only with allgather_reducescatter (expert_map remapping). + # DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible + # post-dispatch layouts for Standard-format grouped GEMM. + return not ( + moe_parallel_config.use_fi_nvl_two_sided_kernels + or moe_parallel_config.use_deepep_ht_kernels + or moe_parallel_config.use_fi_nvl_one_sided_kernels + ) @staticmethod def _supports_batch_invariance() -> bool: @@ -806,6 +823,7 @@ class CutlassExpertsFp4(mk.FusedMoEExpertsModular): e=e, device=hidden_states.device, apply_router_weight_on_input=apply_router_weight_on_input, + expert_map=expert_map, activation_config=self.activation_config, ) @@ -828,10 +846,13 @@ def run_cutlass_moe_mxfp4( e: int, device: torch.device, apply_router_weight_on_input: bool = False, + expert_map: torch.Tensor | None = None, *, activation_config: ApplyMoEActivationConfig | None = None, ) -> None: """MXFP4 x MXFP4 MoE implementation using CUTLASS grouped GEMM.""" + topk_ids = _prepare_topk_ids_for_cutlass_fp4_moe(topk_ids, expert_map) + is_gated = activation.is_gated w1_n = n * 2 if is_gated else n @@ -1036,7 +1057,14 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular): def _supports_parallel_config( moe_parallel_config: FusedMoEParallelConfig, ) -> bool: - return moe_parallel_config.ep_size == 1 + # EP is supported only with allgather_reducescatter (expert_map remapping). + # DeepEP HT and FlashInfer NVLink prepare/finalize use incompatible + # post-dispatch layouts for Standard-format grouped GEMM. + return not ( + moe_parallel_config.use_fi_nvl_two_sided_kernels + or moe_parallel_config.use_deepep_ht_kernels + or moe_parallel_config.use_fi_nvl_one_sided_kernels + ) @staticmethod def activation_format() -> mk.FusedMoEActivationFormat: @@ -1103,6 +1131,7 @@ class CutlassExpertsMxfp4(mk.FusedMoEExpertsModular): e=e, device=hidden_states.device, apply_router_weight_on_input=apply_router_weight_on_input, + expert_map=expert_map, activation_config=self.activation_config, ) diff --git a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py index e3b6995d69..3e70c15893 100644 --- a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py @@ -21,7 +21,10 @@ from vllm.model_executor.layers.fused_moe.experts.lora_experts_mixin import ( from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceNoOP, ) -from vllm.model_executor.layers.fused_moe.utils import _resize_cache +from vllm.model_executor.layers.fused_moe.utils import ( + _resize_cache, + remap_topk_to_local, +) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kMxfp4Static, @@ -994,49 +997,6 @@ def masked_moe_sum( ) -@triton.jit -def _remap_topk_to_local_kernel( - topk_ids_ptr, # [n] global expert IDs (-1 = invalid) - expert_map_ptr, # [num_experts] global->local (-1 for non-local) - out_ptr, # [n] int64 local expert IDs (-1 for invalid/non-local) - n_elements, - BLOCK: tl.constexpr, -): - pid = tl.program_id(0) - offs = pid * BLOCK + tl.arange(0, BLOCK) - mask = offs < n_elements - tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1) - # Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid - # rows don't read OOB, then select -1 for them. Matches - # torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a - # plain expert_map[-1] would wrap to a valid local id and misroute). - valid = tid >= 0 - idx = tl.where(valid, tid, 0) - local = tl.load(expert_map_ptr + idx, mask=mask, other=-1) - out = tl.where(valid, local.to(tl.int64), -1) - tl.store(out_ptr + offs, out, mask=mask) - - -def remap_topk_to_local( - topk_ids: torch.Tensor, expert_map: torch.Tensor -) -> torch.Tensor: - """Fused global->local expert-id mapping over a topk_ids tensor, preserving -1. - - Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)`` - with one kernel. Returns a NEW int64 tensor -- the caller keeps the original - ``topk_ids`` as ``global_topk_ids``, so this must not write in place. - - (Distinct from ``deep_gemm_utils.apply_expert_map``, which is a scalar - ``@triton.jit`` device helper called from within other kernels.) - """ - out = torch.empty_like(topk_ids, dtype=torch.int64) - n = topk_ids.numel() - BLOCK = 1024 - grid = (triton.cdiv(n, BLOCK),) - _remap_topk_to_local_kernel[grid](topk_ids, expert_map, out, n, BLOCK=BLOCK) - return out - - class BaseOAITritonExperts(mk.FusedMoEExpertsModular): @property def expects_unquantized_inputs(self) -> bool: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 7667dfc6ae..cbbc39b943 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -2133,6 +2133,59 @@ def make_mxfp4_moe_quant_config( ) +_MXFP4_SCHEME_CANDIDATES: list[tuple[QuantKey, QuantKey | None]] = [ + (kMxfp4Static, kMxfp4Dynamic), + (kMxfp4Static, kMxfp8Dynamic), + (kMxfp4Static, kFp8Dynamic128Sym), + (kMxfp4Static, kFp8StaticTensorSym), + (kMxfp4Static, None), +] + + +def _mxfp4_quant_keys_for_experts( + experts_cls: type[mk.FusedMoEExperts], + mxfp4_backend: Mxfp4MoeBackend, +) -> tuple[QuantKey | None, QuantKey | None]: + for weight_key, activation_key in _MXFP4_SCHEME_CANDIDATES: + if experts_cls._supports_quant_scheme(weight_key, activation_key): + return weight_key, activation_key + return kMxfp4Static, _backend_activation_key(mxfp4_backend) + + +def _activation_format_for_config( + moe_config: FusedMoEConfig, +) -> mk.FusedMoEActivationFormat: + return ( + mk.FusedMoEActivationFormat.BatchedExperts + if moe_config.moe_parallel_config.use_batched_activation_format + else mk.FusedMoEActivationFormat.Standard + ) + + +def _validate_mxfp4_experts_config( + experts_cls: type[mk.FusedMoEExperts], + moe_config: FusedMoEConfig, + mxfp4_backend: Mxfp4MoeBackend, +) -> None: + """Fail closed when a caller bypasses the oracle and picks experts directly.""" + activation_format = _activation_format_for_config(moe_config) + weight_key, activation_key = _mxfp4_quant_keys_for_experts( + experts_cls, mxfp4_backend + ) + supported, reason = experts_cls.is_supported_config( + experts_cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) + if not supported: + raise ValueError( + f"{experts_cls.__name__} does not support this MoE deployment" + f"{f' since {reason}' if reason else ''}." + ) + + def make_mxfp4_moe_kernel( moe_quant_config: FusedMoEQuantConfig, moe_config: FusedMoEConfig, @@ -2141,6 +2194,8 @@ def make_mxfp4_moe_kernel( routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, ) -> mk.FusedMoEKernel: """Create a FusedMoEKernel for the given MXFP4 backend.""" + _validate_mxfp4_experts_config(experts_cls, moe_config, mxfp4_backend) + is_monolithic = issubclass(experts_cls, mk.FusedMoEExpertsMonolithic) prepare_finalize = maybe_make_prepare_finalize( diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index c724a7c7cb..b1a6dd5632 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -235,6 +235,73 @@ def count_expert_num_tokens( return expert_num_tokens +@triton.jit +def _remap_topk_to_local_kernel( + topk_ids_ptr, # [n] global expert IDs (-1 = invalid) + expert_map_ptr, # [num_experts] global->local (-1 for non-local) + out_ptr, # [n] local expert IDs (-1 for invalid/non-local) + n_elements, + BLOCK: tl.constexpr, + OUT_INT32: tl.constexpr, +): + pid = tl.program_id(0) + offs = pid * BLOCK + tl.arange(0, BLOCK) + mask = offs < n_elements + tid = tl.load(topk_ids_ptr + offs, mask=mask, other=-1) + # Gather expert_map[tid] for valid (tid >= 0); clamp the index so invalid + # rows don't read OOB, then select -1 for them. Matches + # torch.where(tid >= 0, expert_map[clamp(tid, 0)], -1) -- preserving -1 (a + # plain expert_map[-1] would wrap to a valid local id and misroute). + valid = tid >= 0 + idx = tl.where(valid, tid, 0) + local = tl.load(expert_map_ptr + idx, mask=mask, other=-1) + if OUT_INT32: + out = tl.where(valid, local, -1) + else: + out = tl.where(valid, local.to(tl.int64), -1) + tl.store(out_ptr + offs, out, mask=mask) + + +def remap_topk_to_local( + topk_ids: torch.Tensor, + expert_map: torch.Tensor, + *, + out_dtype: torch.dtype = torch.int64, +) -> torch.Tensor: + """Map global expert ids to local ids; non-local or invalid routes become -1. + + Replaces ``torch.where(topk_ids >= 0, expert_map[topk_ids.clamp(min=0)], -1)`` + with one kernel. Returns a NEW tensor -- the caller keeps the original + ``topk_ids`` as ``global_topk_ids``, so this must not write in place. + + Args: + topk_ids: Global expert ids; -1 marks invalid slots. Must be contiguous; + the Triton kernel indexes by flat ``numel()`` offsets. + expert_map: Global-to-local mapping; -1 marks non-local experts. + out_dtype: Output dtype, ``torch.int32`` or ``torch.int64``. + + Returns: + Remapped local expert ids with the same shape as ``topk_ids``. + + """ + if out_dtype not in (torch.int32, torch.int64): + raise ValueError(f"out_dtype must be int32 or int64, got {out_dtype}") + assert topk_ids.is_contiguous() + out = torch.empty(topk_ids.shape, dtype=out_dtype, device=topk_ids.device) + n = topk_ids.numel() + BLOCK = 1024 + grid = (triton.cdiv(n, BLOCK),) + _remap_topk_to_local_kernel[grid]( + topk_ids, + expert_map, + out, + n, + BLOCK=BLOCK, + OUT_INT32=out_dtype == torch.int32, + ) + return out + + def _resize_cache(x: torch.Tensor, v: tuple[int, ...]) -> torch.Tensor: """Shrink the given tensor and apply the given view to it. This is used to resize the intermediate fused_moe caches.