[Bugfix][MoE] Apply expert_map remapping for CUTLASS FP4 MOE expert parallel (#59042)

Signed-off-by: Markov Ilya <markovilya197@gmail.com>
This commit is contained in:
Ilya Markov
2026-10-12 10:02:56 -04:00
committed by GitHub
parent f022975964
commit b3ee560ae8
6 changed files with 470 additions and 59 deletions
+196 -7
View File
@@ -9,7 +9,7 @@ import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_config
from vllm import _custom_ops as ops
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe import (
@@ -26,8 +26,10 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
)
from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
fp8_w8a8_moe_quant_config,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
@@ -41,7 +43,10 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
)
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
from vllm.model_executor.layers.fused_moe.utils import (
moe_kernel_quantize_input,
remap_topk_to_local,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -183,6 +188,189 @@ def test_cutlass_moe_permutation_maps_padding_to_zero():
torch.testing.assert_close(restored[3], torch.zeros_like(restored[3]))
def test_remap_topk_ids_to_local():
device = "cuda"
expert_map = torch.tensor([-1, 0, -1, 1, -1, 2], dtype=torch.int32, device=device)
topk_ids = torch.tensor([[1, 3], [0, 5], [-1, 4]], dtype=torch.int32, device=device)
expected = torch.tensor(
[[0, 1], [-1, 2], [-1, -1]], dtype=torch.int32, device=device
)
actual = remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
torch.testing.assert_close(actual, expected)
def test_remap_topk_ids_to_local_requires_contiguous_topk_ids():
expert_map = torch.tensor([0, 1], dtype=torch.int32)
topk_ids = torch.tensor([[0, 1], [1, 0]], dtype=torch.int32).t()
assert not topk_ids.is_contiguous()
with pytest.raises(AssertionError):
remap_topk_to_local(topk_ids, expert_map)
def test_cutlass_fp4_moe_supports_expert_parallel_config():
ep_config = FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
)
assert CutlassExpertsMxfp4._supports_parallel_config(ep_config)
assert CutlassExpertsFp4._supports_parallel_config(ep_config)
def _slice_moe_quant_config(
quant_config: FusedMoEQuantConfig, start: int, end: int
) -> FusedMoEQuantConfig:
num_experts = quant_config.w1_scale.shape[0]
sliced_config = copy.deepcopy(quant_config)
for desc_name in ("_a1", "_a2", "_w1", "_w2"):
desc = getattr(sliced_config, desc_name)
for field_name in ("scale", "alpha_or_gscale", "bias", "zp"):
tensor = getattr(desc, field_name)
if (
isinstance(tensor, torch.Tensor)
and tensor.ndim > 0
and tensor.shape[0] == num_experts
):
setattr(desc, field_name, tensor[start:end])
return sliced_config
def _swizzle_mxfp4_moe_scales(
w1_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
e, w1_n, w1_scale_k = w1_scale.shape
w1_k = w1_scale_k * 32
w2_m, w2_scale_n = w2_scale.shape[1], w2_scale.shape[2]
w2_n = w2_scale_n * 32
swizzled_w1 = []
swizzled_w2 = []
for expert in range(e):
swizzled_w1.append(
swizzle_mxfp4_scales(w1_scale[expert], w1_n, w1_k).reshape(w1_n, w1_scale_k)
)
swizzled_w2.append(
swizzle_mxfp4_scales(w2_scale[expert], w2_m, w2_n).reshape(w2_m, w2_scale_n)
)
return torch.stack(swizzled_w1), torch.stack(swizzled_w2)
def _make_cutlass_fp4_moe_weights(
quantization: str,
num_experts: int,
n: int,
k: int,
dtype: torch.dtype,
device: torch.device,
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
if quantization == "nvfp4":
return make_test_quant_config(
num_experts,
n,
k,
in_dtype=dtype,
quant_dtype="nvfp4",
block_shape=None,
per_act_token_quant=False,
make_gate=True,
)
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
mxfp4_quantize,
)
w1, w1_scale = mxfp4_quantize(
torch.randn((num_experts, 2 * n, k), device=device, dtype=dtype) / 8
)
w2, w2_scale = mxfp4_quantize(
torch.randn((num_experts, k, n), device=device, dtype=dtype) / 8
)
w1_scale, w2_scale = _swizzle_mxfp4_moe_scales(w1_scale, w2_scale)
return (
w1.contiguous(),
w2.contiguous(),
mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale),
)
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
@torch.inference_mode()
def test_cutlass_fp4_moe_EP(
quantization: str,
workspace_init,
):
"""Sharded CUTLASS FP4/MXFP4 MoE with expert_map must match full weights."""
experts_cls = CutlassExpertsFp4 if quantization == "nvfp4" else CutlassExpertsMxfp4
if not experts_cls._supports_current_device():
pytest.skip(f"CUTLASS {quantization} MoE is not supported on this GPU")
set_random_seed(7)
m, n, k = 8, 128, 128
num_experts, num_local_experts, topk = 8, 2, 2
dtype = torch.bfloat16
device = torch.device("cuda")
w1, w2, quant_config = _make_cutlass_fp4_moe_weights(
quantization, num_experts, n, k, dtype, device
)
hidden_states = torch.randn((m, k), device=device, dtype=dtype) / 10
score = torch.randn((m, num_experts), device=device, dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(
hidden_states, score, topk, renormalize=False
)
moe_config = make_dummy_moe_config(
num_experts=num_experts,
num_local_experts=num_experts,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=n,
in_dtype=dtype,
max_num_tokens=m,
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts_cls(moe_config=moe_config, quant_config=quant_config),
)
apply_kwargs = {
"hidden_states": hidden_states,
"w1": w1,
"w2": w2,
"topk_weights": topk_weights,
"topk_ids": topk_ids,
"global_num_experts": num_experts,
"activation": MoEActivation.SILU,
"expert_map": None,
"apply_router_weight_on_input": False,
}
reference = kernel.apply(**apply_kwargs)
sharded = run_with_expert_maps(
num_experts,
num_local_experts,
quant_config,
experts_cls,
**apply_kwargs,
)
assert reference.abs().max() > 0
torch.testing.assert_close(sharded, reference, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
@torch.inference_mode()
def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization: str):
@@ -486,6 +674,7 @@ def run_with_expert_maps(
num_experts: int,
num_local_experts: int,
quant_config: FusedMoEQuantConfig,
experts_cls: type[mk.FusedMoEExperts],
**cutlass_moe_kwargs,
):
def slice_experts():
@@ -513,9 +702,7 @@ def run_with_expert_maps(
for k, t in full_tensors.items():
cutlass_moe_kwargs[k] = t[s:e]
new_quant_config = copy.deepcopy(quant_config)
new_quant_config._w1.scale = quant_config.w1_scale[s:e]
new_quant_config._w2.scale = quant_config.w2_scale[s:e]
new_quant_config = _slice_moe_quant_config(quant_config, s, e)
yield cutlass_moe_kwargs, new_quant_config
@@ -523,13 +710,14 @@ def run_with_expert_maps(
for kwargs, new_quant_config in slice_experts():
w2 = kwargs["w2"]
a = kwargs["hidden_states"]
w1 = kwargs["w1"]
moe_config = make_dummy_moe_config(
max_num_tokens=kwargs.get("hidden_states").shape[0],
experts_per_token=kwargs.get("topk_ids").shape[1],
num_experts=num_experts,
num_local_experts=num_local_experts,
hidden_dim=w2.shape[1],
intermediate_size=w2.shape[2],
intermediate_size=w1.shape[1] // 2,
in_dtype=a.dtype,
)
kernel = mk.FusedMoEKernel(
@@ -539,7 +727,7 @@ def run_with_expert_maps(
allow_new_interface=True,
use_monolithic=False,
),
CutlassExpertsFp8(
experts_cls(
moe_config=moe_config,
quant_config=new_quant_config,
),
@@ -624,6 +812,7 @@ def run_8_bit(
num_experts,
num_local_experts,
quant_config,
CutlassExpertsFp8,
**kwargs,
)
+111
View File
@@ -2114,6 +2114,117 @@ def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
assert "parallel config" in reason
@pytest.mark.parametrize(
"all2all_backend",
[
"deepep_high_throughput",
"flashinfer_nvlink_two_sided",
"flashinfer_nvlink_one_sided",
],
)
def test_make_mxfp4_moe_kernel_rejects_cutlass_unsupported_expert_parallel(
monkeypatch: pytest.MonkeyPatch,
all2all_backend: str,
):
"""Compressed-tensors CUTLASS W4A4 bypasses the oracle; guard at assembly."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
mxfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
make_mxfp4_moe_kernel,
)
monkeypatch.setattr(
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
)
moe_config = replace(
make_dummy_moe_config(
num_experts=256,
num_local_experts=32,
hidden_dim=4096,
intermediate_size=2048,
),
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend=all2all_backend,
enable_eplb=False,
),
)
w1_scale = torch.empty(32, 64, 128, dtype=torch.uint8)
w2_scale = torch.empty(32, 4096, 64, dtype=torch.uint8)
quant_config = mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale)
with pytest.raises(ValueError, match="parallel config"):
make_mxfp4_moe_kernel(
quant_config,
moe_config,
CutlassExpertsMxfp4,
Mxfp4MoeBackend.MARLIN,
)
def test_make_mxfp4_moe_kernel_validates_cutlass_expert_parallel(
monkeypatch: pytest.MonkeyPatch,
):
"""CUTLASS MXFP4 supports EP with allgather_reducescatter after expert_map."""
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsMxfp4,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
_validate_mxfp4_experts_config,
)
monkeypatch.setattr(
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
)
moe_config = replace(
make_dummy_moe_config(
num_experts=256,
num_local_experts=32,
hidden_dim=4096,
intermediate_size=2048,
),
moe_parallel_config=FusedMoEParallelConfig(
tp_size=1,
tp_rank=0,
pcp_size=1,
pcp_rank=0,
dp_size=8,
dp_rank=0,
ep_size=8,
ep_rank=0,
sp_size=1,
use_ep=True,
all2all_backend="allgather_reducescatter",
enable_eplb=False,
),
)
_validate_mxfp4_experts_config(
CutlassExpertsMxfp4, moe_config, Mxfp4MoeBackend.MARLIN
)
# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that
# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns
# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of