[feat] FlashInfer CuteDSL MegaMoE integration (#54049)

Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
Julien Debache
2026-10-02 15:43:33 -04:00
committed by GitHub
parent c4973d94fe
commit 111f71a6db
33 changed files with 1663 additions and 1036 deletions
+1 -1
View File
@@ -511,7 +511,7 @@ def _check_dummy_hash_model_forward(
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
torch.nn.Module.__init__(model)
model.use_mega_moe = False
model.use_native_mega_moe = False
model.use_sequence_parallel = False
model.fuse_mhc_all_reduce = False
model.engram_hash = state
+1 -1
View File
@@ -167,7 +167,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
draft = SimpleNamespace(
use_mega_moe=False,
use_native_mega_moe=False,
use_sequence_parallel=False,
hc_mult=hc_mult,
layers=[make_layer(mix) for mix in mixes],
+307
View File
@@ -0,0 +1,307 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
MoE geometry of a real model, warms it up and runs a forward whose inputs are
chosen so that every quantization step in the megakernel is exact, for the
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
32). The expected output is then known in closed form and compared bit for
bit.
"""
from dataclasses import dataclass
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_deep_gemm
@dataclass(frozen=True)
class Geometry:
name: str
hidden_size: int
intermediate_size: int
num_experts: int
top_k: int
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
GEOMETRIES = (
Geometry("dsv4-pro", 7168, 3072, 384, 6),
Geometry("dsv4-flash", 4096, 2048, 256, 6),
Geometry("mistral-small-4", 4096, 2048, 128, 4),
Geometry("mistral-large-3", 7168, 4096, 128, 4),
)
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
NUM_TOKENS_PER_RANK = 4
MAX_TOKENS_PER_RANK = 128
# Every block scale below is either 1 or a power of two, so the FP4 codes
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
#
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
# it; every weight column the sentinel meets is zero.
X_SIGNAL = 2.0
X_SENTINEL = 6.0
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
W13_DIAG = 6.0
H_SIGNAL = 144.0
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
# sentinel sits next to a zero intermediate channel and pins the block scale.
W2_SENTINEL = 6.0
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
def _expert_out(expert: int) -> float:
return E2M1_CODES[expert % len(E2M1_CODES)]
def _topk_weights(top_k: int) -> tuple[float, ...]:
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
# weight * 144 must requantize exactly too: powers of two do.
return tuple(2.0 ** -(k + 1) for k in range(top_k))
def _routing_method(backend: str) -> RoutingMethodType:
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
return RoutingMethodType.DeepseekV4
return RoutingMethodType.Default
def _routing(
geometry: Geometry, rank: int, device: torch.device
) -> tuple[torch.Tensor, torch.Tensor]:
"""top_k distinct experts per token, strided across ranks so dispatch goes
remote."""
base = (
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
) % geometry.num_experts
stride = geometry.num_experts // geometry.top_k
offsets = torch.arange(geometry.top_k, device=device) * stride
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
torch.int32
)
topk_weights = torch.tensor(
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
dtype=torch.float32,
device=device,
)
return topk_ids, topk_weights
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
x = torch.zeros(
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
)
for t in range(NUM_TOKENS_PER_RANK):
x[t, 16 * t] = X_SIGNAL
x[t, 16 * t + 1] = X_SENTINEL
return x
def _local_expert_weights(
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
) -> FlashInferMoeEpWeights:
w13 = torch.zeros(
num_local_experts,
2 * geometry.intermediate_size,
geometry.hidden_size,
dtype=torch.bfloat16,
device=device,
)
w2 = torch.zeros(
num_local_experts,
geometry.hidden_size,
geometry.intermediate_size,
dtype=torch.bfloat16,
device=device,
)
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
for local in range(num_local_experts):
expert = rank * num_local_experts + local
w13[local, 0, token_columns] = W13_DIAG
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
w2[local, expert, 0] = _expert_out(expert)
w2[local, expert, 2] = W2_SENTINEL
return FlashInferMoeEpWeights(w13=w13, w2=w2)
def _expected_output(
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
) -> torch.Tensor:
expected = torch.zeros(
NUM_TOKENS_PER_RANK,
geometry.hidden_size,
dtype=torch.float32,
device=topk_ids.device,
)
for t in range(NUM_TOKENS_PER_RANK):
for k in range(geometry.top_k):
expert = int(topk_ids[t, k])
weight = float(topk_weights[t, k])
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
return expected.to(torch.bfloat16)
def _moe_config(
vllm_config: VllmConfig,
geometry: Geometry,
world_size: int,
device: torch.device,
backend: str,
) -> FusedMoEConfig:
parallel = FusedMoEParallelConfig.make(
tp_size_=1,
pcp_size_=1,
dp_size_=world_size,
sp_size_=1,
vllm_parallel_config=vllm_config.parallel_config,
)
return FusedMoEConfig(
num_experts=geometry.num_experts,
experts_per_token=geometry.top_k,
hidden_dim=geometry.hidden_size,
intermediate_size=geometry.intermediate_size,
num_local_experts=geometry.num_experts // world_size,
num_logical_experts=geometry.num_experts,
activation=MoEActivation.SILU,
device=device,
routing_method=_routing_method(backend),
moe_parallel_config=parallel,
in_dtype=torch.bfloat16,
moe_backend=backend,
max_num_tokens=MAX_TOKENS_PER_RANK,
)
def _check_expert_views_alias_kernel_memory(
adapter: FlashInferMoeEp,
geometry: Geometry,
rank: int,
num_local_experts: int,
device: torch.device,
) -> None:
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
Swap local experts 0 and 1 in place on every rank, then route through the
swapped physical slots: the output must be the one the unswapped logical
experts would have produced.
"""
for view in adapter.expert_weight_views():
first = view[0].clone()
view[0].copy_(view[1])
view[1].copy_(first)
torch.distributed.barrier()
topk_ids, topk_weights = _routing(geometry, rank, device)
slot = topk_ids % num_local_experts
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
torch.accelerator.synchronize()
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
def _init_warmup_forward(
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
) -> None:
set_env_vars_and_device(env)
vllm_config = VllmConfig()
vllm_config.parallel_config.data_parallel_size = world_size
vllm_config.parallel_config.enable_expert_parallel = True
vllm_config.kernel_config.moe_backend = backend
with set_current_vllm_config(vllm_config):
ensure_model_parallel_initialized(
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
)
rank = torch.distributed.get_rank()
device = torch.device(f"cuda:{rank}")
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
kMxfp4Static,
None,
mk.FusedMoEActivationFormat.Standard,
)
assert supported, reason
adapter = FlashInferMoeEp(
moe,
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
apply_topk_in_fc1=apply_topk_in_fc1(moe),
)
try:
adapter.warmup()
topk_ids, topk_weights = _routing(geometry, rank, device)
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
torch.accelerator.synchronize()
expected = _expected_output(geometry, topk_ids, topk_weights)
assert output.dtype == expected.dtype and output.shape == expected.shape
assert torch.equal(output, expected), (
f"rank {rank}: max abs diff "
f"{(output.float() - expected.float()).abs().max().item()}, "
f"nonzero output channels per token "
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
)
if backend == FLASHINFER_MOE_EP_CUTEDSL:
_check_expert_views_alias_kernel_memory(
adapter, geometry, rank, moe.num_local_experts, device
)
finally:
adapter.destroy()
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="The FlashInfer MoE-EP megakernels require SM100",
)
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
@pytest.mark.parametrize("world_size", [2, 4])
def test_init_warmup_forward_is_exact_on_grid_inputs(
world_size: int, backend: str, geometry: Geometry
):
pytest.importorskip("flashinfer.moe_ep")
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
pytest.skip("DeepGEMM is not available")
if torch.accelerator.device_count() < world_size:
pytest.skip(f"Need at least {world_size} GPUs")
distributed_run(_init_warmup_forward, world_size, geometry, backend)
+313 -195
View File
@@ -1,116 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the flashinfer moe_ep backend plumbing.
Everything here runs without a GPU or a flashinfer install: the flashinfer
modules the helpers import lazily are replaced with capture fakes.
"""
import sys
from dataclasses import dataclass, field
from types import ModuleType, SimpleNamespace
from typing import Any
from dataclasses import replace
from types import SimpleNamespace
import pytest
import torch
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm.config.kernel import (
FLASHINFER_MOE_EP_BACKENDS,
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
MEGA_MOE_BACKENDS,
NATIVE_MEGA_MOE_BACKENDS,
PASSTHROUGH_ALL2ALL_BACKEND,
bind_passthrough_all2all_backend,
validate_flashinfer_moe_ep_model,
)
from vllm.utils.flashinfer_moe_ep import (
_E2M1_LUT,
FI_MOE_EP_BACKEND_SPECS,
_dequant_fp4_ue8m0_gran32,
build_fi_mega_config,
fi_moe_ep_backend_spec,
make_fi_moe_ep_bootstrap,
megakernel_runtime_requirements,
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
)
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
FlashInferMoeEpExperts,
epilogue_from_quant_config,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
make_mxfp4_moe_quant_config,
map_mxfp4_backend,
mxfp4_round_up_hidden_size_and_intermediate_size,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
backend_to_kernel_cls as mxfp4_backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
NvFp4MoeBackend,
convert_to_nvfp4_moe_kernel_format,
map_nvfp4_backend,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
backend_to_kernel_cls as nvfp4_backend_to_kernel_cls,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
PassThroughPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Static,
kNvfp4Dynamic,
kNvfp4Static,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
@dataclass
class _FakeBootstrapConfig:
world_size: int
rank: int
process_group: Any = None
auto_bootstrap: bool = True
device: int | None = field(default=None, kw_only=True)
def test_flashinfer_backends_are_megakernels_outside_the_native_model_path():
assert FLASHINFER_MOE_EP_BACKENDS <= MEGA_MOE_BACKENDS
assert FLASHINFER_MOE_EP_BACKENDS.isdisjoint(NATIVE_MEGA_MOE_BACKENDS)
@dataclass
class _FakeDeepGemmMegaMoeConfig:
intermediate_size: int
top_k: int
activation_clamp: float | None
fast_math: bool
@dataclass
class _FakeNvfp4CutedslMegaMoeConfig:
intermediate_size: int
top_k: int
activation_clamp: float | None
fast_math: bool
@dataclass
class _FakeMegaConfig:
megakernel: Any
preprocess_weights: bool
quantize_input: bool
@pytest.fixture
def fake_flashinfer(monkeypatch):
"""Install a minimal fake flashinfer.moe_ep for the lazy imports."""
moe_ep = ModuleType("flashinfer.moe_ep")
core = ModuleType("flashinfer.moe_ep.core")
runtime = ModuleType("flashinfer.moe_ep.core.runtime")
flashinfer = ModuleType("flashinfer")
fake_attrs: dict[ModuleType, dict[str, Any]] = {
moe_ep: {
"BootstrapConfig": _FakeBootstrapConfig,
"DeepGemmMegaMoeConfig": _FakeDeepGemmMegaMoeConfig,
"Nvfp4CutedslMegaMoeConfig": _FakeNvfp4CutedslMegaMoeConfig,
"MegaConfig": _FakeMegaConfig,
"core": core,
},
runtime: {"TORCH_DIST": "torch_dist", "NVSHMEM": "nvshmem"},
flashinfer: {"moe_ep": moe_ep},
core: {"runtime": runtime},
}
for mod, attrs in fake_attrs.items():
for attr, value in attrs.items():
setattr(mod, attr, value)
for name, mod in {
"flashinfer": flashinfer,
"flashinfer.moe_ep": moe_ep,
"flashinfer.moe_ep.core": core,
"flashinfer.moe_ep.core.runtime": runtime,
}.items():
monkeypatch.setitem(sys.modules, name, mod)
return moe_ep
def test_fi_backend_strings_are_registered_mega_moe_backends():
assert set(FI_MOE_EP_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
assert FLASHINFER_MOE_EP_BACKENDS < MEGA_MOE_BACKENDS
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
instead of silently falling through to the generic FusedMoE path."""
def test_only_deep_gemm_backend_is_dsv4_specific():
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_CUTEDSL,
["MixtralForCausalLM"],
)
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
validate_flashinfer_moe_ep_model(moe_backend, ["DeepseekV4ForCausalLM"])
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_DEEP_GEMM,
["MixtralForCausalLM"],
)
validate_flashinfer_moe_ep_model(
FLASHINFER_MOE_EP_DEEP_GEMM,
["DeepseekV4ForCausalLM"],
)
@pytest.mark.parametrize(
@@ -118,8 +92,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
)
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
models validate their own constraints at construction time."""
"""Models validate native deep_gemm mega constraints at construction time."""
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
@@ -128,11 +101,7 @@ def test_non_fi_backend_ignores_architectures():
@pytest.mark.parametrize("moe_backend", sorted(MEGA_MOE_BACKENDS))
def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
"""Every mega backend must qualify for sequence-parallel MoE at
TP>1/EP: the predicate once matched only the native backend string,
which silently ran the fi backends full-batch with an all-reduce on
every rank — 0.42-0.65x native e2e at TP8."""
def test_token_sharding_backends_enable_dsv4_sequence_parallel(moe_backend: str):
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
vllm_config = SimpleNamespace(
@@ -147,113 +116,262 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
assert _use_sequence_parallel(vllm_config)
def test_fi_moe_ep_backend_spec_kernel_and_nvshmem_contract():
dg = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
assert dg.megakernel == "deep_gemm_mega"
assert not dg.needs_nvshmem
cd = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
assert cd.megakernel == "nvfp4_cutedsl"
assert cd.needs_nvshmem
with pytest.raises(ValueError, match="not a flashinfer moe_ep backend"):
fi_moe_ep_backend_spec("deep_gemm_mega_moe")
def test_dsv4_requests_pre_fc2_router_weight_placement():
dsv4 = SimpleNamespace(routing_method=RoutingMethodType.DeepseekV4)
default = SimpleNamespace(routing_method=RoutingMethodType.Default)
assert fi_ep.apply_topk_in_fc1(dsv4)
assert not fi_ep.apply_topk_in_fc1(default)
def test_megakernel_runtime_requirements(fake_flashinfer):
dg = megakernel_runtime_requirements(
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
def test_backend_specs_preserve_weight_format_contracts():
cutedsl = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_CUTEDSL)
assert cutedsl.kernel == "cutedsl"
assert cutedsl.weight_formats == frozenset({"nvfp4", "mxfp4"})
deep_gemm = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_DEEP_GEMM)
assert deep_gemm.kernel == "deep_gemm"
assert deep_gemm.weight_formats == frozenset({"mxfp4"})
def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
fields = dict(
moe_backend=moe_backend,
moe_parallel_config=SimpleNamespace(
use_ep=True, use_batched_activation_format=False
),
is_act_and_mul=True,
activation=MoEActivation.SILU,
has_hash_routing=False,
routing_method=RoutingMethodType.DeepseekV4,
router_logits_dtype=torch.float32,
hidden_dim=256,
is_lora_enabled=False,
skip_final_all_reduce=False,
in_dtype=torch.bfloat16,
has_bias=False,
swiglu_alpha=None,
swiglu_beta=None,
)
assert dg == frozenset({"torch_dist"})
fields.update(overrides)
return SimpleNamespace(**fields)
cd = megakernel_runtime_requirements(
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
def _is_supported(moe, weight_key, activation_key) -> tuple[bool, str]:
supported, reason = FlashInferMoeEpExperts.is_supported_config(
FlashInferMoeEpExperts,
moe,
weight_key,
activation_key,
mk.FusedMoEActivationFormat.Standard,
)
assert cd == frozenset({"torch_dist", "nvshmem"})
return supported, reason or ""
def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
device it already bound via BootstrapConfig.device."""
import vllm.utils.flashinfer_moe_ep as mod
pg = object()
@pytest.fixture
def megakernel_host(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
"""A vLLM config without DBO/EPLB/weight transfer, on a supported device."""
config = SimpleNamespace(
weight_transfer_config=None,
parallel_config=SimpleNamespace(enable_dbo=False, enable_eplb=False),
)
monkeypatch.setattr(fi_ep, "get_current_vllm_config", lambda: config)
monkeypatch.setattr(
mod,
"get_ep_group",
lambda: SimpleNamespace(world_size=4, rank_in_group=2, device_group=pg),
FlashInferMoeEpExperts, "_supports_current_device", staticmethod(lambda: True)
)
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 3)
bootstrap = make_fi_moe_ep_bootstrap()
assert bootstrap.world_size == 4
assert bootstrap.rank == 2
assert bootstrap.process_group is pg
assert bootstrap.auto_bootstrap is False
assert bootstrap.device == 3
return config
def test_build_fi_mega_config_selects_kernel_config(fake_flashinfer):
dg = build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=7.0,
megakernel="deep_gemm_mega",
def test_megakernel_requires_expert_parallel(megakernel_host):
"""The oracle rejects the megakernel through the experts' support check."""
moe = _megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL)
assert _is_supported(moe, kNvfp4Static, kNvfp4Dynamic) == (True, "")
moe.moe_parallel_config.use_ep = False
supported, reason = _is_supported(moe, kNvfp4Static, kNvfp4Dynamic)
assert not supported and "parallel config" in reason
def test_megakernel_reports_its_own_constraints(megakernel_host):
"""Constraints the framework does not know about surface as reasons."""
_, reason = _is_supported(
_megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL), kNvfp4Static, None
)
assert isinstance(dg.megakernel, _FakeDeepGemmMegaMoeConfig)
assert dg.megakernel.intermediate_size == 2048
assert dg.megakernel.top_k == 8
assert dg.megakernel.activation_clamp == 7.0
assert dg.preprocess_weights and dg.quantize_input
assert "A16 activations" in reason
cd = build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=None,
megakernel="nvfp4_cutedsl",
deep_gemm = _megakernel_moe(
FLASHINFER_MOE_EP_DEEP_GEMM, routing_method=RoutingMethodType.Renormalize
)
assert isinstance(cd.megakernel, _FakeNvfp4CutedslMegaMoeConfig)
_, reason = _is_supported(deep_gemm, kMxfp4Static, None)
assert "routing method" in reason
with pytest.raises(ValueError, match="Unsupported fi_moe_ep megakernel"):
build_fi_mega_config(
intermediate_size=2048,
top_k=8,
activation_clamp=None,
megakernel="deep_gemm",
megakernel_host.parallel_config.enable_eplb = True
_, reason = _is_supported(
_megakernel_moe(FLASHINFER_MOE_EP_DEEP_GEMM), kMxfp4Static, None
)
assert reason.endswith("EPLB")
def test_megakernels_are_oracle_backends():
"""Quant methods reach the megakernel through the NVFP4/MXFP4 oracles."""
nvfp4 = map_nvfp4_backend(FLASHINFER_MOE_EP_CUTEDSL)
assert nvfp4 is NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
assert nvfp4_backend_to_kernel_cls(nvfp4) == [FlashInferMoeEpExperts]
assert map_mxfp4_backend(FLASHINFER_MOE_EP_DEEP_GEMM) == [
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
]
for backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
assert mxfp4_backend_to_kernel_cls(backend) == [FlashInferMoeEpExperts]
# The megakernel takes the model's own dimensions.
assert mxfp4_round_up_hidden_size_and_intermediate_size(
backend, 2880, 2880
) == (2880, 2880)
quant_config = make_mxfp4_moe_quant_config(
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL, w1_scale=None, w2_scale=None
)
assert quant_config is not None and quant_config.use_nvfp4_w4a4
def test_megakernel_is_a_modular_kernel_with_pass_through_stages():
"""The megakernel plugs into ``FusedMoEKernel`` like any other experts impl.
Routing stays outside (top-k ids in), dispatch and combine happen inside,
so prepare passes tokens through untouched and finalize has nothing to do.
"""
moe = make_dummy_moe_config(
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
)
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
kernel = mk.FusedMoEKernel(
PassThroughPrepareAndFinalize(),
FlashInferMoeEpExperts(moe, quant_config),
)
assert not kernel.is_monolithic
assert kernel.prepare_finalize.output_is_reduced()
# Generic stages: the int32 top-k ids the megakernel wants are its concern.
assert kernel.prepare_finalize.topk_indices_dtype() is None
assert kernel.fused_experts.expects_unquantized_inputs
assert isinstance(
kernel.fused_experts.finalize_weight_and_reduce_impl(), TopKWeightAndReduceNoOP
)
assert kernel.fused_experts.workspace_shapes(
16, 256, 256, 2, 8, 8, None, MoEActivation.SILU
) == ((0,), (0,), (16, 256))
tokens = torch.zeros(4, 256, dtype=torch.bfloat16)
prepared = kernel.prepare_finalize.prepare(
tokens,
torch.ones(4, 2),
torch.zeros(4, 2, dtype=torch.int32),
8,
None,
False,
quant_config,
)
assert prepared[0] is tokens and all(item is None for item in prepared[1:])
def test_epilogue_takes_the_weight_global_scales_from_the_quant_config():
"""The canonical ``g1/g2_alphas`` become the per-expert fc1/fc2 alphas and
keep aliasing the layer's tensors, so EPLB permutations reach the kernel."""
g1_alphas = torch.tensor([1.0, 2.0])
g2_alphas = torch.tensor([3.0, 4.0])
quant_config = nvfp4_moe_quant_config(
g1_alphas=g1_alphas,
g2_alphas=g2_alphas,
a1_gscale=torch.ones(2),
a2_gscale=torch.ones(2),
w1_scale=torch.ones(2, 4, 1),
w2_scale=torch.ones(2, 2, 1),
)
epilogue = epilogue_from_quant_config(quant_config)
assert epilogue.fc1_alpha is not None and epilogue.fc2_alpha is not None
assert epilogue.fc1_alpha.data_ptr() == g1_alphas.data_ptr()
assert epilogue.fc2_alpha.data_ptr() == g2_alphas.data_ptr()
# Activations are quantized dynamically: no norm constants.
assert epilogue.input_norm_const == 1.0 and epilogue.fc1_norm_const is None
def test_epilogue_rejects_unloaded_weight_global_scales():
quant_config = nvfp4_moe_quant_config(
g1_alphas=torch.tensor([1.0, float("nan")]),
g2_alphas=torch.ones(2),
a1_gscale=torch.ones(2),
a2_gscale=torch.ones(2),
w1_scale=torch.ones(2, 4, 1),
w2_scale=torch.ones(2, 2, 1),
)
with pytest.raises(ValueError, match="g1_alphas"):
epilogue_from_quant_config(quant_config)
def test_mxfp4_checkpoints_keep_the_default_epilogue():
"""MXFP4 has no per-tensor global scales; the megakernel needs no alphas."""
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
assert epilogue_from_quant_config(quant_config) == fi_ep.FlashInferMoeEpEpilogue()
def test_nvfp4_kernel_format_passes_megakernel_weights_through():
"""The megakernel takes the canonical tensors as they are; the oracle must
not reject or reshape them."""
tensors = tuple(torch.arange(4.0).reshape(2, 2) + i for i in range(8))
converted = convert_to_nvfp4_moe_kernel_format(
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
SimpleNamespace(),
*tensors,
is_act_and_mul=True,
)
assert all(out is src for out, src in zip(converted, tensors))
def test_passthrough_all2all_backend_selects_the_pass_through_stages():
"""``all2all_backend="passthrough"`` is how the oracles get the
pass-through prepare/finalize."""
moe = make_dummy_moe_config(
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
)
moe.moe_parallel_config = replace(
moe.moe_parallel_config, all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND
)
assert not moe.moe_parallel_config.use_all2all_kernels
prepare_finalize = maybe_make_prepare_finalize(
moe, FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
)
assert isinstance(prepare_finalize, PassThroughPrepareAndFinalize)
def test_passthrough_all2all_backend_is_not_a_cli_choice():
"""Users cannot pick the passthrough all2all backend explicitly."""
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
with pytest.raises(SystemExit):
parser.parse_args(["--all2all-backend", PASSTHROUGH_ALL2ALL_BACKEND])
parser.parse_args(["--all2all-backend", "deepep_low_latency"])
def test_megakernel_moe_backend_binds_the_all2all_backend():
"""A megakernel deployment gets its all2all backend by default."""
parallel = SimpleNamespace(all2all_backend="allgather_reducescatter")
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
assert parallel.all2all_backend == PASSTHROUGH_ALL2ALL_BACKEND
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
with pytest.raises(ValueError, match="cannot be paired"):
bind_passthrough_all2all_backend(
FLASHINFER_MOE_EP_DEEP_GEMM,
SimpleNamespace(all2all_backend="deepep_low_latency"),
)
with pytest.raises(ValueError, match="only valid with"):
bind_passthrough_all2all_backend(
"flashinfer_trtllm",
SimpleNamespace(all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND),
)
def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
from vllm.models.deepseek_v4.nvidia.fi_moe import ckpt_uses_nvfp4_experts
nvfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo="NVFP4"))
assert ckpt_uses_nvfp4_experts(nvfp4)
mxfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo=None))
assert not ckpt_uses_nvfp4_experts(mxfp4)
no_algo = SimpleNamespace(quant_config=SimpleNamespace())
assert not ckpt_uses_nvfp4_experts(no_algo)
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
"""One 32-element scale group per row: low nibble is the even element,
high nibble the odd one, ue8m0 scale applies to the whole group."""
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
out = _dequant_fp4_ue8m0_gran32(packed, sf)
assert out.shape == (2, 32)
assert out.dtype == torch.bfloat16
expected = torch.empty(2, 32)
for row in range(2):
for col in range(16):
byte = int(packed[row, col])
expected[row, 2 * col] = _E2M1_LUT[byte & 0x0F]
expected[row, 2 * col + 1] = _E2M1_LUT[byte >> 4]
expected[row] *= 2.0**row
assert torch.equal(out, expected.to(torch.bfloat16))
untouched = SimpleNamespace(all2all_backend="deepep_low_latency")
bind_passthrough_all2all_backend("flashinfer_trtllm", untouched)
assert untouched.all2all_backend == "deepep_low_latency"
+1 -1
View File
@@ -657,7 +657,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
torch.nn.Module.__init__(moe)
moe.use_mega_moe = True
moe.use_native_mega_moe = True
moe.gate = FakeGate()
moe.experts = FakeExperts()
moe.shared_experts = FakeSharedExperts()
+1 -1
View File
@@ -241,7 +241,7 @@ def test_v41_dspark_loads_linear_scales(
linear_scale_name=scale_name,
pad_shared_expert=False,
model=SimpleNamespace(
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
confidence_head=None,
),
named_parameters=lambda: [(runtime_name, param)],