[feat] FlashInfer CuteDSL MegaMoE integration (#54049)
Signed-off-by: jdebache <jdebache@nvidia.com>
This commit is contained in:
@@ -511,7 +511,7 @@ def _check_dummy_hash_model_forward(
|
||||
|
||||
model = model_ops.DeepseekV4Model.__new__(model_ops.DeepseekV4Model)
|
||||
torch.nn.Module.__init__(model)
|
||||
model.use_mega_moe = False
|
||||
model.use_native_mega_moe = False
|
||||
model.use_sequence_parallel = False
|
||||
model.fuse_mhc_all_reduce = False
|
||||
model.engram_hash = state
|
||||
|
||||
@@ -167,7 +167,7 @@ def test_v41_dspark_head_collapses_with_last_ffn_mix(num_tokens, monkeypatch):
|
||||
|
||||
monkeypatch.setattr(dspark, "mhc_post_tilelang", lambda *args: streams)
|
||||
draft = SimpleNamespace(
|
||||
use_mega_moe=False,
|
||||
use_native_mega_moe=False,
|
||||
use_sequence_parallel=False,
|
||||
hc_mult=hc_mult,
|
||||
layers=[make_layer(mix) for mix in mixes],
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the FlashInfer MoE-EP adapter on real Blackwell GPUs.
|
||||
|
||||
Each test spawns one process per GPU, bootstraps vLLM's expert-parallel
|
||||
groups, builds a ``FlashInferMoeEp`` adapter from bf16 expert weights at the
|
||||
MoE geometry of a real model, warms it up and runs a forward whose inputs are
|
||||
chosen so that every quantization step in the megakernel is exact, for the
|
||||
CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
|
||||
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
|
||||
32). The expected output is then known in closed form and compared bit for
|
||||
bit.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_device
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
FlashInferMoeEp,
|
||||
FlashInferMoeEpWeights,
|
||||
apply_topk_in_fc1,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Static
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Geometry:
|
||||
name: str
|
||||
hidden_size: int
|
||||
intermediate_size: int
|
||||
num_experts: int
|
||||
top_k: int
|
||||
|
||||
|
||||
# Routed-expert geometry of the models the branch was evaluated on. DeepGEMM
|
||||
# keeps one UE8M0 scale byte per 32 elements and requires the per-token scale
|
||||
# rows to be TMA aligned, so hidden and intermediate sizes must be multiples
|
||||
# of 512 (FlashInfer's own validation only checks 128); all four qualify.
|
||||
GEOMETRIES = (
|
||||
Geometry("dsv4-pro", 7168, 3072, 384, 6),
|
||||
Geometry("dsv4-flash", 4096, 2048, 256, 6),
|
||||
Geometry("mistral-small-4", 4096, 2048, 128, 4),
|
||||
Geometry("mistral-large-3", 7168, 4096, 128, 4),
|
||||
)
|
||||
BACKENDS = (FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
|
||||
NUM_TOKENS_PER_RANK = 4
|
||||
MAX_TOKENS_PER_RANK = 128
|
||||
|
||||
# Every block scale below is either 1 or a power of two, so the FP4 codes
|
||||
# and the FP8 values survive quantization exactly: NVFP4 scales a 16-block by
|
||||
# absmax / 6 in E4M3, DeepGEMM scales a 32-block by absmax / 6 (FP4) or
|
||||
# absmax / 448 (FP8) rounded up to a power of two. A 6.0 in a block pins the
|
||||
# FP4 scale to 1; 2.0, 6.0 and 144 are exact FP8 values at any such scale.
|
||||
#
|
||||
# Token t carries X_SIGNAL at hidden column 16 * t and a 6.0 sentinel next to
|
||||
# it; every weight column the sentinel meets is zero.
|
||||
X_SIGNAL = 2.0
|
||||
X_SENTINEL = 6.0
|
||||
# Gate and up rows are identical, so h = silu(12) * 12 = 144 = 6 * 24
|
||||
# whichever half the kernel treats as the gate, and 144 requantizes exactly.
|
||||
W13_DIAG = 6.0
|
||||
H_SIGNAL = 144.0
|
||||
# Expert e writes E2M1_CODES[e % 7] * H_SIGNAL into output channel e. The 6.0
|
||||
# sentinel sits next to a zero intermediate channel and pins the block scale.
|
||||
W2_SENTINEL = 6.0
|
||||
E2M1_CODES = (0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
|
||||
|
||||
|
||||
def _expert_out(expert: int) -> float:
|
||||
return E2M1_CODES[expert % len(E2M1_CODES)]
|
||||
|
||||
|
||||
def _topk_weights(top_k: int) -> tuple[float, ...]:
|
||||
# The DeepSeek-V4 routing policy applies the router weight before fc2, so
|
||||
# weight * 144 must requantize exactly too: powers of two do.
|
||||
return tuple(2.0 ** -(k + 1) for k in range(top_k))
|
||||
|
||||
|
||||
def _routing_method(backend: str) -> RoutingMethodType:
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM:
|
||||
return RoutingMethodType.DeepseekV4
|
||||
return RoutingMethodType.Default
|
||||
|
||||
|
||||
def _routing(
|
||||
geometry: Geometry, rank: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""top_k distinct experts per token, strided across ranks so dispatch goes
|
||||
remote."""
|
||||
base = (
|
||||
torch.arange(NUM_TOKENS_PER_RANK, device=device) + rank * NUM_TOKENS_PER_RANK
|
||||
) % geometry.num_experts
|
||||
stride = geometry.num_experts // geometry.top_k
|
||||
offsets = torch.arange(geometry.top_k, device=device) * stride
|
||||
topk_ids = ((base[:, None] + offsets[None, :]) % geometry.num_experts).to(
|
||||
torch.int32
|
||||
)
|
||||
topk_weights = torch.tensor(
|
||||
[_topk_weights(geometry.top_k)] * NUM_TOKENS_PER_RANK,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
return topk_ids, topk_weights
|
||||
|
||||
|
||||
def _hidden_states(geometry: Geometry, device: torch.device) -> torch.Tensor:
|
||||
x = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK, geometry.hidden_size, dtype=torch.bfloat16, device=device
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
x[t, 16 * t] = X_SIGNAL
|
||||
x[t, 16 * t + 1] = X_SENTINEL
|
||||
return x
|
||||
|
||||
|
||||
def _local_expert_weights(
|
||||
geometry: Geometry, rank: int, num_local_experts: int, device: torch.device
|
||||
) -> FlashInferMoeEpWeights:
|
||||
w13 = torch.zeros(
|
||||
num_local_experts,
|
||||
2 * geometry.intermediate_size,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
w2 = torch.zeros(
|
||||
num_local_experts,
|
||||
geometry.hidden_size,
|
||||
geometry.intermediate_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
token_columns = [16 * t for t in range(NUM_TOKENS_PER_RANK)]
|
||||
for local in range(num_local_experts):
|
||||
expert = rank * num_local_experts + local
|
||||
w13[local, 0, token_columns] = W13_DIAG
|
||||
w13[local, geometry.intermediate_size, token_columns] = W13_DIAG
|
||||
w2[local, expert, 0] = _expert_out(expert)
|
||||
w2[local, expert, 2] = W2_SENTINEL
|
||||
return FlashInferMoeEpWeights(w13=w13, w2=w2)
|
||||
|
||||
|
||||
def _expected_output(
|
||||
geometry: Geometry, topk_ids: torch.Tensor, topk_weights: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
expected = torch.zeros(
|
||||
NUM_TOKENS_PER_RANK,
|
||||
geometry.hidden_size,
|
||||
dtype=torch.float32,
|
||||
device=topk_ids.device,
|
||||
)
|
||||
for t in range(NUM_TOKENS_PER_RANK):
|
||||
for k in range(geometry.top_k):
|
||||
expert = int(topk_ids[t, k])
|
||||
weight = float(topk_weights[t, k])
|
||||
expected[t, expert] += weight * _expert_out(expert) * H_SIGNAL
|
||||
return expected.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _moe_config(
|
||||
vllm_config: VllmConfig,
|
||||
geometry: Geometry,
|
||||
world_size: int,
|
||||
device: torch.device,
|
||||
backend: str,
|
||||
) -> FusedMoEConfig:
|
||||
parallel = FusedMoEParallelConfig.make(
|
||||
tp_size_=1,
|
||||
pcp_size_=1,
|
||||
dp_size_=world_size,
|
||||
sp_size_=1,
|
||||
vllm_parallel_config=vllm_config.parallel_config,
|
||||
)
|
||||
return FusedMoEConfig(
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
num_local_experts=geometry.num_experts // world_size,
|
||||
num_logical_experts=geometry.num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device=device,
|
||||
routing_method=_routing_method(backend),
|
||||
moe_parallel_config=parallel,
|
||||
in_dtype=torch.bfloat16,
|
||||
moe_backend=backend,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
)
|
||||
|
||||
|
||||
def _check_expert_views_alias_kernel_memory(
|
||||
adapter: FlashInferMoeEp,
|
||||
geometry: Geometry,
|
||||
rank: int,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
) -> None:
|
||||
"""EPLB moves experts through ``expert_weight_views``; the kernel must see it.
|
||||
|
||||
Swap local experts 0 and 1 in place on every rank, then route through the
|
||||
swapped physical slots: the output must be the one the unswapped logical
|
||||
experts would have produced.
|
||||
"""
|
||||
for view in adapter.expert_weight_views():
|
||||
first = view[0].clone()
|
||||
view[0].copy_(view[1])
|
||||
view[1].copy_(first)
|
||||
torch.distributed.barrier()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
slot = topk_ids % num_local_experts
|
||||
physical_ids = torch.where(slot == 0, topk_ids + 1, topk_ids)
|
||||
physical_ids = torch.where(slot == 1, topk_ids - 1, physical_ids)
|
||||
output = adapter(_hidden_states(geometry, device), physical_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(output, _expected_output(geometry, topk_ids, topk_weights))
|
||||
|
||||
|
||||
def _init_warmup_forward(
|
||||
env: dict[str, str], world_size: int, geometry: Geometry, backend: str
|
||||
) -> None:
|
||||
set_env_vars_and_device(env)
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.data_parallel_size = world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
vllm_config.kernel_config.moe_backend = backend
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
ensure_model_parallel_initialized(
|
||||
tensor_model_parallel_size=1, pipeline_model_parallel_size=1
|
||||
)
|
||||
rank = torch.distributed.get_rank()
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
moe = _moe_config(vllm_config, geometry, world_size, device, backend)
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
kMxfp4Static,
|
||||
None,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert supported, reason
|
||||
|
||||
adapter = FlashInferMoeEp(
|
||||
moe,
|
||||
_local_expert_weights(geometry, rank, moe.num_local_experts, device),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
try:
|
||||
adapter.warmup()
|
||||
|
||||
topk_ids, topk_weights = _routing(geometry, rank, device)
|
||||
output = adapter(_hidden_states(geometry, device), topk_ids, topk_weights)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = _expected_output(geometry, topk_ids, topk_weights)
|
||||
assert output.dtype == expected.dtype and output.shape == expected.shape
|
||||
assert torch.equal(output, expected), (
|
||||
f"rank {rank}: max abs diff "
|
||||
f"{(output.float() - expected.float()).abs().max().item()}, "
|
||||
f"nonzero output channels per token "
|
||||
f"{[torch.nonzero(row).flatten().tolist() for row in output]}"
|
||||
)
|
||||
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL:
|
||||
_check_expert_views_alias_kernel_memory(
|
||||
adapter, geometry, rank, moe.num_local_experts, device
|
||||
)
|
||||
finally:
|
||||
adapter.destroy()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="The FlashInfer MoE-EP megakernels require SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("geometry", GEOMETRIES, ids=[g.name for g in GEOMETRIES])
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_init_warmup_forward_is_exact_on_grid_inputs(
|
||||
world_size: int, backend: str, geometry: Geometry
|
||||
):
|
||||
pytest.importorskip("flashinfer.moe_ep")
|
||||
if backend == FLASHINFER_MOE_EP_DEEP_GEMM and not has_deep_gemm():
|
||||
pytest.skip("DeepGEMM is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs")
|
||||
distributed_run(_init_warmup_forward, world_size, geometry, backend)
|
||||
@@ -1,116 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the flashinfer moe_ep backend plumbing.
|
||||
|
||||
Everything here runs without a GPU or a flashinfer install: the flashinfer
|
||||
modules the helpers import lazily are replaced with capture fakes.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from typing import Any
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_BACKENDS,
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
MEGA_MOE_BACKENDS,
|
||||
NATIVE_MEGA_MOE_BACKENDS,
|
||||
PASSTHROUGH_ALL2ALL_BACKEND,
|
||||
bind_passthrough_all2all_backend,
|
||||
validate_flashinfer_moe_ep_model,
|
||||
)
|
||||
from vllm.utils.flashinfer_moe_ep import (
|
||||
_E2M1_LUT,
|
||||
FI_MOE_EP_BACKEND_SPECS,
|
||||
_dequant_fp4_ue8m0_gran32,
|
||||
build_fi_mega_config,
|
||||
fi_moe_ep_backend_spec,
|
||||
make_fi_moe_ep_bootstrap,
|
||||
megakernel_runtime_requirements,
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import (
|
||||
FlashInferMoeEpExperts,
|
||||
epilogue_from_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_quant_config,
|
||||
map_mxfp4_backend,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
backend_to_kernel_cls as mxfp4_backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
convert_to_nvfp4_moe_kernel_format,
|
||||
map_nvfp4_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
backend_to_kernel_cls as nvfp4_backend_to_kernel_cls,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
|
||||
PassThroughPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Static,
|
||||
kNvfp4Dynamic,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeBootstrapConfig:
|
||||
world_size: int
|
||||
rank: int
|
||||
process_group: Any = None
|
||||
auto_bootstrap: bool = True
|
||||
device: int | None = field(default=None, kw_only=True)
|
||||
def test_flashinfer_backends_are_megakernels_outside_the_native_model_path():
|
||||
assert FLASHINFER_MOE_EP_BACKENDS <= MEGA_MOE_BACKENDS
|
||||
assert FLASHINFER_MOE_EP_BACKENDS.isdisjoint(NATIVE_MEGA_MOE_BACKENDS)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeDeepGemmMegaMoeConfig:
|
||||
intermediate_size: int
|
||||
top_k: int
|
||||
activation_clamp: float | None
|
||||
fast_math: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeNvfp4CutedslMegaMoeConfig:
|
||||
intermediate_size: int
|
||||
top_k: int
|
||||
activation_clamp: float | None
|
||||
fast_math: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeMegaConfig:
|
||||
megakernel: Any
|
||||
preprocess_weights: bool
|
||||
quantize_input: bool
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_flashinfer(monkeypatch):
|
||||
"""Install a minimal fake flashinfer.moe_ep for the lazy imports."""
|
||||
moe_ep = ModuleType("flashinfer.moe_ep")
|
||||
core = ModuleType("flashinfer.moe_ep.core")
|
||||
runtime = ModuleType("flashinfer.moe_ep.core.runtime")
|
||||
flashinfer = ModuleType("flashinfer")
|
||||
fake_attrs: dict[ModuleType, dict[str, Any]] = {
|
||||
moe_ep: {
|
||||
"BootstrapConfig": _FakeBootstrapConfig,
|
||||
"DeepGemmMegaMoeConfig": _FakeDeepGemmMegaMoeConfig,
|
||||
"Nvfp4CutedslMegaMoeConfig": _FakeNvfp4CutedslMegaMoeConfig,
|
||||
"MegaConfig": _FakeMegaConfig,
|
||||
"core": core,
|
||||
},
|
||||
runtime: {"TORCH_DIST": "torch_dist", "NVSHMEM": "nvshmem"},
|
||||
flashinfer: {"moe_ep": moe_ep},
|
||||
core: {"runtime": runtime},
|
||||
}
|
||||
for mod, attrs in fake_attrs.items():
|
||||
for attr, value in attrs.items():
|
||||
setattr(mod, attr, value)
|
||||
|
||||
for name, mod in {
|
||||
"flashinfer": flashinfer,
|
||||
"flashinfer.moe_ep": moe_ep,
|
||||
"flashinfer.moe_ep.core": core,
|
||||
"flashinfer.moe_ep.core.runtime": runtime,
|
||||
}.items():
|
||||
monkeypatch.setitem(sys.modules, name, mod)
|
||||
return moe_ep
|
||||
|
||||
|
||||
def test_fi_backend_strings_are_registered_mega_moe_backends():
|
||||
assert set(FI_MOE_EP_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
|
||||
assert FLASHINFER_MOE_EP_BACKENDS < MEGA_MOE_BACKENDS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_rejected_for_non_dsv4(moe_backend):
|
||||
"""An FI moe_ep backend with a non-DSv4 model must fail at config time
|
||||
instead of silently falling through to the generic FusedMoE path."""
|
||||
def test_only_deep_gemm_backend_is_dsv4_specific():
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
["MixtralForCausalLM"],
|
||||
)
|
||||
with pytest.raises(ValueError, match="only supported for DeepSeek-V4"):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["MixtralForCausalLM"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(FLASHINFER_MOE_EP_BACKENDS))
|
||||
def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
validate_flashinfer_moe_ep_model(moe_backend, ["DeepseekV4ForCausalLM"])
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
["MixtralForCausalLM"],
|
||||
)
|
||||
validate_flashinfer_moe_ep_model(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
["DeepseekV4ForCausalLM"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -118,8 +92,7 @@ def test_fi_moe_ep_backend_accepted_for_dsv4(moe_backend):
|
||||
[["KimiK3ForConditionalGeneration"], ["MixtralForCausalLM"]],
|
||||
)
|
||||
def test_native_deep_gemm_mega_moe_not_arch_gated(architectures):
|
||||
"""VLLM's own deep_gemm mega path is not DSv4-only (Kimi K3 uses it);
|
||||
models validate their own constraints at construction time."""
|
||||
"""Models validate native deep_gemm mega constraints at construction time."""
|
||||
validate_flashinfer_moe_ep_model("deep_gemm_mega_moe", architectures)
|
||||
|
||||
|
||||
@@ -128,11 +101,7 @@ def test_non_fi_backend_ignores_architectures():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("moe_backend", sorted(MEGA_MOE_BACKENDS))
|
||||
def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
"""Every mega backend must qualify for sequence-parallel MoE at
|
||||
TP>1/EP: the predicate once matched only the native backend string,
|
||||
which silently ran the fi backends full-batch with an all-reduce on
|
||||
every rank — 0.42-0.65x native e2e at TP8."""
|
||||
def test_token_sharding_backends_enable_dsv4_sequence_parallel(moe_backend: str):
|
||||
from vllm.models.deepseek_v4.nvidia.model import _use_sequence_parallel
|
||||
|
||||
vllm_config = SimpleNamespace(
|
||||
@@ -147,113 +116,262 @@ def test_all_mega_backends_get_sequence_parallel_moe(moe_backend):
|
||||
assert _use_sequence_parallel(vllm_config)
|
||||
|
||||
|
||||
def test_fi_moe_ep_backend_spec_kernel_and_nvshmem_contract():
|
||||
dg = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
|
||||
assert dg.megakernel == "deep_gemm_mega"
|
||||
assert not dg.needs_nvshmem
|
||||
|
||||
cd = fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
|
||||
assert cd.megakernel == "nvfp4_cutedsl"
|
||||
assert cd.needs_nvshmem
|
||||
|
||||
with pytest.raises(ValueError, match="not a flashinfer moe_ep backend"):
|
||||
fi_moe_ep_backend_spec("deep_gemm_mega_moe")
|
||||
def test_dsv4_requests_pre_fc2_router_weight_placement():
|
||||
dsv4 = SimpleNamespace(routing_method=RoutingMethodType.DeepseekV4)
|
||||
default = SimpleNamespace(routing_method=RoutingMethodType.Default)
|
||||
assert fi_ep.apply_topk_in_fc1(dsv4)
|
||||
assert not fi_ep.apply_topk_in_fc1(default)
|
||||
|
||||
|
||||
def test_megakernel_runtime_requirements(fake_flashinfer):
|
||||
dg = megakernel_runtime_requirements(
|
||||
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_deep_gemm")
|
||||
def test_backend_specs_preserve_weight_format_contracts():
|
||||
cutedsl = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert cutedsl.kernel == "cutedsl"
|
||||
assert cutedsl.weight_formats == frozenset({"nvfp4", "mxfp4"})
|
||||
|
||||
deep_gemm = fi_ep.flashinfer_moe_ep_backend_spec(FLASHINFER_MOE_EP_DEEP_GEMM)
|
||||
assert deep_gemm.kernel == "deep_gemm"
|
||||
assert deep_gemm.weight_formats == frozenset({"mxfp4"})
|
||||
|
||||
|
||||
def _megakernel_moe(moe_backend: str, **overrides) -> SimpleNamespace:
|
||||
fields = dict(
|
||||
moe_backend=moe_backend,
|
||||
moe_parallel_config=SimpleNamespace(
|
||||
use_ep=True, use_batched_activation_format=False
|
||||
),
|
||||
is_act_and_mul=True,
|
||||
activation=MoEActivation.SILU,
|
||||
has_hash_routing=False,
|
||||
routing_method=RoutingMethodType.DeepseekV4,
|
||||
router_logits_dtype=torch.float32,
|
||||
hidden_dim=256,
|
||||
is_lora_enabled=False,
|
||||
skip_final_all_reduce=False,
|
||||
in_dtype=torch.bfloat16,
|
||||
has_bias=False,
|
||||
swiglu_alpha=None,
|
||||
swiglu_beta=None,
|
||||
)
|
||||
assert dg == frozenset({"torch_dist"})
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
cd = megakernel_runtime_requirements(
|
||||
fi_moe_ep_backend_spec("flashinfer_moe_ep_mega_cutedsl")
|
||||
|
||||
def _is_supported(moe, weight_key, activation_key) -> tuple[bool, str]:
|
||||
supported, reason = FlashInferMoeEpExperts.is_supported_config(
|
||||
FlashInferMoeEpExperts,
|
||||
moe,
|
||||
weight_key,
|
||||
activation_key,
|
||||
mk.FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
assert cd == frozenset({"torch_dist", "nvshmem"})
|
||||
return supported, reason or ""
|
||||
|
||||
|
||||
def test_bootstrap_pins_the_device_vllm_bound(fake_flashinfer, monkeypatch):
|
||||
"""The runtime must not rederive the device from LOCAL_RANK/rank: under a
|
||||
remapped CUDA_VISIBLE_DEVICES that ordinal points at the wrong GPU
|
||||
(CUDA_ERROR_ILLEGAL_ADDRESS in the weight transforms). vLLM passes the
|
||||
device it already bound via BootstrapConfig.device."""
|
||||
import vllm.utils.flashinfer_moe_ep as mod
|
||||
|
||||
pg = object()
|
||||
@pytest.fixture
|
||||
def megakernel_host(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace:
|
||||
"""A vLLM config without DBO/EPLB/weight transfer, on a supported device."""
|
||||
config = SimpleNamespace(
|
||||
weight_transfer_config=None,
|
||||
parallel_config=SimpleNamespace(enable_dbo=False, enable_eplb=False),
|
||||
)
|
||||
monkeypatch.setattr(fi_ep, "get_current_vllm_config", lambda: config)
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(world_size=4, rank_in_group=2, device_group=pg),
|
||||
FlashInferMoeEpExperts, "_supports_current_device", staticmethod(lambda: True)
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 3)
|
||||
|
||||
bootstrap = make_fi_moe_ep_bootstrap()
|
||||
|
||||
assert bootstrap.world_size == 4
|
||||
assert bootstrap.rank == 2
|
||||
assert bootstrap.process_group is pg
|
||||
assert bootstrap.auto_bootstrap is False
|
||||
assert bootstrap.device == 3
|
||||
return config
|
||||
|
||||
|
||||
def test_build_fi_mega_config_selects_kernel_config(fake_flashinfer):
|
||||
dg = build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=7.0,
|
||||
megakernel="deep_gemm_mega",
|
||||
def test_megakernel_requires_expert_parallel(megakernel_host):
|
||||
"""The oracle rejects the megakernel through the experts' support check."""
|
||||
moe = _megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert _is_supported(moe, kNvfp4Static, kNvfp4Dynamic) == (True, "")
|
||||
|
||||
moe.moe_parallel_config.use_ep = False
|
||||
supported, reason = _is_supported(moe, kNvfp4Static, kNvfp4Dynamic)
|
||||
assert not supported and "parallel config" in reason
|
||||
|
||||
|
||||
def test_megakernel_reports_its_own_constraints(megakernel_host):
|
||||
"""Constraints the framework does not know about surface as reasons."""
|
||||
_, reason = _is_supported(
|
||||
_megakernel_moe(FLASHINFER_MOE_EP_CUTEDSL), kNvfp4Static, None
|
||||
)
|
||||
assert isinstance(dg.megakernel, _FakeDeepGemmMegaMoeConfig)
|
||||
assert dg.megakernel.intermediate_size == 2048
|
||||
assert dg.megakernel.top_k == 8
|
||||
assert dg.megakernel.activation_clamp == 7.0
|
||||
assert dg.preprocess_weights and dg.quantize_input
|
||||
assert "A16 activations" in reason
|
||||
|
||||
cd = build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=None,
|
||||
megakernel="nvfp4_cutedsl",
|
||||
deep_gemm = _megakernel_moe(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM, routing_method=RoutingMethodType.Renormalize
|
||||
)
|
||||
assert isinstance(cd.megakernel, _FakeNvfp4CutedslMegaMoeConfig)
|
||||
_, reason = _is_supported(deep_gemm, kMxfp4Static, None)
|
||||
assert "routing method" in reason
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported fi_moe_ep megakernel"):
|
||||
build_fi_mega_config(
|
||||
intermediate_size=2048,
|
||||
top_k=8,
|
||||
activation_clamp=None,
|
||||
megakernel="deep_gemm",
|
||||
megakernel_host.parallel_config.enable_eplb = True
|
||||
_, reason = _is_supported(
|
||||
_megakernel_moe(FLASHINFER_MOE_EP_DEEP_GEMM), kMxfp4Static, None
|
||||
)
|
||||
assert reason.endswith("EPLB")
|
||||
|
||||
|
||||
def test_megakernels_are_oracle_backends():
|
||||
"""Quant methods reach the megakernel through the NVFP4/MXFP4 oracles."""
|
||||
nvfp4 = map_nvfp4_backend(FLASHINFER_MOE_EP_CUTEDSL)
|
||||
assert nvfp4 is NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
|
||||
assert nvfp4_backend_to_kernel_cls(nvfp4) == [FlashInferMoeEpExperts]
|
||||
|
||||
assert map_mxfp4_backend(FLASHINFER_MOE_EP_DEEP_GEMM) == [
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
]
|
||||
for backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
assert mxfp4_backend_to_kernel_cls(backend) == [FlashInferMoeEpExperts]
|
||||
# The megakernel takes the model's own dimensions.
|
||||
assert mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
backend, 2880, 2880
|
||||
) == (2880, 2880)
|
||||
|
||||
quant_config = make_mxfp4_moe_quant_config(
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL, w1_scale=None, w2_scale=None
|
||||
)
|
||||
assert quant_config is not None and quant_config.use_nvfp4_w4a4
|
||||
|
||||
|
||||
def test_megakernel_is_a_modular_kernel_with_pass_through_stages():
|
||||
"""The megakernel plugs into ``FusedMoEKernel`` like any other experts impl.
|
||||
|
||||
Routing stays outside (top-k ids in), dispatch and combine happen inside,
|
||||
so prepare passes tokens through untouched and finalize has nothing to do.
|
||||
"""
|
||||
moe = make_dummy_moe_config(
|
||||
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
|
||||
kernel = mk.FusedMoEKernel(
|
||||
PassThroughPrepareAndFinalize(),
|
||||
FlashInferMoeEpExperts(moe, quant_config),
|
||||
)
|
||||
|
||||
assert not kernel.is_monolithic
|
||||
assert kernel.prepare_finalize.output_is_reduced()
|
||||
# Generic stages: the int32 top-k ids the megakernel wants are its concern.
|
||||
assert kernel.prepare_finalize.topk_indices_dtype() is None
|
||||
assert kernel.fused_experts.expects_unquantized_inputs
|
||||
assert isinstance(
|
||||
kernel.fused_experts.finalize_weight_and_reduce_impl(), TopKWeightAndReduceNoOP
|
||||
)
|
||||
assert kernel.fused_experts.workspace_shapes(
|
||||
16, 256, 256, 2, 8, 8, None, MoEActivation.SILU
|
||||
) == ((0,), (0,), (16, 256))
|
||||
|
||||
tokens = torch.zeros(4, 256, dtype=torch.bfloat16)
|
||||
prepared = kernel.prepare_finalize.prepare(
|
||||
tokens,
|
||||
torch.ones(4, 2),
|
||||
torch.zeros(4, 2, dtype=torch.int32),
|
||||
8,
|
||||
None,
|
||||
False,
|
||||
quant_config,
|
||||
)
|
||||
assert prepared[0] is tokens and all(item is None for item in prepared[1:])
|
||||
|
||||
|
||||
def test_epilogue_takes_the_weight_global_scales_from_the_quant_config():
|
||||
"""The canonical ``g1/g2_alphas`` become the per-expert fc1/fc2 alphas and
|
||||
keep aliasing the layer's tensors, so EPLB permutations reach the kernel."""
|
||||
g1_alphas = torch.tensor([1.0, 2.0])
|
||||
g2_alphas = torch.tensor([3.0, 4.0])
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=g1_alphas,
|
||||
g2_alphas=g2_alphas,
|
||||
a1_gscale=torch.ones(2),
|
||||
a2_gscale=torch.ones(2),
|
||||
w1_scale=torch.ones(2, 4, 1),
|
||||
w2_scale=torch.ones(2, 2, 1),
|
||||
)
|
||||
|
||||
epilogue = epilogue_from_quant_config(quant_config)
|
||||
|
||||
assert epilogue.fc1_alpha is not None and epilogue.fc2_alpha is not None
|
||||
assert epilogue.fc1_alpha.data_ptr() == g1_alphas.data_ptr()
|
||||
assert epilogue.fc2_alpha.data_ptr() == g2_alphas.data_ptr()
|
||||
# Activations are quantized dynamically: no norm constants.
|
||||
assert epilogue.input_norm_const == 1.0 and epilogue.fc1_norm_const is None
|
||||
|
||||
|
||||
def test_epilogue_rejects_unloaded_weight_global_scales():
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=torch.tensor([1.0, float("nan")]),
|
||||
g2_alphas=torch.ones(2),
|
||||
a1_gscale=torch.ones(2),
|
||||
a2_gscale=torch.ones(2),
|
||||
w1_scale=torch.ones(2, 4, 1),
|
||||
w2_scale=torch.ones(2, 2, 1),
|
||||
)
|
||||
with pytest.raises(ValueError, match="g1_alphas"):
|
||||
epilogue_from_quant_config(quant_config)
|
||||
|
||||
|
||||
def test_mxfp4_checkpoints_keep_the_default_epilogue():
|
||||
"""MXFP4 has no per-tensor global scales; the megakernel needs no alphas."""
|
||||
quant_config = FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
|
||||
assert epilogue_from_quant_config(quant_config) == fi_ep.FlashInferMoeEpEpilogue()
|
||||
|
||||
|
||||
def test_nvfp4_kernel_format_passes_megakernel_weights_through():
|
||||
"""The megakernel takes the canonical tensors as they are; the oracle must
|
||||
not reject or reshape them."""
|
||||
tensors = tuple(torch.arange(4.0).reshape(2, 2) + i for i in range(8))
|
||||
converted = convert_to_nvfp4_moe_kernel_format(
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
SimpleNamespace(),
|
||||
*tensors,
|
||||
is_act_and_mul=True,
|
||||
)
|
||||
assert all(out is src for out, src in zip(converted, tensors))
|
||||
|
||||
|
||||
def test_passthrough_all2all_backend_selects_the_pass_through_stages():
|
||||
"""``all2all_backend="passthrough"`` is how the oracles get the
|
||||
pass-through prepare/finalize."""
|
||||
moe = make_dummy_moe_config(
|
||||
num_experts=8, experts_per_token=2, hidden_dim=256, intermediate_size=128
|
||||
)
|
||||
moe.moe_parallel_config = replace(
|
||||
moe.moe_parallel_config, all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND
|
||||
)
|
||||
assert not moe.moe_parallel_config.use_all2all_kernels
|
||||
|
||||
prepare_finalize = maybe_make_prepare_finalize(
|
||||
moe, FusedMoEQuantConfig.make("nvfp4", weight_dtype="nvfp4")
|
||||
)
|
||||
|
||||
assert isinstance(prepare_finalize, PassThroughPrepareAndFinalize)
|
||||
|
||||
|
||||
def test_passthrough_all2all_backend_is_not_a_cli_choice():
|
||||
"""Users cannot pick the passthrough all2all backend explicitly."""
|
||||
parser = EngineArgs.add_cli_args(FlexibleArgumentParser())
|
||||
with pytest.raises(SystemExit):
|
||||
parser.parse_args(["--all2all-backend", PASSTHROUGH_ALL2ALL_BACKEND])
|
||||
parser.parse_args(["--all2all-backend", "deepep_low_latency"])
|
||||
|
||||
|
||||
def test_megakernel_moe_backend_binds_the_all2all_backend():
|
||||
"""A megakernel deployment gets its all2all backend by default."""
|
||||
parallel = SimpleNamespace(all2all_backend="allgather_reducescatter")
|
||||
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
|
||||
assert parallel.all2all_backend == PASSTHROUGH_ALL2ALL_BACKEND
|
||||
bind_passthrough_all2all_backend(FLASHINFER_MOE_EP_CUTEDSL, parallel)
|
||||
|
||||
with pytest.raises(ValueError, match="cannot be paired"):
|
||||
bind_passthrough_all2all_backend(
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
SimpleNamespace(all2all_backend="deepep_low_latency"),
|
||||
)
|
||||
with pytest.raises(ValueError, match="only valid with"):
|
||||
bind_passthrough_all2all_backend(
|
||||
"flashinfer_trtllm",
|
||||
SimpleNamespace(all2all_backend=PASSTHROUGH_ALL2ALL_BACKEND),
|
||||
)
|
||||
|
||||
|
||||
def test_ckpt_uses_nvfp4_experts_reads_moe_quant_algo():
|
||||
from vllm.models.deepseek_v4.nvidia.fi_moe import ckpt_uses_nvfp4_experts
|
||||
|
||||
nvfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo="NVFP4"))
|
||||
assert ckpt_uses_nvfp4_experts(nvfp4)
|
||||
|
||||
mxfp4 = SimpleNamespace(quant_config=SimpleNamespace(moe_quant_algo=None))
|
||||
assert not ckpt_uses_nvfp4_experts(mxfp4)
|
||||
|
||||
no_algo = SimpleNamespace(quant_config=SimpleNamespace())
|
||||
assert not ckpt_uses_nvfp4_experts(no_algo)
|
||||
|
||||
|
||||
def test_dequant_fp4_ue8m0_gran32_decodes_lut_and_scales():
|
||||
"""One 32-element scale group per row: low nibble is the even element,
|
||||
high nibble the odd one, ue8m0 scale applies to the whole group."""
|
||||
packed = torch.arange(32, dtype=torch.uint8).reshape(2, 16)
|
||||
sf = torch.tensor([[127], [128]], dtype=torch.uint8) # 2**0, 2**1
|
||||
|
||||
out = _dequant_fp4_ue8m0_gran32(packed, sf)
|
||||
|
||||
assert out.shape == (2, 32)
|
||||
assert out.dtype == torch.bfloat16
|
||||
expected = torch.empty(2, 32)
|
||||
for row in range(2):
|
||||
for col in range(16):
|
||||
byte = int(packed[row, col])
|
||||
expected[row, 2 * col] = _E2M1_LUT[byte & 0x0F]
|
||||
expected[row, 2 * col + 1] = _E2M1_LUT[byte >> 4]
|
||||
expected[row] *= 2.0**row
|
||||
assert torch.equal(out, expected.to(torch.bfloat16))
|
||||
untouched = SimpleNamespace(all2all_backend="deepep_low_latency")
|
||||
bind_passthrough_all2all_backend("flashinfer_trtllm", untouched)
|
||||
assert untouched.all2all_backend == "deepep_low_latency"
|
||||
|
||||
@@ -657,7 +657,7 @@ def test_deepseek_v4_mega_moe_does_not_double_add_fused_shared_expert(
|
||||
|
||||
moe = DeepseekV4MoE.__new__(DeepseekV4MoE)
|
||||
torch.nn.Module.__init__(moe)
|
||||
moe.use_mega_moe = True
|
||||
moe.use_native_mega_moe = True
|
||||
moe.gate = FakeGate()
|
||||
moe.experts = FakeExperts()
|
||||
moe.shared_experts = FakeSharedExperts()
|
||||
|
||||
@@ -241,7 +241,7 @@ def test_v41_dspark_loads_linear_scales(
|
||||
linear_scale_name=scale_name,
|
||||
pad_shared_expert=False,
|
||||
model=SimpleNamespace(
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_mega_moe=False))],
|
||||
layers=[SimpleNamespace(ffn=SimpleNamespace(use_native_mega_moe=False))],
|
||||
confidence_head=None,
|
||||
),
|
||||
named_parameters=lambda: [(runtime_name, param)],
|
||||
|
||||
Reference in New Issue
Block a user