[Bugfix] Drop enable_in_kernel_fc2_reduce from FlashInfer CuteDSL MegaMoE config (#59900)
Signed-off-by: Shang Wang <samshang.wang@mail.utoronto.ca> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a281ad32f0
commit
716961cf1f
@@ -10,9 +10,14 @@ CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
|
||||
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
|
||||
32). The expected output is then known in closed form and compared bit for
|
||||
bit.
|
||||
|
||||
A test that needs no SM100 also checks the adapter's kwargs against the
|
||||
installed FlashInfer's config types, so config drift fails on any CI runner
|
||||
with FlashInfer installed.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -22,6 +27,7 @@ from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_devic
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
|
||||
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -305,3 +311,59 @@ def test_init_warmup_forward_is_exact_on_grid_inputs(
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs")
|
||||
distributed_run(_init_warmup_forward, world_size, geometry, backend)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
def test_adapter_kwargs_match_installed_flashinfer(
|
||||
monkeypatch: pytest.MonkeyPatch, backend: str
|
||||
):
|
||||
"""The adapter's kwargs construct the installed FlashInfer's config types.
|
||||
|
||||
Needs no SM100, so config kwarg drift fails in CI rather than at model load.
|
||||
Fakes only the megakernel layer (symmetric memory), the EP group, the device
|
||||
index and the DeepGEMM import.
|
||||
"""
|
||||
moe_ep = pytest.importorskip("flashinfer.moe_ep")
|
||||
layer_args: list[tuple] = []
|
||||
|
||||
class FakeMegaLayer:
|
||||
def __init__(self, *args) -> None:
|
||||
layer_args.append(args)
|
||||
|
||||
def destroy(self) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(moe_ep, "MoEEpMegaLayer", FakeMegaLayer)
|
||||
monkeypatch.setattr(fi_ep, "_expose_deep_gemm_to_flashinfer", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
fi_ep,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(world_size=2, rank_in_group=0, device_group=None),
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0)
|
||||
|
||||
geometry = GEOMETRIES[0]
|
||||
moe = SimpleNamespace(
|
||||
moe_backend=backend,
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
swiglu_limit=10.0,
|
||||
routing_method=_routing_method(backend),
|
||||
)
|
||||
weight = torch.zeros(1, dtype=torch.bfloat16)
|
||||
FlashInferMoeEp(
|
||||
moe,
|
||||
FlashInferMoeEpWeights(w13=weight, w2=weight),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
|
||||
((_, _, _, mega_config),) = layer_args
|
||||
expected = (
|
||||
moe_ep.Nvfp4CutedslMegaMoeConfig
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL
|
||||
else moe_ep.DeepGemmMegaMoeConfig
|
||||
)
|
||||
assert isinstance(mega_config.megakernel, expected)
|
||||
|
||||
@@ -315,13 +315,14 @@ class FlashInferMoeEp:
|
||||
w2_scale=weights.w2_scale,
|
||||
)
|
||||
if spec.kernel == "cutedsl":
|
||||
# In-kernel fc2 reduce keeps its default (off); FlashInfer 0.7.1rc1
|
||||
# renamed in_kernel_fc2_reduce to enable_in_kernel_fc2_reduce.
|
||||
megakernel = api.Nvfp4CutedslMegaMoeConfig(
|
||||
intermediate_size=moe.intermediate_size,
|
||||
top_k=moe.experts_per_token,
|
||||
gate_up_clamp=moe.swiglu_limit,
|
||||
fast_math=True,
|
||||
apply_topk_in_fc1=apply_topk_in_fc1,
|
||||
enable_in_kernel_fc2_reduce=False,
|
||||
combine_dtype="bf16",
|
||||
input_norm_const=epilogue.input_norm_const,
|
||||
fc1_alpha=None,
|
||||
|
||||
Reference in New Issue
Block a user