[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-25 15:38:29 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 24928a4d62
commit 378504a544
26 changed files with 652 additions and 117 deletions
+122 -2
View File
@@ -1,14 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
from dataclasses import dataclass
from dataclasses import dataclass, replace
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.moe.utils import (
check_deferred_moe_finalize,
make_dummy_moe_config,
make_test_weights,
)
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -32,11 +37,20 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
@@ -767,3 +781,109 @@ def test_trtllm_fp8_swiglu_clamp_support(
assert supported == expected, reason
if not expected:
assert "SwiGLU" in reason
def _make_mxfp8_moe_weights(
e: int, n: int, k: int
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q, s = zip(
*(mxfp8_e4m3_quantize(w[i], is_sf_swizzled_layout=False) for i in range(e))
)
return torch.stack(q), torch.stack(s)
w1, w1_scale = quantize(
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
)
w2, w2_scale = quantize(
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
)
return w1, w2, w1_scale, w2_scale
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
@pytest.mark.parametrize("m", [1, 16])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_fp8_block_moe_deferred_finalize(
m: int, block_shape: list[int], workspace_init
):
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
finalize to the caller."""
e, topk, n, k = 32, 4, 1024, 1024
set_random_seed(7)
with set_current_vllm_config(vllm_config):
if block_shape == [1, 32]:
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k)
else:
(_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights(
e, n, k, quant_dtype=torch.float8_e4m3fn, block_shape=block_shape
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.FLASHINFER_TRTLLM,
SimpleNamespace(
weight_block_size=block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=True, intermediate_size_per_partition=n
),
activation=MoEActivation.SILU,
),
w1,
w2,
w1_scale,
w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
quant_config = make_fp8_moe_quant_config(
Fp8MoeBackend.FLASHINFER_TRTLLM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=2 * n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config, quant_config=quant_config, allow_new_interface=True
),
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=a,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
)
+149 -1
View File
@@ -7,7 +7,7 @@ from dataclasses import dataclass, replace
import pytest
import torch
from tests.kernels.moe.utils import check_accuracy
from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize
from vllm._aiter_ops import (
is_aiter_found,
is_aiter_found_and_supported,
@@ -663,6 +663,154 @@ def test_trtllm_gen_mxfp4_fused_moe(
check_accuracy(ref_result, tg_result, atol=0, rtol=0.3, percent=0.8)
@pytest.mark.parametrize(
"monolithic,num_tokens,chunk_size",
[(False, 1, None), (False, 16, None), (False, 16, 5), (True, 16, None)],
)
@pytest.mark.skipif(
not TRTLLM_GEN_MXFP4_AVAILABLE,
reason="nvidia gpu and compute capability sm100 is required for this test",
)
def test_trtllm_mxfp4_deferred_finalize(
monolithic: bool,
num_tokens: int,
chunk_size: int | None,
monkeypatch: pytest.MonkeyPatch,
workspace_init,
):
"""TRTLLM-Gen MXFP4 experts can leave the top-k finalize to the caller."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsModular,
TrtLlmMxfp4ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
)
num_experts, topk, hidden_size, intermediate_size = 32, 4, 1024, 512
device = torch.device("cuda:0")
torch.manual_seed(0)
if chunk_size is not None:
monkeypatch.setattr(
TrtLlmMxfp4ExpertsModular,
"_max_supported_tokens",
lambda self, top_k, global_num_experts: chunk_size,
)
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
w_q, w_scale = fp4_quantize(
w,
torch.tensor(1.0, device=device),
32,
sf_use_ue8m0=True,
is_sf_swizzled_layout=False,
)
return (
w_q.reshape(*w.shape[:-1], -1),
w_scale.view(torch.uint8).reshape(*w.shape[:-1], -1),
)
w13, w13_scale = quantize(
torch.randn(
num_experts,
2 * intermediate_size,
hidden_size,
device=device,
dtype=torch.bfloat16,
)
)
w2, w2_scale = quantize(
torch.randn(
num_experts,
hidden_size,
intermediate_size,
device=device,
dtype=torch.bfloat16,
)
)
backend = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
w13, w2, w13_scale, w2_scale, _, _ = convert_weight_to_mxfp4_moe_kernel_format(
backend,
torch.nn.Module(),
w13,
w2,
w13_scale,
w2_scale,
_cache_permute_indices={},
)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=hidden_size,
intermediate_size=2 * intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
activation=MoEActivation.SILU,
device=device,
routing_method=RoutingMethodType.Renormalize,
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=torch.bfloat16,
)
kernel = make_mxfp4_moe_kernel(
make_mxfp4_moe_quant_config(backend, w1_scale=w13_scale, w2_scale=w2_scale),
moe_config,
TrtLlmMxfp4ExpertsMonolithic if monolithic else TrtLlmMxfp4ExpertsModular,
backend,
)
hidden_states = torch.randn(
num_tokens, hidden_size, device=device, dtype=torch.bfloat16
)
if monolithic:
router_logits = torch.randn(num_tokens, num_experts, device=device)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply_monolithic(
hidden_states,
w13,
w2,
router_logits,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
),
)
return
topk_ids = torch.rand(num_tokens, num_experts, device=device).argsort(dim=-1)
topk_ids = topk_ids[:, :topk].to(torch.int32)
topk_weights = torch.rand(num_tokens, topk, device=device).softmax(dim=-1)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=hidden_states,
w1=w13,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
chunked=chunk_size is not None,
)
@pytest.mark.parametrize("topk", [1, 4])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("num_tokens", [1, 128])
+102 -28
View File
@@ -8,11 +8,13 @@ and GELU — including a Gemma4-shaped case (128 experts, top-k 8,
intermediate_size 704) that exercises the non-256-aligned padding path.
"""
from dataclasses import replace
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_test_quant_config
from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
@@ -31,6 +33,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
@@ -92,6 +95,36 @@ SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
SITU = op_registry[_SITU_OP_NAME]
def _make_trtllm_fp4_moe_kernel(
moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig
) -> mk.FusedMoEKernel:
experts = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config, quant_config=quant_config
)
# Mimic the production weight-loader path so per-expert tensors that
# are normally precomputed in process_weights_after_loading (g1_scale_c
# and the rescaled gemm1_clamp_limit) get materialized. The test's
# synthetic quant_config has g1_alphas/g2_alphas already at their
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
# (same tensor) and use input_scale=1 to make the in-place
# weight_scale_2 *= input_scale step a no-op.
fake_layer = torch.nn.Module()
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
experts.process_weights_after_loading(fake_layer)
return mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts,
)
ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
@@ -191,34 +224,11 @@ def test_trtllm_fp4_moe_no_graph(
),
)
trtllm_inner = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config, quant_config=quant_config
)
# Mimic the production weight-loader path so per-expert tensors that
# are normally precomputed in process_weights_after_loading (g1_scale_c
# and the rescaled gemm1_clamp_limit) get materialized. The test's
# synthetic quant_config has g1_alphas/g2_alphas already at their
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
# (same tensor) and use input_scale=1 to make the in-place
# weight_scale_2 *= input_scale step a no-op.
fake_layer = torch.nn.Module()
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
trtllm_inner.process_weights_after_loading(fake_layer)
trtllm_experts = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
if activation == MoEActivation.SITU:
torch.testing.assert_close(trtllm_inner.g1_scale_c, quant_config.a2_gscale)
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
trtllm_inner,
)
torch.testing.assert_close(
trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale
)
trtllm_output = trtllm_experts.apply(
hidden_states=a,
@@ -331,6 +341,70 @@ def test_trtllm_fp4_moe_reprocess_does_not_refold_swiglu_params():
torch.testing.assert_close(experts.gemm1_beta, beta * expected)
@pytest.mark.parametrize("m,chunk_size", [(1, None), (16, None), (16, 5)])
@torch.inference_mode()
def test_trtllm_fp4_moe_deferred_finalize(
m: int,
chunk_size: int | None,
monkeypatch: pytest.MonkeyPatch,
workspace_init,
):
"""TRTLLM-Gen NvFP4 modular experts can leave the top-k finalize to the caller."""
e, topk, n, k = 128, 8, 1024, 1024
dtype = torch.bfloat16
if chunk_size is not None:
monkeypatch.setattr(
TrtLlmNvFp4ExpertsModular, "_get_chunk_size", lambda self: chunk_size
)
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1_q, w2_q, quant_config = make_test_quant_config(
e, n, k, in_dtype=dtype, quant_dtype="nvfp4", is_scale_swizzled=False
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=2 * n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
chunked=chunk_size is not None,
)
if __name__ == "__main__":
test_trtllm_fp4_moe_no_graph(
64,
+48
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
@@ -28,6 +29,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
BatchedPrepareAndFinalize,
)
@@ -707,6 +710,51 @@ def check_accuracy(a, b, atol, rtol, percent):
)
def check_deferred_moe_finalize(
moe_config: FusedMoEConfig,
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
router_weights: torch.Tensor | None = None,
chunked: bool = False,
) -> None:
"""Check a kernel that defers its finalize against the finalize it skips.
``run`` calls the kernel on fixed inputs, first as built and then with
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
report truthfully. A deferred output reduced by ``moe_unpermute``, the
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
bit, and modular experts hand the router's weights back as-is. A call the
experts split across kernel launches finalizes instead.
"""
# The deferred output views the router's buffer, so compare with a copy.
expected_weights = None if router_weights is None else router_weights.clone()
finalized = run()
assert isinstance(finalized, torch.Tensor)
moe_config.defer_moe_finalize()
output = run()
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
if chunked:
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
return
assert isinstance(output, UnfinalizedMoEOutput)
if expected_weights is not None:
torch.testing.assert_close(
output.expert_weights, expected_weights, atol=0, rtol=0
)
reference = torch.empty_like(finalized)
moe_unpermute(
reference,
output.gemm2_permuted,
output.expert_weights.float(),
output.expanded_idx_to_permuted_idx,
# The kernel reads its valid-row count through this pointer either way.
expert_first_token_offset=output.gemm2_permuted.new_full(
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
),
)
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
+6 -6
View File
@@ -15,6 +15,7 @@ from tests.utils import (
multi_process_parallel,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsMonolithic,
)
@@ -43,12 +44,11 @@ def test_deferred_finalize_enabled_before_moe_kernel_setup(
hidden_dim = LATENT_SIZE
hidden_dim_unpadded = LATENT_SIZE
experts_per_token = 16
defer_moe_finalize = False
_defer_moe_finalize = False
defer_moe_finalize_max_num_tokens = -1
@property
def use_deferred_moe_finalize(self) -> bool:
return self.defer_moe_finalize
defer_moe_finalize = FusedMoEConfig.defer_moe_finalize
limit_deferred_moe_finalize = FusedMoEConfig.limit_deferred_moe_finalize
use_deferred_moe_finalize = FusedMoEConfig.use_deferred_moe_finalize
moe_config = FakeMoEConfig()
quant_method = SimpleNamespace(
@@ -99,7 +99,7 @@ def test_deferred_finalize_enabled_before_moe_kernel_setup(
latent_moe_runner.LatentMoERunner()
assert moe_config.defer_moe_finalize
assert moe_config.use_deferred_moe_finalize
assert moe_config.defer_moe_finalize_max_num_tokens == 128
assert initialized_with["experts_per_token"] == 16