[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-25 15:38:29 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 24928a4d62
commit 378504a544
26 changed files with 652 additions and 117 deletions
+122 -2
View File
@@ -1,14 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
from dataclasses import dataclass
from dataclasses import dataclass, replace
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_dummy_moe_config
from tests.kernels.moe.utils import (
check_deferred_moe_finalize,
make_dummy_moe_config,
make_test_weights,
)
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
@@ -32,11 +37,20 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
TrtLlmFp8ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
Fp8MoeBackend,
convert_to_fp8_moe_kernel_format,
make_fp8_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
swap_w13_to_w31,
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8Dynamic128Sym,
@@ -767,3 +781,109 @@ def test_trtllm_fp8_swiglu_clamp_support(
assert supported == expected, reason
if not expected:
assert "SwiGLU" in reason
def _make_mxfp8_moe_weights(
e: int, n: int, k: int
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q, s = zip(
*(mxfp8_e4m3_quantize(w[i], is_sf_swizzled_layout=False) for i in range(e))
)
return torch.stack(q), torch.stack(s)
w1, w1_scale = quantize(
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
)
w2, w2_scale = quantize(
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
)
return w1, w2, w1_scale, w2_scale
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
@pytest.mark.parametrize("m", [1, 16])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_fp8_block_moe_deferred_finalize(
m: int, block_shape: list[int], workspace_init
):
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
finalize to the caller."""
e, topk, n, k = 32, 4, 1024, 1024
set_random_seed(7)
with set_current_vllm_config(vllm_config):
if block_shape == [1, 32]:
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k)
else:
(_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights(
e, n, k, quant_dtype=torch.float8_e4m3fn, block_shape=block_shape
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.FLASHINFER_TRTLLM,
SimpleNamespace(
weight_block_size=block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=True, intermediate_size_per_partition=n
),
activation=MoEActivation.SILU,
),
w1,
w2,
w1_scale,
w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
quant_config = make_fp8_moe_quant_config(
Fp8MoeBackend.FLASHINFER_TRTLLM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=2 * n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config, quant_config=quant_config, allow_new_interface=True
),
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=a,
w1=w1,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
)
+149 -1
View File
@@ -7,7 +7,7 @@ from dataclasses import dataclass, replace
import pytest
import torch
from tests.kernels.moe.utils import check_accuracy
from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize
from vllm._aiter_ops import (
is_aiter_found,
is_aiter_found_and_supported,
@@ -663,6 +663,154 @@ def test_trtllm_gen_mxfp4_fused_moe(
check_accuracy(ref_result, tg_result, atol=0, rtol=0.3, percent=0.8)
@pytest.mark.parametrize(
"monolithic,num_tokens,chunk_size",
[(False, 1, None), (False, 16, None), (False, 16, 5), (True, 16, None)],
)
@pytest.mark.skipif(
not TRTLLM_GEN_MXFP4_AVAILABLE,
reason="nvidia gpu and compute capability sm100 is required for this test",
)
def test_trtllm_mxfp4_deferred_finalize(
monolithic: bool,
num_tokens: int,
chunk_size: int | None,
monkeypatch: pytest.MonkeyPatch,
workspace_init,
):
"""TRTLLM-Gen MXFP4 experts can leave the top-k finalize to the caller."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsModular,
TrtLlmMxfp4ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
)
num_experts, topk, hidden_size, intermediate_size = 32, 4, 1024, 512
device = torch.device("cuda:0")
torch.manual_seed(0)
if chunk_size is not None:
monkeypatch.setattr(
TrtLlmMxfp4ExpertsModular,
"_max_supported_tokens",
lambda self, top_k, global_num_experts: chunk_size,
)
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
w_q, w_scale = fp4_quantize(
w,
torch.tensor(1.0, device=device),
32,
sf_use_ue8m0=True,
is_sf_swizzled_layout=False,
)
return (
w_q.reshape(*w.shape[:-1], -1),
w_scale.view(torch.uint8).reshape(*w.shape[:-1], -1),
)
w13, w13_scale = quantize(
torch.randn(
num_experts,
2 * intermediate_size,
hidden_size,
device=device,
dtype=torch.bfloat16,
)
)
w2, w2_scale = quantize(
torch.randn(
num_experts,
hidden_size,
intermediate_size,
device=device,
dtype=torch.bfloat16,
)
)
backend = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
w13, w2, w13_scale, w2_scale, _, _ = convert_weight_to_mxfp4_moe_kernel_format(
backend,
torch.nn.Module(),
w13,
w2,
w13_scale,
w2_scale,
_cache_permute_indices={},
)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=num_experts,
experts_per_token=topk,
hidden_dim=hidden_size,
intermediate_size=2 * intermediate_size,
num_local_experts=num_experts,
num_logical_experts=num_experts,
activation=MoEActivation.SILU,
device=device,
routing_method=RoutingMethodType.Renormalize,
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=torch.bfloat16,
)
kernel = make_mxfp4_moe_kernel(
make_mxfp4_moe_quant_config(backend, w1_scale=w13_scale, w2_scale=w2_scale),
moe_config,
TrtLlmMxfp4ExpertsMonolithic if monolithic else TrtLlmMxfp4ExpertsModular,
backend,
)
hidden_states = torch.randn(
num_tokens, hidden_size, device=device, dtype=torch.bfloat16
)
if monolithic:
router_logits = torch.randn(num_tokens, num_experts, device=device)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply_monolithic(
hidden_states,
w13,
w2,
router_logits,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
),
)
return
topk_ids = torch.rand(num_tokens, num_experts, device=device).argsort(dim=-1)
topk_ids = topk_ids[:, :topk].to(torch.int32)
topk_weights = torch.rand(num_tokens, topk, device=device).softmax(dim=-1)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=hidden_states,
w1=w13,
w2=w2,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=num_experts,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
chunked=chunk_size is not None,
)
@pytest.mark.parametrize("topk", [1, 4])
@pytest.mark.parametrize("num_experts", [32])
@pytest.mark.parametrize("num_tokens", [1, 128])
+102 -28
View File
@@ -8,11 +8,13 @@ and GELU — including a Gemma4-shaped case (128 experts, top-k 8,
intermediate_size 704) that exercises the non-256-aligned padding path.
"""
from dataclasses import replace
import pytest
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from tests.kernels.moe.utils import make_test_quant_config
from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config
from tests.kernels.quantization.nvfp4_utils import (
FLOAT4_E2M1_MAX,
FLOAT8_E4M3_MAX,
@@ -31,6 +33,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
@@ -92,6 +95,36 @@ SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
SITU = op_registry[_SITU_OP_NAME]
def _make_trtllm_fp4_moe_kernel(
moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig
) -> mk.FusedMoEKernel:
experts = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config, quant_config=quant_config
)
# Mimic the production weight-loader path so per-expert tensors that
# are normally precomputed in process_weights_after_loading (g1_scale_c
# and the rescaled gemm1_clamp_limit) get materialized. The test's
# synthetic quant_config has g1_alphas/g2_alphas already at their
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
# (same tensor) and use input_scale=1 to make the in-place
# weight_scale_2 *= input_scale step a no-op.
fake_layer = torch.nn.Module()
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
experts.process_weights_after_loading(fake_layer)
return mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
experts,
)
ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
@@ -191,34 +224,11 @@ def test_trtllm_fp4_moe_no_graph(
),
)
trtllm_inner = TrtLlmNvFp4ExpertsModular(
moe_config=moe_config, quant_config=quant_config
)
# Mimic the production weight-loader path so per-expert tensors that
# are normally precomputed in process_weights_after_loading (g1_scale_c
# and the rescaled gemm1_clamp_limit) get materialized. The test's
# synthetic quant_config has g1_alphas/g2_alphas already at their
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
# (same tensor) and use input_scale=1 to make the in-place
# weight_scale_2 *= input_scale step a no-op.
fake_layer = torch.nn.Module()
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
trtllm_inner.process_weights_after_loading(fake_layer)
trtllm_experts = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
if activation == MoEActivation.SITU:
torch.testing.assert_close(trtllm_inner.g1_scale_c, quant_config.a2_gscale)
trtllm_experts = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=False,
),
trtllm_inner,
)
torch.testing.assert_close(
trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale
)
trtllm_output = trtllm_experts.apply(
hidden_states=a,
@@ -331,6 +341,70 @@ def test_trtllm_fp4_moe_reprocess_does_not_refold_swiglu_params():
torch.testing.assert_close(experts.gemm1_beta, beta * expected)
@pytest.mark.parametrize("m,chunk_size", [(1, None), (16, None), (16, 5)])
@torch.inference_mode()
def test_trtllm_fp4_moe_deferred_finalize(
m: int,
chunk_size: int | None,
monkeypatch: pytest.MonkeyPatch,
workspace_init,
):
"""TRTLLM-Gen NvFP4 modular experts can leave the top-k finalize to the caller."""
e, topk, n, k = 128, 8, 1024, 1024
dtype = torch.bfloat16
if chunk_size is not None:
monkeypatch.setattr(
TrtLlmNvFp4ExpertsModular, "_get_chunk_size", lambda self: chunk_size
)
set_random_seed(7)
with set_current_vllm_config(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
w1_q, w2_q, quant_config = make_test_quant_config(
e, n, k, in_dtype=dtype, quant_dtype="nvfp4", is_scale_swizzled=False
)
score = torch.randn((m, e), device="cuda", dtype=dtype)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=k,
intermediate_size=2 * n,
num_local_experts=e,
num_logical_experts=e,
activation=MoEActivation.SILU,
device="cuda",
moe_parallel_config=replace(
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
),
in_dtype=dtype,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
hidden_states=a,
w1=w1_q,
w2=w2_q,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=MoEActivation.SILU,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
router_weights=topk_weights,
chunked=chunk_size is not None,
)
if __name__ == "__main__":
test_trtllm_fp4_moe_no_graph(
64,
+48
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
@@ -28,6 +29,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
BatchedPrepareAndFinalize,
)
@@ -707,6 +710,51 @@ def check_accuracy(a, b, atol, rtol, percent):
)
def check_deferred_moe_finalize(
moe_config: FusedMoEConfig,
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
router_weights: torch.Tensor | None = None,
chunked: bool = False,
) -> None:
"""Check a kernel that defers its finalize against the finalize it skips.
``run`` calls the kernel on fixed inputs, first as built and then with
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
report truthfully. A deferred output reduced by ``moe_unpermute``, the
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
bit, and modular experts hand the router's weights back as-is. A call the
experts split across kernel launches finalizes instead.
"""
# The deferred output views the router's buffer, so compare with a copy.
expected_weights = None if router_weights is None else router_weights.clone()
finalized = run()
assert isinstance(finalized, torch.Tensor)
moe_config.defer_moe_finalize()
output = run()
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
if chunked:
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
return
assert isinstance(output, UnfinalizedMoEOutput)
if expected_weights is not None:
torch.testing.assert_close(
output.expert_weights, expected_weights, atol=0, rtol=0
)
reference = torch.empty_like(finalized)
moe_unpermute(
reference,
output.gemm2_permuted,
output.expert_weights.float(),
output.expanded_idx_to_permuted_idx,
# The kernel reads its valid-row count through this pointer either way.
expert_first_token_offset=output.gemm2_permuted.new_full(
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
),
)
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
+6 -6
View File
@@ -15,6 +15,7 @@ from tests.utils import (
multi_process_parallel,
)
from vllm.distributed import get_tp_group
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsMonolithic,
)
@@ -43,12 +44,11 @@ def test_deferred_finalize_enabled_before_moe_kernel_setup(
hidden_dim = LATENT_SIZE
hidden_dim_unpadded = LATENT_SIZE
experts_per_token = 16
defer_moe_finalize = False
_defer_moe_finalize = False
defer_moe_finalize_max_num_tokens = -1
@property
def use_deferred_moe_finalize(self) -> bool:
return self.defer_moe_finalize
defer_moe_finalize = FusedMoEConfig.defer_moe_finalize
limit_deferred_moe_finalize = FusedMoEConfig.limit_deferred_moe_finalize
use_deferred_moe_finalize = FusedMoEConfig.use_deferred_moe_finalize
moe_config = FakeMoEConfig()
quant_method = SimpleNamespace(
@@ -99,7 +99,7 @@ def test_deferred_finalize_enabled_before_moe_kernel_setup(
latent_moe_runner.LatentMoERunner()
assert moe_config.defer_moe_finalize
assert moe_config.use_deferred_moe_finalize
assert moe_config.defer_moe_finalize_max_num_tokens == 128
assert initialized_with["experts_per_token"] == 16
+57 -13
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import IntEnum
from typing import Union
@@ -1290,13 +1290,12 @@ class FusedMoEConfig:
# Only honored on the non-reduced (late-AR) TP path. Default False.
skip_final_all_reduce: bool = False
# When True, experts that can stop after GEMM2 are allowed to hand back an
# UnfinalizedMoEOutput instead of finalized states, leaving the top-k
# reduction to fuse into the consumer. Set by layers that have such a
# consumer; read through `use_deferred_moe_finalize`, which applies the
# guards. Kernels without the capability ignore it. Default False.
defer_moe_finalize: bool = False
# Optional consumer capacity for deferred finalize. Negative means unbounded.
# Requested through `defer_moe_finalize()`, read through
# `should_defer_moe_finalize()`.
_defer_moe_finalize: bool = field(default=False, init=False)
# Most tokens a deferred call covers: the consumer's capacity, lowered by
# experts that would split a larger call across kernel launches. Negative
# means unbounded.
defer_moe_finalize_max_num_tokens: int = -1
# SwiGLU clamp limit. When set, backends that do not implement the clamp
@@ -1422,21 +1421,57 @@ class FusedMoEConfig:
def use_deferred_moe_finalize(self) -> bool:
"""Whether experts may return an unfinalized output on this deployment.
Evaluated on read rather than in ``__post_init__`` because
``defer_moe_finalize`` is set after construction, like
``skip_final_all_reduce``.
Evaluated on read rather than in ``__post_init__`` because deferral is
requested after construction, like ``skip_final_all_reduce``.
"""
# The consumer fuses a TP all-reduce. Other parallel modes require a
# combine or reduce-scatter after the experts and cannot defer it.
# combine or reduce-scatter after the experts and cannot defer it, and
# the consumer has no way to strip hidden-dim padding from GEMM2 rows.
return (
self.defer_moe_finalize
self._defer_moe_finalize
and self.tp_size > 1
and self.dp_size == 1
and self.ep_size == 1
and self.pcp_size == 1
and not self.is_sequence_parallel
and self.hidden_dim == self.hidden_dim_unpadded
)
def defer_moe_finalize(self, max_num_tokens: int = -1) -> None:
"""Ask the experts to leave the top-k reduction to the layer's consumer.
A layer whose consumer can fuse the top-k reduction (e.g. into its TP
all-reduce) calls this once while the model is built, and only when its
experts are TRTLLM-Gen ones that can stop after GEMM2 (the
``do_finalize=False`` path). Other experts ignore the request and
always finalize, which ``should_defer_moe_finalize`` can't see, so the
layer checks the quant method's ``experts_cls`` first, as Kimi-K3 does.
From then on:
- The experts return an ``UnfinalizedMoEOutput`` instead of finalized
states for every call ``should_defer_moe_finalize`` accepts.
- ``should_defer_moe_finalize(num_tokens)`` is the answer for a call:
it requires a TP-only deployment without hidden-dim padding
(``use_deferred_moe_finalize``), a non-empty call and at most
``defer_moe_finalize_max_num_tokens`` tokens. The cap starts at
``max_num_tokens`` and only ever goes down: experts that would split a
larger call across kernel launches lower it to their single-launch
size when they are built, since each launch permutes into its own
buffer.
- The model asks ``should_defer_moe_finalize`` before each call and takes
the matching path. A deferred call runs the runner's ``_forward_impl``
directly, since the MoE custom op returns tensors only, and the
consumer then owns the top-k reduction, the shared-expert add and the
all-reduce.
Args:
max_num_tokens: Most tokens per call the consumer can take in
deferred form. Negative means no limit of its own.
"""
self._defer_moe_finalize = True
self.limit_deferred_moe_finalize(max_num_tokens)
def should_defer_moe_finalize(self, num_tokens: int) -> bool:
"""Return whether this invocation may defer the top-k reduction."""
max_num_tokens = self.defer_moe_finalize_max_num_tokens
@@ -1446,6 +1481,15 @@ class FusedMoEConfig:
and (max_num_tokens < 0 or num_tokens <= max_num_tokens)
)
def limit_deferred_moe_finalize(self, max_num_tokens: int) -> None:
"""Finalize calls above ``max_num_tokens`` even when deferring.
Negative means no limit, and leaves the current one in place.
"""
current = self.defer_moe_finalize_max_num_tokens
if max_num_tokens >= 0 and (current < 0 or max_num_tokens < current):
self.defer_moe_finalize_max_num_tokens = max_num_tokens
@property
def use_deepep_ht_kernels(self):
return self.moe_parallel_config.use_deepep_ht_kernels
@@ -12,6 +12,10 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import (
UnfinalizedMoEOutput,
convert_flashinfer_moe_output,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
@@ -273,7 +277,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
) -> UnfinalizedMoEOutput | None:
import flashinfer
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
@@ -297,7 +301,9 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
weight_layout = WeightLayout.BlockMajorK
hidden_states_scale = prepare_deepseek_fp8_x_sf(hidden_states, a1q_scale)
flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe(
num_tokens = hidden_states.shape[0]
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
flashinfer_output = flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
hidden_states=hidden_states,
@@ -321,9 +327,20 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
use_shuffled_weight=use_shuffled_weight,
weight_layout=weight_layout,
fp8_quantization_type=fp8_quant_type,
output=output,
do_finalize=not defer,
output=None if defer else output,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
if not defer:
return None
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=False,
num_tokens=num_tokens,
top_k=topk_ids.size(1),
)
assert isinstance(routed_output, UnfinalizedMoEOutput)
return routed_output
class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolithic):
@@ -289,6 +289,18 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
Moved from trtllm_moe.py.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
**kwargs,
):
super().__init__(moe_config, quant_config, **kwargs)
# Each launch permutes into its own buffer, so a chunked call finalizes.
moe_config.limit_deferred_moe_finalize(
self._max_supported_tokens(self.topk, moe_config.num_experts)
)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
@@ -343,7 +355,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
def _invoke_kernel(
self,
output: torch.Tensor,
output: torch.Tensor | None,
x_quant: torch.Tensor,
x_scale: torch.Tensor | None,
topk_ids: torch.Tensor,
@@ -355,10 +367,11 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
local_num_experts: int,
local_expert_offset: int,
topk: int,
) -> None:
) -> UnfinalizedMoEOutput | None:
"""Finalize into ``output``, or stop after GEMM2 when it is None."""
from flashinfer import trtllm_fp4_block_scale_routed_moe
trtllm_fp4_block_scale_routed_moe(
flashinfer_output = trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
hidden_states=x_quant,
@@ -386,12 +399,22 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
# Modular kernel receives pre-routed tokens, so routing is already
# done. Use Renormalize as a safe default the TRTLLM kernel supports.
routing_method_type=RoutingMethodType.Renormalize,
do_finalize=True,
do_finalize=output is not None,
enable_pdl=True,
activation_type=self._flashinfer_activation_type(activation),
output=output,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
if output is not None:
return None
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=False,
num_tokens=x_quant.shape[0],
top_k=topk,
)
assert isinstance(routed_output, UnfinalizedMoEOutput)
return routed_output
def apply(
self,
@@ -410,7 +433,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
) -> UnfinalizedMoEOutput | None:
topk_ids = topk_ids.to(dtype=torch.int32)
topk = topk_ids.size(-1)
@@ -428,13 +451,17 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
assert self.w1_scale is not None
assert self.w2_scale is not None
# Chunk tokens so the batched-GEMM grid stays within CUDA limits.
# Chunk tokens so the batched-GEMM grid stays within CUDA limits. Each
# launch permutes into its own buffer, so only a run that fits in one
# launch can leave the top-k reduction to a deferring consumer.
M = x_quant.size(0)
chunk_size = self._max_supported_tokens(topk, global_num_experts)
defer = chunk_size >= M and self.moe_config.should_defer_moe_finalize(M)
unfinalized: UnfinalizedMoEOutput | None = None
for start in range(0, M, chunk_size):
end = min(start + chunk_size, M)
self._invoke_kernel(
output[start:end],
unfinalized = self._invoke_kernel(
None if defer else output[start:end],
x_quant[start:end],
None if x_scale is None else x_scale[start:end],
topk_ids[start:end],
@@ -448,4 +475,4 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
topk,
)
return output
return unfinalized
@@ -302,6 +302,16 @@ class TrtLlmNvFp4ExpertsBase:
class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModular):
"""Modular version of the implementation (just the experts)."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
):
super().__init__(moe_config, quant_config, per_token_activation)
# Each launch permutes into its own buffer, so a chunked call finalizes.
moe_config.limit_deferred_moe_finalize(self._get_chunk_size())
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
"""The modular implementation supports all parallel configs."""
@@ -333,7 +343,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
def _invoke_kernel(
self,
output: torch.Tensor,
output: torch.Tensor | None,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
@@ -342,7 +352,8 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
activation: MoEActivation,
global_num_experts: int,
a1q_scale: torch.Tensor | None,
):
) -> UnfinalizedMoEOutput | None:
"""Finalize into ``output``, or stop after GEMM2 when it is None."""
import flashinfer
assert self.quant_config.w1_scale is not None
@@ -359,7 +370,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
flashinfer_output = flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
hidden_states=hidden_states,
@@ -387,7 +398,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routing_method_type=1, # not used
do_finalize=True,
do_finalize=output is not None,
activation_type=activation_to_flashinfer_int(activation),
per_token_scale=per_token_scale,
output=output,
@@ -395,6 +406,16 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
fi_moe_largest_bucket(self.moe_config), self._get_chunk_size()
),
)
if output is not None:
return None
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=False,
num_tokens=hidden_states.shape[0],
top_k=self.topk,
)
assert isinstance(routed_output, UnfinalizedMoEOutput)
return routed_output
def apply(
self,
@@ -413,7 +434,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
) -> UnfinalizedMoEOutput | None:
assert self._supports_activation(activation)
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
assert a1q_scale is not None or self.per_token_activation
@@ -425,8 +446,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
chunk_size = self._get_chunk_size()
if chunk_size >= M:
self._invoke_kernel(
output,
# Each launch permutes into its own buffer, so only a run that fits
# in one launch can leave the top-k reduction to a deferring
# consumer.
defer = self.moe_config.should_defer_moe_finalize(M)
return self._invoke_kernel(
None if defer else output,
hidden_states,
w1,
w2,
@@ -450,6 +475,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
global_num_experts,
None if a1q_scale is None else a1q_scale[start:end],
)
return None
class TrtLlmNvFp4ExpertsMonolithic(
@@ -148,7 +148,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None",
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Apply the MoE operation using modular kernels.
Args:
@@ -160,7 +160,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
shared_experts: The shared experts module, if any
Returns:
Output tensor from routed experts.
Finalized routed states or a deferred-finalize output.
"""
raise NotImplementedError
@@ -238,6 +238,15 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def supports_deferred_moe_finalize(self) -> bool:
"""Whether ``finalize`` can be skipped for a deferring consumer.
An implementation opts in only if everything it does in ``finalize``
-- the top-k reduction, and any combine or reduce-scatter -- is work
the consumer takes over.
"""
return False
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
@@ -404,9 +413,6 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
described above for the monolithic case.
"""
def supports_deferred_moe_finalize(self) -> bool:
return False
@abstractmethod
def prepare(
self,
@@ -895,10 +901,14 @@ class FusedMoEExpertsModular(FusedMoEExperts):
workspace2: torch.Tensor,
expert_tokens_meta: ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
) -> UnfinalizedMoEOutput | None:
"""This function computes the intermediate result of a Mixture of Experts
(MoE) layer using two sets of weights, w1 and w2.
Writes into `output` and returns None, unless the implementation stopped
after GEMM2 and left the top-k reduction to a fused consumer, in which
case `output` is untouched and the unfinalized result is returned.
Args:
output: (torch.Tensor): The unweighted, unreduced output tensor.
hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
@@ -1270,7 +1280,7 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input: bool,
expert_tokens_meta: ExpertTokensMetadata | None,
output_alias: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
_, M_full, N, K, top_k = self.fused_experts.moe_problem_size(
a1q, w1, w2, topk_ids
)
@@ -1318,7 +1328,7 @@ class FusedMoEKernelModularImpl:
elif use_output_alias:
fused_out = output_alias
self.fused_experts.apply(
unfinalized = self.fused_experts.apply(
output=fused_out,
hidden_states=a1q,
w1=w1,
@@ -1336,6 +1346,11 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input=apply_router_weight_on_input,
)
# Experts that stopped after GEMM2 wrote nothing into `fused_out`; the
# top-k reduction they left open belongs to whoever takes this.
if isinstance(unfinalized, UnfinalizedMoEOutput):
return unfinalized
return fused_out
def _finalize(
@@ -1426,7 +1441,7 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input: bool = False,
shared_experts: SharedExperts | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""This function computes a Mixture of Experts (MoE) layer using two sets
of weights, w1 and w2, and top-k gating mechanism.
@@ -1453,7 +1468,9 @@ class FusedMoEKernelModularImpl:
hidden_states before latent projection.
Returns:
torch.Tensor: The output tensor after applying the MoE layer.
torch.Tensor: The output tensor after applying the MoE layer, or
the unfinalized output when the experts left the top-k reduction to
a fused consumer.
"""
output = torch.empty_like(hidden_states)
@@ -1498,6 +1515,17 @@ class FusedMoEKernelModularImpl:
if lora_ctx is not None:
lora_ctx.original_hidden_states = None
if isinstance(fused_out, UnfinalizedMoEOutput):
# Nothing below can run on an unfinalized output: finalize is the
# local top-k reduction (and, for DP/EP, the combine) that the
# consumer takes over.
if not self.prepare_finalize.supports_deferred_moe_finalize():
raise RuntimeError(
f"{type(self.prepare_finalize).__name__} cannot pass through "
"a deferred MoE output."
)
return fused_out
return self._finalize(
output,
fused_out,
@@ -1653,10 +1681,7 @@ class FusedMoEKernel:
return self.prepare_finalize.output_is_reduced()
def supports_deferred_moe_finalize(self) -> bool:
return (
isinstance(self.prepare_finalize, FusedMoEPrepareAndFinalizeMonolithic)
and self.prepare_finalize.supports_deferred_moe_finalize()
)
return self.prepare_finalize.supports_deferred_moe_finalize()
def apply_monolithic(
self,
@@ -1703,7 +1728,7 @@ class FusedMoEKernel:
apply_router_weight_on_input: bool,
shared_experts: SharedExperts | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelModularImpl)
return self.impl.apply(
hidden_states=hidden_states,
@@ -9,7 +9,7 @@ import torch
@dataclass
class UnfinalizedMoEOutput:
"""Unfinalized output of a monolithic MoE kernel.
"""Unfinalized output of a MoE kernel.
Kernels that can stop after GEMM2 (the TRTLLM-Gen ``do_finalize=False`` path)
hand back their permuted, unweighted output plus the routing weights and the
@@ -24,10 +24,11 @@ class UnfinalizedMoEOutput:
# [num_permuted_rows, hidden_dim] permuted, unweighted GEMM2 output.
gemm2_permuted: torch.Tensor
# [num_tokens, top_k] routing weights, in the activation dtype -- consumers
# index them as such, and a buffer typed wider than its contents reads as
# garbage. These already carry routed_scaling_factor for routing methods that
# fold it in.
# [num_tokens, top_k] routing weights, in the activation dtype when the
# kernel routed itself and in the router's dtype for modular experts --
# consumers index them as such, and a buffer typed wider than its contents
# reads as garbage. These already carry routed_scaling_factor for routing
# methods that fold it in.
expert_weights: torch.Tensor
# [num_tokens, top_k] int32 permute map; -1 = expert not local to this rank.
expanded_idx_to_permuted_idx: torch.Tensor
@@ -38,6 +38,11 @@ def _quantize_input(
class MoEPrepareAndFinalizeNoDPEPModular(mk.FusedMoEPrepareAndFinalizeModular):
def supports_deferred_moe_finalize(self) -> bool:
# finalize() here is the top-k weight-and-reduce and nothing else,
# which is exactly the work a deferring consumer takes over.
return True
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@@ -1224,7 +1224,7 @@ class RoutedExperts(PluggableLayer):
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None" = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Execute routed experts using the quantization method's apply function.
This is called by the runner after router selection (for modular kernels)
@@ -1239,7 +1239,7 @@ class RoutedExperts(PluggableLayer):
shared_experts_input: Input for shared experts (if any)
Returns:
Output tensor from routed experts.
Finalized routed states or a deferred-finalize output.
"""
assert not self.quant_method.is_monolithic
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
Mxfp4MoeBackend,
@@ -227,7 +228,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -303,7 +303,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -375,7 +376,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -188,7 +189,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -827,7 +828,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
Mxfp4MoeBackend,
@@ -228,7 +229,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -179,7 +180,7 @@ class INCMxfp8MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -690,7 +690,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -1102,7 +1102,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -1471,7 +1471,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -433,7 +433,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -860,7 +860,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -166,7 +166,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -976,7 +976,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
@@ -1935,7 +1935,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
@@ -2182,7 +2182,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -115,10 +115,11 @@ class LatentMoERunner(MoERunner):
experts_cls = getattr(self._quant_method, "experts_cls", None)
# The kernel instance is built after weight loading, while the tail
# must register its CuTeDSL warmup units during runner construction.
self.moe_config.defer_moe_finalize = (
if (
experts_cls is TrtLlmMxfp4ExpertsMonolithic
or experts_cls is TrtLlmNvFp4ExpertsMonolithic
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded
) and self.moe_config.hidden_dim == self.moe_config.hidden_dim_unpadded:
self.moe_config.defer_moe_finalize()
from vllm.models.kimi_k3.nvidia.ops.latent_moe_tail import (
KimiK3LatentMoETailOp,
)
@@ -136,20 +137,13 @@ class LatentMoERunner(MoERunner):
rms_eps=norm.variance_epsilon,
)
self._k3_latent_moe_tail_op = op
self.moe_config.defer_moe_finalize_max_num_tokens = (
op.contract.max_num_tokens
if self.moe_config.use_deferred_moe_finalize
else -1
)
if self.moe_config.use_deferred_moe_finalize:
self.moe_config.limit_deferred_moe_finalize(op.contract.max_num_tokens)
logger.info_once(
"K3 latent-MoE tail fusion with deferred top-k finalization "
"is enabled for up to %d tokens.",
op.contract.max_num_tokens,
)
else:
self.moe_config.defer_moe_finalize = False
self.moe_config.defer_moe_finalize_max_num_tokens = -1
def _get_zero_residual(
self,