[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-25 15:38:29 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 24928a4d62
commit 378504a544
26 changed files with 652 additions and 117 deletions
+48
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import torch
@@ -28,6 +29,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
fused_experts,
)
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
BatchedPrepareAndFinalize,
)
@@ -707,6 +710,51 @@ def check_accuracy(a, b, atol, rtol, percent):
)
def check_deferred_moe_finalize(
moe_config: FusedMoEConfig,
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
router_weights: torch.Tensor | None = None,
chunked: bool = False,
) -> None:
"""Check a kernel that defers its finalize against the finalize it skips.
``run`` calls the kernel on fixed inputs, first as built and then with
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
report truthfully. A deferred output reduced by ``moe_unpermute``, the
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
bit, and modular experts hand the router's weights back as-is. A call the
experts split across kernel launches finalizes instead.
"""
# The deferred output views the router's buffer, so compare with a copy.
expected_weights = None if router_weights is None else router_weights.clone()
finalized = run()
assert isinstance(finalized, torch.Tensor)
moe_config.defer_moe_finalize()
output = run()
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
if chunked:
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
return
assert isinstance(output, UnfinalizedMoEOutput)
if expected_weights is not None:
torch.testing.assert_close(
output.expert_weights, expected_weights, atol=0, rtol=0
)
reference = torch.empty_like(finalized)
moe_unpermute(
reference,
output.gemm2_permuted,
output.expert_weights.float(),
output.expanded_idx_to_permuted_idx,
# The kernel reads its valid-row count through this pointer either way.
expert_first_token_offset=output.gemm2_permuted.new_full(
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
),
)
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.