[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
24928a4d62
commit
378504a544
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
@@ -28,6 +29,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
|
||||
BatchedPrepareAndFinalize,
|
||||
)
|
||||
@@ -707,6 +710,51 @@ def check_accuracy(a, b, atol, rtol, percent):
|
||||
)
|
||||
|
||||
|
||||
def check_deferred_moe_finalize(
|
||||
moe_config: FusedMoEConfig,
|
||||
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
|
||||
router_weights: torch.Tensor | None = None,
|
||||
chunked: bool = False,
|
||||
) -> None:
|
||||
"""Check a kernel that defers its finalize against the finalize it skips.
|
||||
|
||||
``run`` calls the kernel on fixed inputs, first as built and then with
|
||||
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
|
||||
report truthfully. A deferred output reduced by ``moe_unpermute``, the
|
||||
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
|
||||
bit, and modular experts hand the router's weights back as-is. A call the
|
||||
experts split across kernel launches finalizes instead.
|
||||
"""
|
||||
# The deferred output views the router's buffer, so compare with a copy.
|
||||
expected_weights = None if router_weights is None else router_weights.clone()
|
||||
finalized = run()
|
||||
assert isinstance(finalized, torch.Tensor)
|
||||
moe_config.defer_moe_finalize()
|
||||
output = run()
|
||||
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
|
||||
if chunked:
|
||||
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
|
||||
return
|
||||
|
||||
assert isinstance(output, UnfinalizedMoEOutput)
|
||||
if expected_weights is not None:
|
||||
torch.testing.assert_close(
|
||||
output.expert_weights, expected_weights, atol=0, rtol=0
|
||||
)
|
||||
reference = torch.empty_like(finalized)
|
||||
moe_unpermute(
|
||||
reference,
|
||||
output.gemm2_permuted,
|
||||
output.expert_weights.float(),
|
||||
output.expanded_idx_to_permuted_idx,
|
||||
# The kernel reads its valid-row count through this pointer either way.
|
||||
expert_first_token_offset=output.gemm2_permuted.new_full(
|
||||
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
|
||||
|
||||
|
||||
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
|
||||
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user