[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
24928a4d62
commit
378504a544
@@ -1,14 +1,19 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from tests.kernels.moe.utils import (
|
||||
check_deferred_moe_finalize,
|
||||
make_dummy_moe_config,
|
||||
make_test_weights,
|
||||
)
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -32,11 +37,20 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import (
|
||||
TrtLlmFp8ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
rotate_weights_for_fi_trtllm_fp8_per_tensor_moe,
|
||||
swap_w13_to_w31,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic128Sym,
|
||||
@@ -767,3 +781,109 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
assert supported == expected, reason
|
||||
if not expected:
|
||||
assert "SwiGLU" in reason
|
||||
|
||||
|
||||
def _make_mxfp8_moe_weights(
|
||||
e: int, n: int, k: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q, s = zip(
|
||||
*(mxfp8_e4m3_quantize(w[i], is_sf_swizzled_layout=False) for i in range(e))
|
||||
)
|
||||
return torch.stack(q), torch.stack(s)
|
||||
|
||||
w1, w1_scale = quantize(
|
||||
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
)
|
||||
w2, w2_scale = quantize(
|
||||
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
|
||||
)
|
||||
return w1, w2, w1_scale, w2_scale
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
|
||||
@pytest.mark.parametrize("m", [1, 16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
m: int, block_shape: list[int], workspace_init
|
||||
):
|
||||
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
|
||||
finalize to the caller."""
|
||||
e, topk, n, k = 32, 4, 1024, 1024
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if block_shape == [1, 32]:
|
||||
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k)
|
||||
else:
|
||||
(_, w1, w1_scale, _), (_, w2, w2_scale, _) = make_test_weights(
|
||||
e, n, k, quant_dtype=torch.float8_e4m3fn, block_shape=block_shape
|
||||
)
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
SimpleNamespace(
|
||||
weight_block_size=block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=True, intermediate_size_per_partition=n
|
||||
),
|
||||
activation=MoEActivation.SILU,
|
||||
),
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
# One rank of a TP group, which deferral needs.
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=2 * n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=replace(
|
||||
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
|
||||
),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config, quant_config=quant_config, allow_new_interface=True
|
||||
),
|
||||
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
router_weights=topk_weights,
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ from dataclasses import dataclass, replace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import check_accuracy
|
||||
from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize
|
||||
from vllm._aiter_ops import (
|
||||
is_aiter_found,
|
||||
is_aiter_found_and_supported,
|
||||
@@ -663,6 +663,154 @@ def test_trtllm_gen_mxfp4_fused_moe(
|
||||
check_accuracy(ref_result, tg_result, atol=0, rtol=0.3, percent=0.8)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"monolithic,num_tokens,chunk_size",
|
||||
[(False, 1, None), (False, 16, None), (False, 16, 5), (True, 16, None)],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not TRTLLM_GEN_MXFP4_AVAILABLE,
|
||||
reason="nvidia gpu and compute capability sm100 is required for this test",
|
||||
)
|
||||
def test_trtllm_mxfp4_deferred_finalize(
|
||||
monolithic: bool,
|
||||
num_tokens: int,
|
||||
chunk_size: int | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
workspace_init,
|
||||
):
|
||||
"""TRTLLM-Gen MXFP4 experts can leave the top-k finalize to the caller."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
||||
TrtLlmMxfp4ExpertsModular,
|
||||
TrtLlmMxfp4ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
)
|
||||
|
||||
num_experts, topk, hidden_size, intermediate_size = 32, 4, 1024, 512
|
||||
device = torch.device("cuda:0")
|
||||
torch.manual_seed(0)
|
||||
if chunk_size is not None:
|
||||
monkeypatch.setattr(
|
||||
TrtLlmMxfp4ExpertsModular,
|
||||
"_max_supported_tokens",
|
||||
lambda self, top_k, global_num_experts: chunk_size,
|
||||
)
|
||||
|
||||
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
w_q, w_scale = fp4_quantize(
|
||||
w,
|
||||
torch.tensor(1.0, device=device),
|
||||
32,
|
||||
sf_use_ue8m0=True,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
return (
|
||||
w_q.reshape(*w.shape[:-1], -1),
|
||||
w_scale.view(torch.uint8).reshape(*w.shape[:-1], -1),
|
||||
)
|
||||
|
||||
w13, w13_scale = quantize(
|
||||
torch.randn(
|
||||
num_experts,
|
||||
2 * intermediate_size,
|
||||
hidden_size,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
)
|
||||
w2, w2_scale = quantize(
|
||||
torch.randn(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
)
|
||||
backend = Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
w13, w2, w13_scale, w2_scale, _, _ = convert_weight_to_mxfp4_moe_kernel_format(
|
||||
backend,
|
||||
torch.nn.Module(),
|
||||
w13,
|
||||
w2,
|
||||
w13_scale,
|
||||
w2_scale,
|
||||
_cache_permute_indices={},
|
||||
)
|
||||
# One rank of a TP group, which deferral needs.
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=hidden_size,
|
||||
intermediate_size=2 * intermediate_size,
|
||||
num_local_experts=num_experts,
|
||||
num_logical_experts=num_experts,
|
||||
activation=MoEActivation.SILU,
|
||||
device=device,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
moe_parallel_config=replace(
|
||||
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
|
||||
),
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
kernel = make_mxfp4_moe_kernel(
|
||||
make_mxfp4_moe_quant_config(backend, w1_scale=w13_scale, w2_scale=w2_scale),
|
||||
moe_config,
|
||||
TrtLlmMxfp4ExpertsMonolithic if monolithic else TrtLlmMxfp4ExpertsModular,
|
||||
backend,
|
||||
)
|
||||
|
||||
hidden_states = torch.randn(
|
||||
num_tokens, hidden_size, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
if monolithic:
|
||||
router_logits = torch.randn(num_tokens, num_experts, device=device)
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply_monolithic(
|
||||
hidden_states,
|
||||
w13,
|
||||
w2,
|
||||
router_logits,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
)
|
||||
return
|
||||
|
||||
topk_ids = torch.rand(num_tokens, num_experts, device=device).argsort(dim=-1)
|
||||
topk_ids = topk_ids[:, :topk].to(torch.int32)
|
||||
topk_weights = torch.rand(num_tokens, topk, device=device).softmax(dim=-1)
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply(
|
||||
hidden_states=hidden_states,
|
||||
w1=w13,
|
||||
w2=w2,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=num_experts,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
router_weights=topk_weights,
|
||||
chunked=chunk_size is not None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("topk", [1, 4])
|
||||
@pytest.mark.parametrize("num_experts", [32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 128])
|
||||
|
||||
@@ -8,11 +8,13 @@ and GELU — including a Gemma4-shaped case (128 experts, top-k 8,
|
||||
intermediate_size 704) that exercises the non-256-aligned padding path.
|
||||
"""
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_test_quant_config
|
||||
from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
@@ -31,6 +33,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
||||
@@ -92,6 +95,36 @@ SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
|
||||
SITU = op_registry[_SITU_OP_NAME]
|
||||
|
||||
|
||||
def _make_trtllm_fp4_moe_kernel(
|
||||
moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig
|
||||
) -> mk.FusedMoEKernel:
|
||||
experts = TrtLlmNvFp4ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
)
|
||||
# Mimic the production weight-loader path so per-expert tensors that
|
||||
# are normally precomputed in process_weights_after_loading (g1_scale_c
|
||||
# and the rescaled gemm1_clamp_limit) get materialized. The test's
|
||||
# synthetic quant_config has g1_alphas/g2_alphas already at their
|
||||
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
|
||||
# (same tensor) and use input_scale=1 to make the in-place
|
||||
# weight_scale_2 *= input_scale step a no-op.
|
||||
fake_layer = torch.nn.Module()
|
||||
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
|
||||
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
|
||||
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
|
||||
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
|
||||
experts.process_weights_after_loading(fake_layer)
|
||||
return mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
experts,
|
||||
)
|
||||
|
||||
|
||||
ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
|
||||
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
|
||||
@@ -191,34 +224,11 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
),
|
||||
)
|
||||
|
||||
trtllm_inner = TrtLlmNvFp4ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
)
|
||||
# Mimic the production weight-loader path so per-expert tensors that
|
||||
# are normally precomputed in process_weights_after_loading (g1_scale_c
|
||||
# and the rescaled gemm1_clamp_limit) get materialized. The test's
|
||||
# synthetic quant_config has g1_alphas/g2_alphas already at their
|
||||
# post-fusion values, so we set w13_weight_scale_2 to alias g1_alphas
|
||||
# (same tensor) and use input_scale=1 to make the in-place
|
||||
# weight_scale_2 *= input_scale step a no-op.
|
||||
fake_layer = torch.nn.Module()
|
||||
fake_layer.w13_weight_scale_2 = quant_config.g1_alphas
|
||||
fake_layer.w2_weight_scale_2 = quant_config.g2_alphas
|
||||
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
|
||||
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
|
||||
trtllm_inner.process_weights_after_loading(fake_layer)
|
||||
trtllm_experts = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
|
||||
if activation == MoEActivation.SITU:
|
||||
torch.testing.assert_close(trtllm_inner.g1_scale_c, quant_config.a2_gscale)
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
trtllm_inner,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale
|
||||
)
|
||||
|
||||
trtllm_output = trtllm_experts.apply(
|
||||
hidden_states=a,
|
||||
@@ -331,6 +341,70 @@ def test_trtllm_fp4_moe_reprocess_does_not_refold_swiglu_params():
|
||||
torch.testing.assert_close(experts.gemm1_beta, beta * expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,chunk_size", [(1, None), (16, None), (16, 5)])
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_fp4_moe_deferred_finalize(
|
||||
m: int,
|
||||
chunk_size: int | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
workspace_init,
|
||||
):
|
||||
"""TRTLLM-Gen NvFP4 modular experts can leave the top-k finalize to the caller."""
|
||||
e, topk, n, k = 128, 8, 1024, 1024
|
||||
dtype = torch.bfloat16
|
||||
if chunk_size is not None:
|
||||
monkeypatch.setattr(
|
||||
TrtLlmNvFp4ExpertsModular, "_get_chunk_size", lambda self: chunk_size
|
||||
)
|
||||
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
a = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
w1_q, w2_q, quant_config = make_test_quant_config(
|
||||
e, n, k, in_dtype=dtype, quant_dtype="nvfp4", is_scale_swizzled=False
|
||||
)
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
# One rank of a TP group, which deferral needs.
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=2 * n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=MoEActivation.SILU,
|
||||
device="cuda",
|
||||
moe_parallel_config=replace(
|
||||
FusedMoEParallelConfig.make_no_parallel(), tp_size=2
|
||||
),
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
kernel = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
|
||||
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply(
|
||||
hidden_states=a,
|
||||
w1=w1_q,
|
||||
w2=w2_q,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
router_weights=topk_weights,
|
||||
chunked=chunk_size is not None,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_trtllm_fp4_moe_no_graph(
|
||||
64,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
@@ -28,6 +29,8 @@ from vllm.model_executor.layers.fused_moe.fused_moe import (
|
||||
fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import moe_unpermute
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import (
|
||||
BatchedPrepareAndFinalize,
|
||||
)
|
||||
@@ -707,6 +710,51 @@ def check_accuracy(a, b, atol, rtol, percent):
|
||||
)
|
||||
|
||||
|
||||
def check_deferred_moe_finalize(
|
||||
moe_config: FusedMoEConfig,
|
||||
run: Callable[[], torch.Tensor | UnfinalizedMoEOutput],
|
||||
router_weights: torch.Tensor | None = None,
|
||||
chunked: bool = False,
|
||||
) -> None:
|
||||
"""Check a kernel that defers its finalize against the finalize it skips.
|
||||
|
||||
``run`` calls the kernel on fixed inputs, first as built and then with
|
||||
``moe_config`` asking to defer, which ``should_defer_moe_finalize`` must
|
||||
report truthfully. A deferred output reduced by ``moe_unpermute``, the
|
||||
TRT-LLM finalize kernel, must give the kernel's own finalized output bit for
|
||||
bit, and modular experts hand the router's weights back as-is. A call the
|
||||
experts split across kernel launches finalizes instead.
|
||||
"""
|
||||
# The deferred output views the router's buffer, so compare with a copy.
|
||||
expected_weights = None if router_weights is None else router_weights.clone()
|
||||
finalized = run()
|
||||
assert isinstance(finalized, torch.Tensor)
|
||||
moe_config.defer_moe_finalize()
|
||||
output = run()
|
||||
assert moe_config.should_defer_moe_finalize(finalized.shape[0]) != chunked
|
||||
if chunked:
|
||||
torch.testing.assert_close(output, finalized, atol=0, rtol=0)
|
||||
return
|
||||
|
||||
assert isinstance(output, UnfinalizedMoEOutput)
|
||||
if expected_weights is not None:
|
||||
torch.testing.assert_close(
|
||||
output.expert_weights, expected_weights, atol=0, rtol=0
|
||||
)
|
||||
reference = torch.empty_like(finalized)
|
||||
moe_unpermute(
|
||||
reference,
|
||||
output.gemm2_permuted,
|
||||
output.expert_weights.float(),
|
||||
output.expanded_idx_to_permuted_idx,
|
||||
# The kernel reads its valid-row count through this pointer either way.
|
||||
expert_first_token_offset=output.gemm2_permuted.new_full(
|
||||
(1,), output.gemm2_permuted.shape[0], dtype=torch.int64
|
||||
),
|
||||
)
|
||||
torch.testing.assert_close(reference, finalized, atol=0, rtol=0)
|
||||
|
||||
|
||||
def mxfp4_w_layouts(mx_axis: int, num_warps: int = 8):
|
||||
"""Weight/scale layouts for mxfp4 MoE, as (layout, opts) pairs.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user