From 97a668152b9033caf602b37b88289f03e8faffd8 Mon Sep 17 00:00:00 2001 From: Xuanyu Zhang Date: Tue, 21 Jul 2026 01:45:10 +0200 Subject: [PATCH] [RL Infra][FlashInfer] Enable router replay output from FlashInfer monolithic MoE kernel (#44214) Signed-off-by: Xuanyu Zhang Co-authored-by: Claude Co-authored-by: aoshen02 Co-authored-by: OpenAI Codex --- .../test_routed_experts_capture_monolithic.py | 880 ++++++++++++++++++ .../test_routed_experts_capture.py | 55 ++ .../fused_moe/experts/trtllm_bf16_moe.py | 14 +- .../fused_moe/experts/trtllm_fp8_moe.py | 24 +- .../fused_moe/experts/trtllm_mxfp4_moe.py | 12 +- .../fused_moe/experts/trtllm_mxint4_moe.py | 15 +- .../fused_moe/experts/trtllm_nvfp4_moe.py | 14 +- .../layers/fused_moe/modular_kernel.py | 51 + .../fused_moe/routed_experts_capturer.py | 12 +- .../utils/flashinfer_mxint4_moe.py | 2 + vllm/v1/worker/gpu_model_runner.py | 28 +- 11 files changed, 1097 insertions(+), 10 deletions(-) create mode 100644 tests/kernels/moe/test_routed_experts_capture_monolithic.py diff --git a/tests/kernels/moe/test_routed_experts_capture_monolithic.py b/tests/kernels/moe/test_routed_experts_capture_monolithic.py new file mode 100644 index 0000000000..047438762c --- /dev/null +++ b/tests/kernels/moe/test_routed_experts_capture_monolithic.py @@ -0,0 +1,880 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end tests for routed-expert capture on the monolithic MoE path. + +These tests exercise the wiring that lets ``RoutedExpertsCapturer`` see the +expert IDs picked by FlashInfer's fused router-and-experts kernels (the +"monolithic" path). When ``set_capture_fn`` is installed on +a ``FusedMoEExpertsMonolithic`` subclass that supports it, the kernel call +should: + + * allocate an int16 ``(num_tokens, top_k)`` buffer, + * pass it to FlashInfer as ``routing_replay_out``, + * invoke the callback after the kernel returns. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEParallelConfig, + FusedMoEQuantConfig, + RoutingMethodType, + fp8_w8a8_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import ( + TrtLlmBf16ExpertsMonolithic, +) +from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import ( + TrtLlmFp8ExpertsMonolithic, +) +from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import ( + TrtLlmNvFp4ExpertsMonolithic, +) +from vllm.platforms import current_platform + +try: + from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe +except ImportError: + pytest.skip("flashinfer not available", allow_module_level=True) + +if not has_flashinfer_trtllm_fused_moe() or not current_platform.is_cuda(): + pytest.skip( + "Requires FlashInfer TRT-LLM fused MoE on CUDA", + allow_module_level=True, + ) + +if not current_platform.has_device_capability(100): + pytest.skip( + "TRT-LLM fused MoE kernels require SM100+", + allow_module_level=True, + ) + + +def _shuffle_bf16_weights_block_major_k( + w: torch.Tensor, epilogue_tile_m: int = 64, block_k: int = 128 +) -> torch.Tensor: + """Reshape ``w`` (E, M, K) into the ``BlockMajorK`` layout expected by + ``trtllm_bf16_moe``: ``(E, K/block_k, M, block_k)`` after a per-expert + row shuffle. + """ + from flashinfer import shuffle_matrix_a + from flashinfer.fused_moe import convert_to_block_layout + + num_experts = w.shape[0] + shuffled = [] + for i in range(num_experts): + t = shuffle_matrix_a(w[i].view(torch.uint8), epilogue_tile_m) + shuffled.append(convert_to_block_layout(t, block_k)) + return torch.stack(shuffled).view(torch.bfloat16) + + +def _make_bf16_monolithic_experts( + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + routing_method: RoutingMethodType, + device: torch.device, +) -> tuple[TrtLlmBf16ExpertsMonolithic, torch.Tensor, torch.Tensor]: + """Construct the monolithic BF16 experts plus the BlockMajorK weights + expected by ``trtllm_bf16_moe``. + """ + parallel_cfg = FusedMoEParallelConfig.make_no_parallel() + moe_config = FusedMoEConfig( + num_experts=num_experts, + experts_per_token=top_k, + hidden_dim=hidden_size, + intermediate_size=intermediate_size, + num_local_experts=num_experts, + num_logical_experts=num_experts, + moe_parallel_config=parallel_cfg, + in_dtype=torch.bfloat16, + activation=MoEActivation.SILU, + device=device, + routing_method=routing_method, + max_num_tokens=max(8, 1), + ) + quant_config = FusedMoEQuantConfig.make( + quant_dtype=None, + per_act_token_quant=False, + per_out_ch_quant=False, + block_shape=None, + ) + + experts = TrtLlmBf16ExpertsMonolithic( + moe_config=moe_config, quant_config=quant_config + ) + + gemm1 = ( + torch.randn( + num_experts, + 2 * intermediate_size, + hidden_size, + device=device, + dtype=torch.bfloat16, + ) + * 0.1 + ) + gemm2 = ( + torch.randn( + num_experts, + hidden_size, + intermediate_size, + device=device, + dtype=torch.bfloat16, + ) + * 0.1 + ) + w13 = _shuffle_bf16_weights_block_major_k(gemm1) + w2 = _shuffle_bf16_weights_block_major_k(gemm2) + return experts, w13, w2 + + +def _run_bf16_monolithic( + experts: TrtLlmBf16ExpertsMonolithic, + hidden_states: torch.Tensor, + w13: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + num_experts: int, + *, + n_group: int | None = None, + topk_group: int | None = None, + routed_scaling_factor: float | None = None, + e_score_correction_bias: torch.Tensor | None = None, +) -> torch.Tensor: + return experts.apply( + hidden_states=hidden_states, + w1=w13, + w2=w2, + router_logits=router_logits, + activation=MoEActivation.SILU, + global_num_experts=num_experts, + expert_map=None, + a1q_scale=None, + apply_router_weight_on_input=False, + num_expert_group=n_group, + topk_group=topk_group, + e_score_correction_bias=e_score_correction_bias, + routed_scaling_factor=routed_scaling_factor, + ) + + +_DSV3_NUM_EXPERTS = 32 +_DSV3_N_GROUP = 4 +_DSV3_TOPK_GROUP = 2 + + +def _make_dsv3_routing_bias(num_experts: int, device: torch.device) -> torch.Tensor: + return torch.randn(num_experts, device=device, dtype=torch.bfloat16) + + +@pytest.mark.parametrize("num_tokens", [2, 7, 16]) +@pytest.mark.parametrize("top_k", [2, 4]) +def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts( + num_tokens: int, + top_k: int, +) -> None: + """The capture callback should receive the int16 routed-expert IDs the + kernel actually used, the values should be valid expert indices, and + each token should pick ``top_k`` distinct experts.""" + if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP: + pytest.skip( + f"DSV3 requires top_k <= n_group * topk_group " + f"({_DSV3_N_GROUP * _DSV3_TOPK_GROUP})" + ) + torch.manual_seed(0) + device = torch.device("cuda:0") + + num_experts = _DSV3_NUM_EXPERTS + hidden_size = 1024 + intermediate_size = 1024 + + experts, w13, w2 = _make_bf16_monolithic_experts( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + routing_method=RoutingMethodType.DeepSeekV3, + device=device, + ) + + captured: list[torch.Tensor] = [] + + def capture_fn(replay_out: torch.Tensor) -> None: + captured.append(replay_out.clone()) + + assert experts.supports_routing_replay_capture() + experts.set_capture_fn(capture_fn) + + hidden_states = ( + torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 + ) + router_logits = torch.rand( + num_tokens, num_experts, device=device, dtype=torch.float32 + ) + routing_bias = _make_dsv3_routing_bias(num_experts, device) + + _ = _run_bf16_monolithic( + experts, + hidden_states=hidden_states, + w13=w13, + w2=w2, + router_logits=router_logits, + num_experts=num_experts, + n_group=_DSV3_N_GROUP, + topk_group=_DSV3_TOPK_GROUP, + routed_scaling_factor=1.0, + e_score_correction_bias=routing_bias, + ) + + assert len(captured) == 1 + replay = captured[0] + assert replay.dtype == torch.int16 + assert replay.shape == (num_tokens, top_k) + assert (replay >= 0).all(), f"got out-of-range values: {replay}" + assert (replay < num_experts).all(), f"got out-of-range values: {replay}" + + for t in range(num_tokens): + unique = replay[t].unique() + assert unique.numel() == top_k, ( + f"token {t}: expected {top_k} distinct experts, " + f"got {unique.numel()} ({replay[t].tolist()})" + ) + + +@pytest.mark.parametrize("num_tokens", [2, 7, 16]) +@pytest.mark.parametrize( + "routing_method", + [ + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ], +) +def test_trtllm_bf16_monolithic_routing_replay_non_dsv3( + num_tokens: int, + routing_method: RoutingMethodType, +) -> None: + """Routing replay works for non-DeepSeekV3 routing methods too. + FlashInfer's ``routing_replay_out`` is routing-method-agnostic.""" + torch.manual_seed(0) + device = torch.device("cuda:0") + + num_experts = 8 + top_k = 2 + hidden_size = 1024 + intermediate_size = 1024 + + experts, w13, w2 = _make_bf16_monolithic_experts( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + routing_method=routing_method, + device=device, + ) + + captured: list[torch.Tensor] = [] + experts.set_capture_fn(lambda r: captured.append(r.clone())) + + hidden_states = ( + torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 + ) + router_logits = torch.rand( + num_tokens, num_experts, device=device, dtype=torch.float32 + ) + + _ = _run_bf16_monolithic( + experts, + hidden_states=hidden_states, + w13=w13, + w2=w2, + router_logits=router_logits, + num_experts=num_experts, + ) + + assert len(captured) == 1 + replay = captured[0] + assert replay.dtype == torch.int16 + assert replay.shape == (num_tokens, top_k) + assert (replay >= 0).all(), f"got out-of-range values: {replay}" + assert (replay < num_experts).all(), f"got out-of-range values: {replay}" + for t in range(num_tokens): + unique = replay[t].unique() + assert unique.numel() == top_k, ( + f"token {t}: expected {top_k} distinct experts, " + f"got {unique.numel()} ({replay[t].tolist()})" + ) + + +def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None: + """With no callback installed the kernel should not see a + ``routing_replay_out`` tensor — verify the helper short-circuits.""" + torch.manual_seed(0) + device = torch.device("cuda:0") + experts, _, _ = _make_bf16_monolithic_experts( + num_experts=_DSV3_NUM_EXPERTS, + top_k=2, + hidden_size=1024, + intermediate_size=1024, + routing_method=RoutingMethodType.DeepSeekV3, + device=device, + ) + # No callback installed. + buf = experts._maybe_make_routing_replay_buffer(num_tokens=4, device=device) + assert buf is None + + # Dispatch is also a no-op. + experts._maybe_dispatch_routing_replay(buf, num_tokens=4) + + +def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None: + """FlashInfer's ``routing_replay_out`` is supported by all routing + methods, so ``supports_routing_replay_capture`` should be True + regardless of routing method.""" + device = torch.device("cuda:0") + for routing_method in ( + RoutingMethodType.DeepSeekV3, + RoutingMethodType.Renormalize, + RoutingMethodType.RenormalizeNaive, + ): + experts, _, _ = _make_bf16_monolithic_experts( + num_experts=_DSV3_NUM_EXPERTS, + top_k=2, + hidden_size=1024, + intermediate_size=1024, + routing_method=routing_method, + device=device, + ) + assert experts.supports_routing_replay_capture() is True, ( + f"{routing_method!r} should support routing replay capture" + ) + + +def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None: + """When capture is installed, the allocated buffer is int16 and shaped + ``(num_tokens, experts_per_token)``.""" + device = torch.device("cuda:0") + experts, _, _ = _make_bf16_monolithic_experts( + num_experts=_DSV3_NUM_EXPERTS, + top_k=4, + hidden_size=1024, + intermediate_size=1024, + routing_method=RoutingMethodType.DeepSeekV3, + device=device, + ) + experts.set_capture_fn(lambda r: None) + buf = experts._maybe_make_routing_replay_buffer(num_tokens=11, device=device) + assert buf is not None + assert buf.dtype == torch.int16 + assert buf.shape[0] >= 11 + assert buf.shape[1] == 4 + assert buf.device.type == "cuda" + + +def test_routed_experts_capturer_e2e_via_monolithic_experts() -> None: + """End-to-end: bind ``RoutedExpertsCapturer.capture`` as the callback + on the monolithic experts and verify the captured rows land in the + capturer's device buffer at the correct layer slot. + + Mirrors the wiring done in ``GPUModelRunner._bind_routed_experts_capturer`` + for the monolithic path: a single closure is installed on the monolithic + ``fused_experts`` (in addition to ``router.set_capture_fn`` on the + non-monolithic path) and the capturer routes per-layer based on the + closed-over ``layer_id``. + """ + from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( + RoutedExpertsCapturer, + ) + + torch.manual_seed(7) + device = torch.device("cuda:0") + num_tokens = 4 + top_k = 2 + num_experts = _DSV3_NUM_EXPERTS + hidden_size = 1024 + intermediate_size = 1024 + + experts, w13, w2 = _make_bf16_monolithic_experts( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + routing_method=RoutingMethodType.DeepSeekV3, + device=device, + ) + + num_layers = 3 + layer_id = 1 + capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer) + capturer.dp_rank = 0 + capturer.tp_size = 1 + capturer.device_buffer = torch.full( + (num_tokens + 4, num_layers, top_k), + -1, + dtype=torch.int32, + device=device, + ) + + def capture_fn(replay_out: torch.Tensor) -> None: + capturer.capture(layer_id, replay_out) + + experts.set_capture_fn(capture_fn) + + hidden_states = ( + torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 + ) + router_logits = torch.rand( + num_tokens, num_experts, device=device, dtype=torch.float32 + ) + routing_bias = _make_dsv3_routing_bias(num_experts, device) + + # Patch get_forward_context to return a dp_metadata=None context so the + # capturer takes the single-DP branch. + import vllm.model_executor.layers.fused_moe.routed_experts_capturer as rec + + with patch.object( + rec, + "get_forward_context", + return_value=SimpleNamespace(dp_metadata=None), + ): + _ = _run_bf16_monolithic( + experts, + hidden_states=hidden_states, + w13=w13, + w2=w2, + router_logits=router_logits, + num_experts=num_experts, + n_group=_DSV3_N_GROUP, + topk_group=_DSV3_TOPK_GROUP, + routed_scaling_factor=1.0, + e_score_correction_bias=routing_bias, + ) + + captured = capturer.device_buffer[:num_tokens, layer_id, :].cpu() + # Valid expert IDs at this layer. + assert (captured >= 0).all() + assert (captured < num_experts).all() + for t in range(num_tokens): + unique = captured[t].unique() + assert unique.numel() == top_k, ( + f"token {t}: expected {top_k} distinct experts at layer " + f"{layer_id}, got {unique.numel()}" + ) + + # Other layers / trailing token rows untouched. + for other_layer in range(num_layers): + if other_layer == layer_id: + continue + assert (capturer.device_buffer[:, other_layer, :].cpu() == -1).all(), ( + f"layer {other_layer} should be untouched, got writes" + ) + assert (capturer.device_buffer[num_tokens:, layer_id, :].cpu() == -1).all(), ( + "tail rows beyond num_tokens should remain sentinel" + ) + + +# ---------------------------------------------------------------------------- +# FP8 block-scale (DeepSeekFp8) — vLLM's ``TrtLlmFp8ExpertsMonolithic`` +# ---------------------------------------------------------------------------- + + +def _make_fp8_block_scale_monolithic_experts( + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + device: torch.device, +) -> tuple[TrtLlmFp8ExpertsMonolithic, torch.Tensor, torch.Tensor]: + """Set up ``TrtLlmFp8ExpertsMonolithic`` for the DeepSeekFp8 block-scale + code path with DSV3 routing. + + Weights are shuffled into the BlockMajorK layout the kernel expects + (same helper the vLLM weight loader uses for DeepSeek-FP8 models). + """ + from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( + _shuffle_deepseek_fp8_moe_weights, + ) + + block_k = 128 + parallel_cfg = FusedMoEParallelConfig.make_no_parallel() + moe_config = FusedMoEConfig( + num_experts=num_experts, + experts_per_token=top_k, + hidden_dim=hidden_size, + intermediate_size=intermediate_size, + num_local_experts=num_experts, + num_logical_experts=num_experts, + moe_parallel_config=parallel_cfg, + in_dtype=torch.bfloat16, + activation=MoEActivation.SILU, + device=device, + routing_method=RoutingMethodType.DeepSeekV3, + max_num_tokens=max(8, 1), + ) + + # Random fp8 weights + ones-block scales (the kernel decoder only cares + # that the per-block scales are present and finite for routing/replay). + gemm1 = torch.randn( + num_experts, 2 * intermediate_size, hidden_size, device=device + ).to(torch.float8_e4m3fn) + gemm2 = torch.randn(num_experts, hidden_size, intermediate_size, device=device).to( + torch.float8_e4m3fn + ) + w13_shuffled, w2_shuffled = _shuffle_deepseek_fp8_moe_weights(gemm1, gemm2) + + w1_scale = torch.ones( + num_experts, + 2 * intermediate_size // block_k, + hidden_size // block_k, + device=device, + dtype=torch.float32, + ) + w2_scale = torch.ones( + num_experts, + hidden_size // block_k, + intermediate_size // block_k, + device=device, + dtype=torch.float32, + ) + quant_config = fp8_w8a8_moe_quant_config( + w1_scale=w1_scale, + w2_scale=w2_scale, + block_shape=[block_k, block_k], + per_act_token_quant=False, + ) + + experts = TrtLlmFp8ExpertsMonolithic( + moe_config=moe_config, quant_config=quant_config + ) + return experts, w13_shuffled, w2_shuffled + + +def _run_fp8_block_scale_monolithic( + experts: TrtLlmFp8ExpertsMonolithic, + hidden_states_fp8: torch.Tensor, + hidden_states_scale: torch.Tensor, + w13: torch.Tensor, + w2: torch.Tensor, + router_logits: torch.Tensor, + num_experts: int, + routing_bias: torch.Tensor, +) -> torch.Tensor: + return experts.apply( + hidden_states=hidden_states_fp8, + w1=w13, + w2=w2, + router_logits=router_logits, + activation=MoEActivation.SILU, + global_num_experts=num_experts, + expert_map=None, + # The block-scale apply path reads ``a1q_scale`` and transposes it + # to ``(hidden_size/128, num_tokens)`` for the kernel call. + a1q_scale=hidden_states_scale, + apply_router_weight_on_input=False, + num_expert_group=_DSV3_N_GROUP, + topk_group=_DSV3_TOPK_GROUP, + e_score_correction_bias=routing_bias, + routed_scaling_factor=1.0, + ) + + +@pytest.mark.parametrize("num_tokens", [2, 7, 16]) +@pytest.mark.parametrize("top_k", [2, 4]) +def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts( + num_tokens: int, + top_k: int, +) -> None: + """End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale + path, DSV3 routing) captures valid expert IDs.""" + if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP: + pytest.skip( + f"DSV3 requires top_k <= n_group * topk_group " + f"({_DSV3_N_GROUP * _DSV3_TOPK_GROUP})" + ) + torch.manual_seed(0) + device = torch.device("cuda:0") + + num_experts = _DSV3_NUM_EXPERTS + hidden_size = 1024 + intermediate_size = 1024 + + experts, w13, w2 = _make_fp8_block_scale_monolithic_experts( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + device=device, + ) + assert experts.supports_routing_replay_capture() + + captured: list[torch.Tensor] = [] + experts.set_capture_fn(lambda r: captured.append(r.clone())) + + # Per-token / per-block hidden scales (ones is fine for the routing + # path; the GEMM output isn't being asserted on). + hidden_states = ( + torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 + ).to(torch.float8_e4m3fn) + hidden_states_scale = torch.ones( + num_tokens, hidden_size // 128, device=device, dtype=torch.float32 + ) + router_logits = torch.rand( + num_tokens, num_experts, device=device, dtype=torch.float32 + ) + routing_bias = _make_dsv3_routing_bias(num_experts, device) + + _ = _run_fp8_block_scale_monolithic( + experts, + hidden_states_fp8=hidden_states, + hidden_states_scale=hidden_states_scale, + w13=w13, + w2=w2, + router_logits=router_logits, + num_experts=num_experts, + routing_bias=routing_bias, + ) + + assert len(captured) == 1 + replay = captured[0] + assert replay.dtype == torch.int16 + assert replay.shape == (num_tokens, top_k) + assert (replay >= 0).all(), f"got out-of-range values: {replay}" + assert (replay < num_experts).all(), f"got out-of-range values: {replay}" + for t in range(num_tokens): + unique = replay[t].unique() + assert unique.numel() == top_k, ( + f"token {t}: expected {top_k} distinct experts, " + f"got {unique.numel()} ({replay[t].tolist()})" + ) + + +# ---------------------------------------------------------------------------- +# NVFP4 — vLLM's ``TrtLlmNvFp4ExpertsMonolithic`` +# ---------------------------------------------------------------------------- + + +def _make_nvfp4_monolithic_experts( + num_experts: int, + top_k: int, + hidden_size: int, + intermediate_size: int, + device: torch.device, +) -> tuple[ + TrtLlmNvFp4ExpertsMonolithic, + torch.Tensor, # w13 (packed nvfp4 uint8) + torch.Tensor, # w13 block-scale (fp8) + torch.Tensor, # w2 (packed nvfp4 uint8) + torch.Tensor, # w2 block-scale (fp8) + torch.Tensor, # input global scale (per-tensor float32) +]: + """Set up ``TrtLlmNvFp4ExpertsMonolithic`` with NVFP4-quantized weights + and DSV3 routing. + + NVFP4 = per-block-of-16 fp4 with an fp8 scale, plus a per-tensor + "global" scale. We follow the layout in ``test_ocp_mx_moe.py`` / + ``flashinfer/tests/moe/test_trtllm_gen_routed_fused_moe.py``: + * weights: uint8 (packed fp4) ``(E, M, K//2)`` + * weight scales: fp8 ``(E, M, K//16)`` + * hidden states: uint8 (packed fp4) ``(N, K//2)`` + * hidden state scales: fp8 ``(N, K//16)`` + """ + from flashinfer import fp4_quantize + + block_size = 16 + parallel_cfg = FusedMoEParallelConfig.make_no_parallel() + moe_config = FusedMoEConfig( + num_experts=num_experts, + experts_per_token=top_k, + hidden_dim=hidden_size, + intermediate_size=intermediate_size, + num_local_experts=num_experts, + num_logical_experts=num_experts, + moe_parallel_config=parallel_cfg, + in_dtype=torch.bfloat16, + activation=MoEActivation.SILU, + device=device, + routing_method=RoutingMethodType.DeepSeekV3, + max_num_tokens=max(8, 1), + ) + + gemm1 = torch.randn( + num_experts, + 2 * intermediate_size, + hidden_size, + device=device, + dtype=torch.bfloat16, + ) + gemm2 = torch.randn( + num_experts, + hidden_size, + intermediate_size, + device=device, + dtype=torch.bfloat16, + ) + # Per-tensor weight scaling factor (used to build ``g1_alphas`` / + # ``g2_alphas`` below). + w_global_scale = torch.tensor(1.0, device=device) + # Per-tensor input scaling factor. + a_global_scale = torch.tensor(1.0, device=device) + + w13_q, w13_scale = fp4_quantize( + gemm1, + w_global_scale, + block_size, + sf_use_ue8m0=False, + is_sf_swizzled_layout=False, + ) + w13_scale = w13_scale.view(torch.float8_e4m3fn).reshape( + num_experts, 2 * intermediate_size, hidden_size // block_size + ) + w2_q, w2_scale = fp4_quantize( + gemm2, + w_global_scale, + block_size, + sf_use_ue8m0=False, + is_sf_swizzled_layout=False, + ) + w2_scale = w2_scale.view(torch.float8_e4m3fn).reshape( + num_experts, hidden_size, intermediate_size // block_size + ) + + # NVFP4 dq scale chain: g1_alphas = w1_scale_2 * a1_scale_2, + # g2_alphas = w2_scale_2 * a2_scale_2. The kernel multiplies by these. + g_alphas = torch.full((num_experts,), 1.0, device=device, dtype=torch.float32) + a2_gscale = torch.full((num_experts,), 1.0, device=device, dtype=torch.float32) + + quant_config = FusedMoEQuantConfig.make( + quant_dtype="nvfp4", + per_act_token_quant=False, + per_out_ch_quant=False, + block_shape=None, + w1_scale=w13_scale, + w2_scale=w2_scale, + g1_alphas=g_alphas, + g2_alphas=g_alphas, + a1_gscale=a_global_scale, + a2_gscale=a2_gscale, + ) + + experts = TrtLlmNvFp4ExpertsMonolithic( + moe_config=moe_config, quant_config=quant_config + ) + return experts, w13_q, w13_scale, w2_q, w2_scale, a_global_scale + + +def _run_nvfp4_monolithic( + experts: TrtLlmNvFp4ExpertsMonolithic, + hidden_states_q: torch.Tensor, + hidden_states_scale: torch.Tensor, + router_logits: torch.Tensor, + num_experts: int, + routing_bias: torch.Tensor, +) -> torch.Tensor: + """The monolithic NVFP4 apply expects packed fp4 hidden states + the + matching fp8 per-block scale stored in the ``a1q_scale`` slot.""" + # Stash the weight tensors on the experts in the locations the apply() + # implementation reads from (it pulls them from quant_config / scales + # already; w1/w2 come in as args). + return experts.apply( + hidden_states=hidden_states_q, + w1=experts._w13_packed, + w2=experts._w2_packed, + router_logits=router_logits, + activation=MoEActivation.SILU, + global_num_experts=num_experts, + expert_map=None, + a1q_scale=hidden_states_scale, + apply_router_weight_on_input=False, + num_expert_group=_DSV3_N_GROUP, + topk_group=_DSV3_TOPK_GROUP, + e_score_correction_bias=routing_bias, + routed_scaling_factor=1.0, + ) + + +@pytest.mark.parametrize("num_tokens", [2, 7, 16]) +@pytest.mark.parametrize("top_k", [2, 4]) +def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts( + num_tokens: int, + top_k: int, +) -> None: + """End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs + on the DSV3 routing path.""" + if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP: + pytest.skip( + f"DSV3 requires top_k <= n_group * topk_group " + f"({_DSV3_N_GROUP * _DSV3_TOPK_GROUP})" + ) + from flashinfer import fp4_quantize + + torch.manual_seed(0) + device = torch.device("cuda:0") + + num_experts = _DSV3_NUM_EXPERTS + hidden_size = 1024 + intermediate_size = 1024 + block_size = 16 + + experts, w13_q, _w13_s, w2_q, _w2_s, a_gs = _make_nvfp4_monolithic_experts( + num_experts=num_experts, + top_k=top_k, + hidden_size=hidden_size, + intermediate_size=intermediate_size, + device=device, + ) + # The apply() reads w1/w2 from its args, but we keep them on the experts + # for convenience of the helper. + experts._w13_packed = w13_q + experts._w2_packed = w2_q + + assert experts.supports_routing_replay_capture() + captured: list[torch.Tensor] = [] + experts.set_capture_fn(lambda r: captured.append(r.clone())) + + hidden_states = ( + torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 + ) + hidden_states_q, hidden_states_scale = fp4_quantize( + hidden_states, + a_gs, + block_size, + sf_use_ue8m0=False, + is_sf_swizzled_layout=False, + ) + # The vLLM apply() does the .view(fp8_e4m3fn).reshape itself, so leave + # ``hidden_states_scale`` in its native (uint8 packed) form. + router_logits = torch.rand( + num_tokens, num_experts, device=device, dtype=torch.float32 + ) + routing_bias = _make_dsv3_routing_bias(num_experts, device) + + _ = _run_nvfp4_monolithic( + experts, + hidden_states_q=hidden_states_q, + hidden_states_scale=hidden_states_scale, + router_logits=router_logits, + num_experts=num_experts, + routing_bias=routing_bias, + ) + + assert len(captured) == 1 + replay = captured[0] + assert replay.dtype == torch.int16 + assert replay.shape == (num_tokens, top_k) + assert (replay >= 0).all(), f"got out-of-range values: {replay}" + assert (replay < num_experts).all(), f"got out-of-range values: {replay}" + for t in range(num_tokens): + unique = replay[t].unique() + assert unique.numel() == top_k, ( + f"token {t}: expected {top_k} distinct experts, " + f"got {unique.numel()} ({replay[t].tolist()})" + ) diff --git a/tests/model_executor/test_routed_experts_capture.py b/tests/model_executor/test_routed_experts_capture.py index 7bb7b8ba51..b80b9039e0 100644 --- a/tests/model_executor/test_routed_experts_capture.py +++ b/tests/model_executor/test_routed_experts_capture.py @@ -66,6 +66,12 @@ def _make_router(eplb_state: EplbLayerState | None = None) -> DummyRouter: ) +def _make_modular_routed_experts(): + return types.SimpleNamespace( + quant_method=types.SimpleNamespace(is_monolithic=False), + ) + + def test_base_router_capture_pre_eplb_mapping(): router = _make_router() captured = [] @@ -122,6 +128,8 @@ def test_gpu_model_runner_binds_router_capture(monkeypatch): def __init__(self): self.layer_id = 7 self.router = _make_router() + self.routed_experts = _make_modular_routed_experts() + self._quant_method = self.routed_experts.quant_method class DummyCapturer: def __init__(self): @@ -160,6 +168,8 @@ def test_gpu_model_runner_binding_stage(monkeypatch): def __init__(self): self.layer_id = 11 self.router = _make_router() + self.routed_experts = _make_modular_routed_experts() + self._quant_method = self.routed_experts.quant_method class DummyCapturer: def __init__(self): @@ -197,6 +207,8 @@ def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch): def __init__(self, layer_id): self.layer_id = layer_id self.router = _make_router() + self.routed_experts = _make_modular_routed_experts() + self._quant_method = self.routed_experts.quant_method target_module = DummyFusedMoE(layer_id=7) draft_module = DummyFusedMoE(layer_id=0) @@ -222,6 +234,49 @@ def test_gpu_model_runner_does_not_bind_draft_router_capture(monkeypatch): assert draft_module.router.capture_fn is None +def test_gpu_model_runner_rejects_monolithic_without_replay_support(monkeypatch): + from vllm.v1.worker import gpu_model_runner as gmr + + class DummyFusedMoE: + def __init__(self): + self.layer_id = 3 + self.router = _make_router() + # Use a concrete monolithic expert and override its capability + # instead of instantiating the abstract base class directly. + from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( + CPUExpertsFp8, + ) + + fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8) + self.routed_experts = types.SimpleNamespace( + quant_method=types.SimpleNamespace( + is_monolithic=True, + moe_kernel=types.SimpleNamespace( + impl=types.SimpleNamespace(fused_experts=fused_experts) + ), + ) + ) + self._quant_method = self.routed_experts.quant_method + self._quant_method.moe_kernel.impl.fused_experts = fused_experts + fused_experts.supports_routing_replay_capture = lambda: False + + class DummyCapturer: + def capture(self, layer_id, topk_ids): + pass + + dummy_module = DummyFusedMoE() + import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer + + monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE) + + dummy_self = types.SimpleNamespace( + model=types.SimpleNamespace(modules=lambda: [dummy_module]) + ) + + with pytest.raises(ValueError, match="monolithic MoE kernel"): + gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, DummyCapturer()) + + def test_routed_experts_capturer_single_dp_no_metadata(): """dp_metadata is None: capture writes the full topk_ids rows.""" capturer = _capturer_with_buffer(dp_rank=0) diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py index 143df9e81c..e3a85a1365 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py @@ -34,6 +34,9 @@ class TrtLlmBf16ExpertsBase: monolithic interfaces. """ + def supports_routing_replay_capture(self) -> bool: + return True + def __init__( self, moe_config: FusedMoEConfig, @@ -246,7 +249,11 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL] - return flashinfer.fused_moe.trtllm_bf16_moe( + routing_replay_out = self._maybe_make_routing_replay_buffer( + num_tokens=hidden_states.shape[0], + device=hidden_states.device, + ) + out = flashinfer.fused_moe.trtllm_bf16_moe( routing_logits=router_logits, routing_bias=e_score_correction_bias, hidden_states=hidden_states, @@ -263,4 +270,9 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol routing_method_type=self.routing_method_type, activation_type=activation_to_flashinfer_int(activation), tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), + routing_replay_out=routing_replay_out, ) + self._maybe_dispatch_routing_replay( + routing_replay_out, num_tokens=hidden_states.shape[0] + ) + return out diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py index 55ddd1a2c9..50ed4ed5c3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py @@ -265,6 +265,9 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit Fp8 TRTLLM-Gen MoE kernels. Supports monolithic interface. """ + def supports_routing_replay_capture(self) -> bool: + return True + def __init__( self, moe_config: FusedMoEConfig, @@ -403,6 +406,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit n_group = num_expert_group or 0 selected_topk_group = topk_group or 0 + routing_replay_out = self._maybe_make_routing_replay_buffer( + num_tokens=hidden_states.shape[0], + device=hidden_states.device, + ) + kwargs = dict( routing_logits=router_logits, routing_bias=e_score_correction_bias, @@ -427,11 +435,16 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit use_shuffled_weight=use_shuffled_weight, weight_layout=weight_layout, fp8_quantization_type=fp8_quant_type, + routing_replay_out=routing_replay_out, tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), ) if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL: kwargs["activation_type"] = activation_type - return flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs) + result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs) + self._maybe_dispatch_routing_replay( + routing_replay_out, num_tokens=hidden_states.shape[0] + ) + return result def _apply_per_tensor( self, @@ -464,6 +477,11 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit else: assert not apply_router_weight_on_input + routing_replay_out = self._maybe_make_routing_replay_buffer( + num_tokens=hidden_states.shape[0], + device=hidden_states.device, + ) + out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe( routing_logits=router_logits, routing_bias=e_score_correction_bias, @@ -485,6 +503,10 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit routing_method_type=self.routing_method_type, activation_type=activation_type, tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), + routing_replay_out=routing_replay_out, + ) + self._maybe_dispatch_routing_replay( + routing_replay_out, num_tokens=hidden_states.shape[0] ) return out diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py index 2b8a5529ab..e00f6901aa 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py @@ -122,6 +122,9 @@ class TrtLlmMxfp4ExpertsMonolithic( Wraps flashinfer.trtllm_fp4_block_scale_moe(). """ + def supports_routing_replay_capture(self) -> bool: + return True + @staticmethod def _supports_parallel_config( moe_parallel_config: FusedMoEParallelConfig, @@ -184,6 +187,10 @@ class TrtLlmMxfp4ExpertsMonolithic( device=hidden_states.device, ) + routing_replay_out = self._maybe_make_routing_replay_buffer( + num_tokens=hidden_states.shape[0], + device=hidden_states.device, + ) trtllm_fp4_block_scale_moe( routing_logits=router_logits.to(torch.bfloat16), routing_bias=None, @@ -213,8 +220,11 @@ class TrtLlmMxfp4ExpertsMonolithic( do_finalize=True, tune_max_num_tokens=max(self.max_capture_size, 1), output=output, + routing_replay_out=routing_replay_out, + ) + self._maybe_dispatch_routing_replay( + routing_replay_out, num_tokens=hidden_states.shape[0] ) - return output diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py index c6e5e70a14..a9e725f296 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py @@ -122,6 +122,9 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): # The kernel handles quantization internally. return True + def supports_routing_replay_capture(self) -> bool: + return True + def apply( self, hidden_states: torch.Tensor, @@ -144,7 +147,12 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): assert self.w1_scale is not None assert self.w2_scale is not None - return flashinfer_trtllm_mxint4_moe( + + routing_replay_out = self._maybe_make_routing_replay_buffer( + num_tokens=hidden_states.shape[0], + device=hidden_states.device, + ) + result = flashinfer_trtllm_mxint4_moe( x=hidden_states, router_logits=router_logits, w13_weight_packed=w1, @@ -160,4 +168,9 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): topk_group=topk_group, e_score_correction_bias=e_score_correction_bias, routing_method_type=self.routing_method, + routing_replay_out=routing_replay_out, ) + self._maybe_dispatch_routing_replay( + routing_replay_out, num_tokens=hidden_states.shape[0] + ) + return result diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py index 2a14434173..b6c2429139 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -435,6 +435,9 @@ class TrtLlmNvFp4ExpertsMonolithic( Monolithic version of the kernel (router + experts). """ + def supports_routing_replay_capture(self) -> bool: + return True + @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: """The modular implementation should be used for the Dp/Ep or EPLB case.""" @@ -509,10 +512,14 @@ class TrtLlmNvFp4ExpertsMonolithic( output1_scale_gate_scalar = self.quant_config.g1_alphas + routing_replay_out = self._maybe_make_routing_replay_buffer( + num_tokens=hidden_states.shape[0], + device=hidden_states.device, + ) # Invoke kernel. # NOTE: Activation padding and output # truncation are handled by the MoE runner's - return flashinfer.fused_moe.trtllm_fp4_block_scale_moe( + result = flashinfer.fused_moe.trtllm_fp4_block_scale_moe( routing_logits=router_logits, routing_bias=e_score_correction_bias, hidden_states=hidden_states, @@ -544,4 +551,9 @@ class TrtLlmNvFp4ExpertsMonolithic( activation_type=activation_to_flashinfer_int(activation), per_token_scale=per_token_scale, tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), + routing_replay_out=routing_replay_out, )[0] + self._maybe_dispatch_routing_replay( + routing_replay_out, num_tokens=hidden_states.shape[0] + ) + return result diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index 5c188e5660..0eb411dabd 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -1000,6 +1000,57 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts): def is_monolithic() -> bool: return True + routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None + _routing_replay_buffer: torch.Tensor | None = None + + def supports_routing_replay_capture(self) -> bool: + """Whether this expert supports routing replay capture. + + Subclasses backed by a kernel that exposes routed expert IDs + (e.g. FlashInfer's ``routing_replay_out``) should override. + """ + return False + + def set_capture_fn( + self, + capture_fn: Callable[[torch.Tensor], None] | None, + ) -> None: + self.routing_replay_capture_fn = capture_fn + if capture_fn is None: + self._routing_replay_buffer = None + return + self._routing_replay_buffer = torch.empty( + (self.moe_config.max_num_tokens, self.moe_config.experts_per_token), + dtype=torch.int16, + device=self.moe_config.device, + ) + + def _maybe_make_routing_replay_buffer( + self, + num_tokens: int, + device: torch.device, + ) -> torch.Tensor | None: + if self.routing_replay_capture_fn is None: + return None + buf = self._routing_replay_buffer + assert buf is not None + if buf.shape[0] < num_tokens or buf.device != device: + raise ValueError( + "Routing replay buffer was initialized for " + f"{buf.shape[0]} tokens on {buf.device}, but the kernel " + f"received {num_tokens} tokens on {device}." + ) + return buf + + def _maybe_dispatch_routing_replay( + self, + routing_replay_out: torch.Tensor | None, + num_tokens: int, + ) -> None: + if routing_replay_out is None or self.routing_replay_capture_fn is None: + return + self.routing_replay_capture_fn(routing_replay_out[:num_tokens]) + def apply( self, hidden_states: torch.Tensor, diff --git a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py index 115f43a58b..c5553a90eb 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py @@ -90,10 +90,20 @@ class RoutedExpertsCapturer: ) -> None: hf_config = vllm_config.model_config.hf_text_config num_experts_per_tok = _get_num_experts_per_tok(hf_config) + num_layers = hf_config.num_hidden_layers + logger.info( + "RoutedExpertsCapturer: allocating buffer with " + "max_tokens=%d, num_layers=%d, num_experts_per_tok=%d " + "(hf_config.model_type=%s)", + max_num_batched_tokens, + num_layers, + num_experts_per_tok, + getattr(hf_config, "model_type", "unknown"), + ) self.device_buffer = torch.zeros( ( max_num_batched_tokens, - hf_config.num_hidden_layers, + num_layers, num_experts_per_tok, ), # Use int32 for the device / host transit buffers: it diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_mxint4_moe.py b/vllm/model_executor/layers/quantization/utils/flashinfer_mxint4_moe.py index 4e08a73a69..1b5320615f 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_mxint4_moe.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_mxint4_moe.py @@ -190,6 +190,7 @@ def flashinfer_trtllm_mxint4_moe( topk_group: int | None = None, e_score_correction_bias: torch.Tensor | None = None, routing_method_type: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: """ Apply FlashInfer TensorRT-LLM MxInt4 MoE kernel. @@ -262,6 +263,7 @@ def flashinfer_trtllm_mxint4_moe( do_finalize=True, output=None, tune_max_num_tokens=8192, + routing_replay_out=routing_replay_out, ) if isinstance(out, (tuple, list)): out = out[0] diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index f3f0afb130..3d68cb08d4 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -7676,17 +7676,37 @@ class GPUModelRunner( def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None: from vllm.model_executor.layers.fused_moe.layer import MoERunner + from vllm.model_executor.layers.fused_moe.modular_kernel import ( + FusedMoEExpertsMonolithic, + ) from vllm.model_executor.layers.fused_moe.router.base_router import ( BaseRouter, ) for module in self.model.modules(): - if isinstance(module, MoERunner) and isinstance(module.router, BaseRouter): - layer_id = module.layer_id + if not isinstance(module, MoERunner): + continue + layer_id = module.layer_id - def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer): - _capturer.capture(_layer_id, topk_ids) + def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer): + _capturer.capture(_layer_id, topk_ids) + quant_method = module._quant_method + moe_kernel = getattr(quant_method, "moe_kernel", None) + impl = getattr(moe_kernel, "impl", None) + fused_experts = getattr(impl, "fused_experts", None) + if quant_method.is_monolithic: + if not ( + isinstance(fused_experts, FusedMoEExpertsMonolithic) + and fused_experts.supports_routing_replay_capture() + ): + raise ValueError( + "--enable-return-routed-experts is not supported with " + f"monolithic MoE kernel {type(fused_experts).__name__}; " + "routed expert IDs would be silently all-zero." + ) + fused_experts.set_capture_fn(_capture_fn) + elif isinstance(module.router, BaseRouter): module.router.set_capture_fn(_capture_fn) def may_add_encoder_only_layers_to_kv_cache_config(self) -> None: