From 4c4003bb5f847e7a84152cbf0e6ff58cadfceb52 Mon Sep 17 00:00:00 2001 From: Eric Tang <46737979+erictang000@users.noreply.github.com> Date: Fri, 2 Oct 2026 08:50:38 -0700 Subject: [PATCH] [Bugfix] Bind routed-experts capture to the MoE layer, not the kernel (#59455) Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Claude Opus 5.5 --- tests/kernels/moe/test_flashinfer.py | 2 + tests/kernels/moe/test_ocp_mx_moe.py | 2 + .../test_routed_experts_capture_monolithic.py | 985 ++++-------------- .../test_routed_experts_capture.py | 160 ++- .../experts/aiter_mxfp4_w4a16_moe.py | 1 + .../fused_moe/experts/aiter_mxfp4_w4a8_moe.py | 1 + .../layers/fused_moe/experts/cpu_int4_moe.py | 2 + .../experts/gpt_oss_triton_kernels_moe.py | 1 + .../fused_moe/experts/trtllm_bf16_moe.py | 6 +- .../fused_moe/experts/trtllm_fp8_moe.py | 21 +- .../fused_moe/experts/trtllm_mxfp4_moe.py | 6 +- .../fused_moe/experts/trtllm_mxint4_moe.py | 8 +- .../fused_moe/experts/trtllm_nvfp4_moe.py | 6 +- .../layers/fused_moe/experts/zentorch_moe.py | 1 + .../layers/fused_moe/modular_kernel.py | 74 +- .../layers/fused_moe/routed_experts.py | 5 + .../fused_moe/routed_experts_capturer.py | 22 +- .../fused_moe/unquantized_fused_moe_method.py | 1 + .../layers/quantization/auto_awq.py | 1 + .../layers/quantization/auto_gptq.py | 1 + .../compressed_tensors_moe_w4a4_nvfp4.py | 1 + .../compressed_tensors_moe_w4a8_int8.py | 1 + .../compressed_tensors_moe_w8a8_fp8.py | 1 + .../compressed_tensors_moe_w8a8_int8.py | 1 + .../compressed_tensors_moe_w8a8_mxfp8.py | 1 + .../compressed_tensors_moe_wna16.py | 1 + .../model_executor/layers/quantization/fp8.py | 1 + .../layers/quantization/modelopt.py | 3 + .../layers/quantization/moe_wna16.py | 1 + .../layers/quantization/mxfp4.py | 2 + .../layers/quantization/online/moe_base.py | 1 + .../layers/quantization/quark/quark_moe.py | 1 + 32 files changed, 423 insertions(+), 898 deletions(-) diff --git a/tests/kernels/moe/test_flashinfer.py b/tests/kernels/moe/test_flashinfer.py index 2a46a0a56a..629759de40 100644 --- a/tests/kernels/moe/test_flashinfer.py +++ b/tests/kernels/moe/test_flashinfer.py @@ -295,6 +295,7 @@ def test_flashinfer_per_tensor_moe_fp8_no_graph( expert_map=None, apply_router_weight_on_input=True, routed_scaling_factor=1.0, + routing_sink=None, ) check_accuracy( @@ -968,6 +969,7 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in router_logits=logits, e_score_correction_bias=bias, routed_scaling_factor=scale, + routing_sink=None, **common, ) topk_weights, topk_ids = fused_topk_bias( diff --git a/tests/kernels/moe/test_ocp_mx_moe.py b/tests/kernels/moe/test_ocp_mx_moe.py index f574c4d5d1..5a7053a8b1 100644 --- a/tests/kernels/moe/test_ocp_mx_moe.py +++ b/tests/kernels/moe/test_ocp_mx_moe.py @@ -781,6 +781,7 @@ def test_trtllm_mxfp4_deferred_finalize( global_num_experts=num_experts, expert_map=None, apply_router_weight_on_input=False, + routing_sink=None, ), ) return @@ -1780,6 +1781,7 @@ def test_rocm_mxfp4_moe_oracle( global_num_experts=num_experts, expert_map=None, apply_router_weight_on_input=False, + routing_sink=None, ) else: # Modular impl uses topk_weights and topk_ids diff --git a/tests/kernels/moe/test_routed_experts_capture_monolithic.py b/tests/kernels/moe/test_routed_experts_capture_monolithic.py index eec6180550..f15d8ab77f 100644 --- a/tests/kernels/moe/test_routed_experts_capture_monolithic.py +++ b/tests/kernels/moe/test_routed_experts_capture_monolithic.py @@ -1,22 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""End-to-end tests for routed-expert capture on the monolithic MoE path. +"""Routed-expert capture on the monolithic (fused router + experts) MoE path. -These tests exercise the wiring that lets ``RoutedExpertsCapturer`` see the -expert IDs picked by FlashInfer's fused router-and-experts kernels (the -"monolithic" path). When ``set_capture_fn`` is installed on -a ``FusedMoEExpertsMonolithic`` subclass that supports it, the kernel call -should: - - * allocate an int16 ``(num_tokens, top_k)`` buffer, - * pass it to FlashInfer as ``routing_replay_out``, - * invoke the callback after the kernel returns. +A monolithic kernel that ``supports_routing_replay_capture`` must write, into +the leading rows of ``routing_replay_out``, exactly the experts it routed each +token to. The router logits plant an unambiguous top-k per token, so the test +checks the reported experts against that answer and not just their range. """ from __future__ import annotations -from types import SimpleNamespace -from unittest.mock import patch +from collections.abc import Callable import pytest import torch @@ -38,6 +32,9 @@ from vllm.model_executor.layers.fused_moe.experts.trtllm_fp8_moe import ( from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import ( TrtLlmNvFp4ExpertsMonolithic, ) +from vllm.model_executor.layers.fused_moe.modular_kernel import ( + FusedMoEExpertsMonolithic, +) from vllm.platforms import current_platform try: @@ -57,822 +54,250 @@ if not current_platform.is_device_capability_family(100): allow_module_level=True, ) +NUM_EXPERTS = 32 +NUM_GROUPS = 4 # DeepSeekV3 grouped routing +TOPK_GROUPS = 2 +HIDDEN = 1024 +INTERMEDIATE = 1024 +DEVICE = torch.device("cuda:0") -def _shuffle_bf16_weights_block_major_k( - w: torch.Tensor, epilogue_tile_m: int = 64, block_k: int = 128 -) -> torch.Tensor: - """Reshape ``w`` (E, M, K) into the ``BlockMajorK`` layout expected by - ``trtllm_bf16_moe``: ``(E, K/block_k, M, block_k)`` after a per-expert - row shuffle. +# A kernel under test, and its call: (router_logits, routing_replay_out) -> None. +Kernel = tuple[FusedMoEExpertsMonolithic, Callable[[torch.Tensor, torch.Tensor], None]] + + +def _planted_routing(num_tokens: int, top_k: int) -> tuple[torch.Tensor, torch.Tensor]: + """Router logits with one unambiguous answer under every routing method. + + Each token gives +4 to ``top_k`` experts drawn from at most two expert + groups and -4 to the rest, so softmax top-k, sigmoid top-k and DeepSeekV3 + grouped top-k all pick the same experts, with margins no rounding can flip. + Returns the logits and the planted experts of each token, sorted. """ - from flashinfer import shuffle_matrix_a - from flashinfer.fused_moe import convert_to_block_layout - - num_experts = w.shape[0] - shuffled = [] - for i in range(num_experts): - t = shuffle_matrix_a(w[i].view(torch.uint8), epilogue_tile_m) - shuffled.append(convert_to_block_layout(t, block_k)) - return torch.stack(shuffled).view(torch.bfloat16) + group_size = NUM_EXPERTS // NUM_GROUPS + planted = [] + for _ in range(num_tokens): + groups = torch.randperm(NUM_GROUPS)[:TOPK_GROUPS] + candidates = (groups[:, None] * group_size + torch.arange(group_size)).flatten() + planted.append(candidates[torch.randperm(len(candidates))[:top_k]]) + experts = torch.stack(planted) + logits = torch.full((num_tokens, NUM_EXPERTS), -4.0).scatter_(1, experts, 4.0) + return logits.to(DEVICE), experts.sort(dim=-1).values.to(DEVICE) -def _make_bf16_monolithic_experts( - num_experts: int, - top_k: int, - hidden_size: int, - intermediate_size: int, - routing_method: RoutingMethodType, - device: torch.device, -) -> tuple[TrtLlmBf16ExpertsMonolithic, torch.Tensor, torch.Tensor]: - """Construct the monolithic BF16 experts plus the BlockMajorK weights - expected by ``trtllm_bf16_moe``. - """ - parallel_cfg = FusedMoEParallelConfig.make_no_parallel() - moe_config = FusedMoEConfig( - num_experts=num_experts, +def _routing_kwargs(routing_method: RoutingMethodType) -> dict: + if routing_method != RoutingMethodType.DeepSeekV3: + return {} + return dict( + num_expert_group=NUM_GROUPS, + topk_group=TOPK_GROUPS, + e_score_correction_bias=torch.zeros( + NUM_EXPERTS, device=DEVICE, dtype=torch.bfloat16 + ), + routed_scaling_factor=1.0, + ) + + +def _moe_config(top_k: int, routing_method: RoutingMethodType) -> FusedMoEConfig: + return FusedMoEConfig( + num_experts=NUM_EXPERTS, experts_per_token=top_k, - hidden_dim=hidden_size, - intermediate_size=intermediate_size, - num_local_experts=num_experts, - num_logical_experts=num_experts, - moe_parallel_config=parallel_cfg, + hidden_dim=HIDDEN, + intermediate_size=INTERMEDIATE, + num_local_experts=NUM_EXPERTS, + num_logical_experts=NUM_EXPERTS, + moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(), in_dtype=torch.bfloat16, activation=MoEActivation.SILU, - device=device, + device=DEVICE, routing_method=routing_method, max_num_tokens=16, ) - quant_config = FusedMoEQuantConfig.make( - quant_dtype=None, - per_act_token_quant=False, - per_out_ch_quant=False, - block_shape=None, - ) + + +def _bf16_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel: + from flashinfer import shuffle_matrix_a + from flashinfer.fused_moe import convert_to_block_layout + + def block_major_k(w: torch.Tensor) -> torch.Tensor: + """(E, M, K) -> the BlockMajorK layout ``trtllm_bf16_moe`` expects.""" + return torch.stack( + [ + convert_to_block_layout(shuffle_matrix_a(e.view(torch.uint8), 64), 128) + for e in w + ] + ).view(torch.bfloat16) experts = TrtLlmBf16ExpertsMonolithic( - moe_config=moe_config, quant_config=quant_config + moe_config=_moe_config(top_k, routing_method), + quant_config=FusedMoEQuantConfig.make( + quant_dtype=None, + per_act_token_quant=False, + per_out_ch_quant=False, + block_shape=None, + ), ) - - gemm1 = ( - torch.randn( - num_experts, - 2 * intermediate_size, - hidden_size, - device=device, - dtype=torch.bfloat16, - ) + w13 = block_major_k( + torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).bfloat16() * 0.1 ) - gemm2 = ( - torch.randn( - num_experts, - hidden_size, - intermediate_size, - device=device, - dtype=torch.bfloat16, - ) - * 0.1 - ) - w13 = _shuffle_bf16_weights_block_major_k(gemm1) - w2 = _shuffle_bf16_weights_block_major_k(gemm2) - return experts, w13, w2 - - -def _run_bf16_monolithic( - experts: TrtLlmBf16ExpertsMonolithic, - hidden_states: torch.Tensor, - w13: torch.Tensor, - w2: torch.Tensor, - router_logits: torch.Tensor, - num_experts: int, - *, - n_group: int | None = None, - topk_group: int | None = None, - routed_scaling_factor: float | None = None, - e_score_correction_bias: torch.Tensor | None = None, -) -> torch.Tensor: - return experts.apply( - hidden_states=hidden_states, - w1=w13, - w2=w2, - router_logits=router_logits, - activation=MoEActivation.SILU, - global_num_experts=num_experts, - expert_map=None, - a1q_scale=None, - apply_router_weight_on_input=False, - num_expert_group=n_group, - topk_group=topk_group, - e_score_correction_bias=e_score_correction_bias, - routed_scaling_factor=routed_scaling_factor, + w2 = block_major_k( + torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).bfloat16() * 0.1 ) - -_DSV3_NUM_EXPERTS = 32 -_DSV3_N_GROUP = 4 -_DSV3_TOPK_GROUP = 2 - - -def _make_dsv3_routing_bias(num_experts: int, device: torch.device) -> torch.Tensor: - return torch.randn(num_experts, device=device, dtype=torch.bfloat16) - - -@pytest.mark.parametrize("num_tokens", [2, 7, 16]) -@pytest.mark.parametrize("top_k", [2, 4]) -def test_trtllm_bf16_monolithic_routing_replay_records_valid_experts( - num_tokens: int, - top_k: int, -) -> None: - """The capture callback should receive the int16 routed-expert IDs the - kernel actually used, the values should be valid expert indices, and - each token should pick ``top_k`` distinct experts.""" - if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP: - pytest.skip( - f"DSV3 requires top_k <= n_group * topk_group " - f"({_DSV3_N_GROUP * _DSV3_TOPK_GROUP})" - ) - torch.manual_seed(0) - device = torch.device("cuda:0") - - num_experts = _DSV3_NUM_EXPERTS - hidden_size = 1024 - intermediate_size = 1024 - - experts, w13, w2 = _make_bf16_monolithic_experts( - num_experts=num_experts, - top_k=top_k, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - routing_method=RoutingMethodType.DeepSeekV3, - device=device, - ) - - captured: list[torch.Tensor] = [] - - def capture_fn(replay_out: torch.Tensor) -> None: - captured.append(replay_out.clone()) - - assert experts.supports_routing_replay_capture() - experts.set_capture_fn(capture_fn) - - hidden_states = ( - torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 - ) - router_logits = torch.rand( - num_tokens, num_experts, device=device, dtype=torch.float32 - ) - routing_bias = _make_dsv3_routing_bias(num_experts, device) - - _ = _run_bf16_monolithic( - experts, - hidden_states=hidden_states, - w13=w13, - w2=w2, - router_logits=router_logits, - num_experts=num_experts, - n_group=_DSV3_N_GROUP, - topk_group=_DSV3_TOPK_GROUP, - routed_scaling_factor=1.0, - e_score_correction_bias=routing_bias, - ) - - assert len(captured) == 1 - replay = captured[0] - assert replay.dtype == torch.int16 - assert replay.shape == (num_tokens, top_k) - assert (replay >= 0).all(), f"got out-of-range values: {replay}" - assert (replay < num_experts).all(), f"got out-of-range values: {replay}" - - for t in range(num_tokens): - unique = replay[t].unique() - assert unique.numel() == top_k, ( - f"token {t}: expected {top_k} distinct experts, " - f"got {unique.numel()} ({replay[t].tolist()})" - ) - - -@pytest.mark.parametrize("num_tokens", [2, 7, 16]) -@pytest.mark.parametrize( - "routing_method", - [ - RoutingMethodType.Renormalize, - RoutingMethodType.RenormalizeNaive, - ], -) -def test_trtllm_bf16_monolithic_routing_replay_non_dsv3( - num_tokens: int, - routing_method: RoutingMethodType, -) -> None: - """Routing replay works for non-DeepSeekV3 routing methods too. - FlashInfer's ``routing_replay_out`` is routing-method-agnostic.""" - torch.manual_seed(0) - device = torch.device("cuda:0") - - num_experts = 8 - top_k = 2 - hidden_size = 1024 - intermediate_size = 1024 - - experts, w13, w2 = _make_bf16_monolithic_experts( - num_experts=num_experts, - top_k=top_k, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - routing_method=routing_method, - device=device, - ) - - captured: list[torch.Tensor] = [] - experts.set_capture_fn(lambda r: captured.append(r.clone())) - - hidden_states = ( - torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 - ) - router_logits = torch.rand( - num_tokens, num_experts, device=device, dtype=torch.float32 - ) - - _ = _run_bf16_monolithic( - experts, - hidden_states=hidden_states, - w13=w13, - w2=w2, - router_logits=router_logits, - num_experts=num_experts, - ) - - assert len(captured) == 1 - replay = captured[0] - assert replay.dtype == torch.int16 - assert replay.shape == (num_tokens, top_k) - assert (replay >= 0).all(), f"got out-of-range values: {replay}" - assert (replay < num_experts).all(), f"got out-of-range values: {replay}" - for t in range(num_tokens): - unique = replay[t].unique() - assert unique.numel() == top_k, ( - f"token {t}: expected {top_k} distinct experts, " - f"got {unique.numel()} ({replay[t].tolist()})" - ) - - -def test_trtllm_bf16_monolithic_capture_disabled_skips_buffer_alloc() -> None: - """With no callback installed the kernel should not see a - ``routing_replay_out`` tensor — verify the helper short-circuits.""" - torch.manual_seed(0) - device = torch.device("cuda:0") - experts, _, _ = _make_bf16_monolithic_experts( - num_experts=_DSV3_NUM_EXPERTS, - top_k=2, - hidden_size=1024, - intermediate_size=1024, - routing_method=RoutingMethodType.DeepSeekV3, - device=device, - ) - # No callback installed. - buf = experts._maybe_make_routing_replay_buffer(num_tokens=4, device=device) - assert buf is None - - # Dispatch is also a no-op. - experts._maybe_dispatch_routing_replay(buf, num_tokens=4) - - -def test_trtllm_bf16_monolithic_supports_capture_for_all_routing() -> None: - """FlashInfer's ``routing_replay_out`` is supported by all routing - methods, so ``supports_routing_replay_capture`` should be True - regardless of routing method.""" - device = torch.device("cuda:0") - for routing_method in ( - RoutingMethodType.DeepSeekV3, - RoutingMethodType.Renormalize, - RoutingMethodType.RenormalizeNaive, - ): - experts, _, _ = _make_bf16_monolithic_experts( - num_experts=_DSV3_NUM_EXPERTS, - top_k=2, - hidden_size=1024, - intermediate_size=1024, - routing_method=routing_method, - device=device, - ) - assert experts.supports_routing_replay_capture() is True, ( - f"{routing_method!r} should support routing replay capture" - ) - - -def test_trtllm_bf16_monolithic_capture_buffer_shape_and_dtype() -> None: - """When capture is installed, the allocated buffer is int16 and shaped - ``(num_tokens, experts_per_token)``.""" - device = torch.device("cuda:0") - experts, _, _ = _make_bf16_monolithic_experts( - num_experts=_DSV3_NUM_EXPERTS, - top_k=4, - hidden_size=1024, - intermediate_size=1024, - routing_method=RoutingMethodType.DeepSeekV3, - device=device, - ) - experts.set_capture_fn(lambda r: None) - buf = experts._maybe_make_routing_replay_buffer(num_tokens=11, device=device) - assert buf is not None - assert buf.dtype == torch.int16 - assert buf.shape[0] >= 11 - assert buf.shape[1] == 4 - assert buf.device.type == "cuda" - - -def test_routed_experts_capturer_e2e_via_monolithic_experts() -> None: - """End-to-end: bind ``RoutedExpertsCapturer.capture`` as the callback - on the monolithic experts and verify the captured rows land in the - capturer's device buffer at the correct layer slot. - - Mirrors the monolithic path in ``bind_routed_experts_capturer``: a single - closure is installed on ``fused_experts`` and routes per-layer based on - the closed-over ``layer_id``. - """ - from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( - RoutedExpertsCapturer, - ) - - torch.manual_seed(7) - device = torch.device("cuda:0") - num_tokens = 4 - top_k = 2 - num_experts = _DSV3_NUM_EXPERTS - hidden_size = 1024 - intermediate_size = 1024 - - experts, w13, w2 = _make_bf16_monolithic_experts( - num_experts=num_experts, - top_k=top_k, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - routing_method=RoutingMethodType.DeepSeekV3, - device=device, - ) - - num_layers = 3 - layer_id = 1 - capturer = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer) - capturer.dp_rank = 0 - capturer.tp_size = 1 - capturer.device_buffer = torch.full( - (num_tokens + 4, num_layers, top_k), - -1, - dtype=torch.int32, - device=device, - ) - - def capture_fn(replay_out: torch.Tensor) -> None: - capturer.capture(layer_id, replay_out) - - experts.set_capture_fn(capture_fn) - - hidden_states = ( - torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 - ) - router_logits = torch.rand( - num_tokens, num_experts, device=device, dtype=torch.float32 - ) - routing_bias = _make_dsv3_routing_bias(num_experts, device) - - # Patch get_forward_context to return a dp_metadata=None context so the - # capturer takes the single-DP branch. - import vllm.model_executor.layers.fused_moe.routed_experts_capturer as rec - - with patch.object( - rec, - "get_forward_context", - return_value=SimpleNamespace(dp_metadata=None), - ): - _ = _run_bf16_monolithic( - experts, - hidden_states=hidden_states, - w13=w13, + def forward(router_logits, routing_replay_out): + hidden_states = torch.randn(len(router_logits), HIDDEN, device=DEVICE) * 0.1 + experts.apply( + hidden_states=hidden_states.bfloat16(), + w1=w13, w2=w2, router_logits=router_logits, - num_experts=num_experts, - n_group=_DSV3_N_GROUP, - topk_group=_DSV3_TOPK_GROUP, - routed_scaling_factor=1.0, - e_score_correction_bias=routing_bias, + activation=MoEActivation.SILU, + global_num_experts=NUM_EXPERTS, + expert_map=None, + a1q_scale=None, + apply_router_weight_on_input=False, + routing_replay_out=routing_replay_out, + **_routing_kwargs(routing_method), ) - captured = capturer.device_buffer[:num_tokens, layer_id, :].cpu() - # Valid expert IDs at this layer. - assert (captured >= 0).all() - assert (captured < num_experts).all() - for t in range(num_tokens): - unique = captured[t].unique() - assert unique.numel() == top_k, ( - f"token {t}: expected {top_k} distinct experts at layer " - f"{layer_id}, got {unique.numel()}" - ) - - # Other layers / trailing token rows untouched. - for other_layer in range(num_layers): - if other_layer == layer_id: - continue - assert (capturer.device_buffer[:, other_layer, :].cpu() == -1).all(), ( - f"layer {other_layer} should be untouched, got writes" - ) - assert (capturer.device_buffer[num_tokens:, layer_id, :].cpu() == -1).all(), ( - "tail rows beyond num_tokens should remain sentinel" - ) + return experts, forward -# ---------------------------------------------------------------------------- -# FP8 block-scale (DeepSeekFp8) — vLLM's ``TrtLlmFp8ExpertsMonolithic`` -# ---------------------------------------------------------------------------- - - -def _make_fp8_block_scale_monolithic_experts( - num_experts: int, - top_k: int, - hidden_size: int, - intermediate_size: int, - device: torch.device, -) -> tuple[TrtLlmFp8ExpertsMonolithic, torch.Tensor, torch.Tensor]: - """Set up ``TrtLlmFp8ExpertsMonolithic`` for the DeepSeekFp8 block-scale - code path with DSV3 routing. - - Weights are shuffled into the BlockMajorK layout the kernel expects - (same helper the vLLM weight loader uses for DeepSeek-FP8 models). - """ +def _fp8_block_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel: from vllm.model_executor.layers.quantization.utils.flashinfer_utils import ( _shuffle_deepseek_fp8_moe_weights, ) - block_k = 128 - parallel_cfg = FusedMoEParallelConfig.make_no_parallel() - moe_config = FusedMoEConfig( - num_experts=num_experts, - experts_per_token=top_k, - hidden_dim=hidden_size, - intermediate_size=intermediate_size, - num_local_experts=num_experts, - num_logical_experts=num_experts, - moe_parallel_config=parallel_cfg, - in_dtype=torch.bfloat16, - activation=MoEActivation.SILU, - device=device, - routing_method=RoutingMethodType.DeepSeekV3, - max_num_tokens=16, + block = 128 + w13, w2 = _shuffle_deepseek_fp8_moe_weights( + torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).to( + torch.float8_e4m3fn + ), + torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).to( + torch.float8_e4m3fn + ), ) - - # Random fp8 weights + ones-block scales (the kernel decoder only cares - # that the per-block scales are present and finite for routing/replay). - gemm1 = torch.randn( - num_experts, 2 * intermediate_size, hidden_size, device=device - ).to(torch.float8_e4m3fn) - gemm2 = torch.randn(num_experts, hidden_size, intermediate_size, device=device).to( - torch.float8_e4m3fn - ) - w13_shuffled, w2_shuffled = _shuffle_deepseek_fp8_moe_weights(gemm1, gemm2) - - w1_scale = torch.ones( - num_experts, - 2 * intermediate_size // block_k, - hidden_size // block_k, - device=device, - dtype=torch.float32, - ) - w2_scale = torch.ones( - num_experts, - hidden_size // block_k, - intermediate_size // block_k, - device=device, - dtype=torch.float32, - ) - quant_config = fp8_w8a8_moe_quant_config( - w1_scale=w1_scale, - w2_scale=w2_scale, - block_shape=[block_k, block_k], - per_act_token_quant=False, - ) - experts = TrtLlmFp8ExpertsMonolithic( - moe_config=moe_config, quant_config=quant_config - ) - return experts, w13_shuffled, w2_shuffled - - -def _run_fp8_block_scale_monolithic( - experts: TrtLlmFp8ExpertsMonolithic, - hidden_states_fp8: torch.Tensor, - hidden_states_scale: torch.Tensor, - w13: torch.Tensor, - w2: torch.Tensor, - router_logits: torch.Tensor, - num_experts: int, - routing_bias: torch.Tensor, -) -> torch.Tensor: - return experts.apply( - hidden_states=hidden_states_fp8, - w1=w13, - w2=w2, - router_logits=router_logits, - activation=MoEActivation.SILU, - global_num_experts=num_experts, - expert_map=None, - # The block-scale apply path reads ``a1q_scale`` and transposes it - # to ``(hidden_size/128, num_tokens)`` for the kernel call. - a1q_scale=hidden_states_scale, - apply_router_weight_on_input=False, - num_expert_group=_DSV3_N_GROUP, - topk_group=_DSV3_TOPK_GROUP, - e_score_correction_bias=routing_bias, - routed_scaling_factor=1.0, + moe_config=_moe_config(top_k, routing_method), + quant_config=fp8_w8a8_moe_quant_config( + w1_scale=torch.ones( + NUM_EXPERTS, 2 * INTERMEDIATE // block, HIDDEN // block, device=DEVICE + ), + w2_scale=torch.ones( + NUM_EXPERTS, HIDDEN // block, INTERMEDIATE // block, device=DEVICE + ), + block_shape=[block, block], + per_act_token_quant=False, + ), ) - -@pytest.mark.parametrize("num_tokens", [2, 7, 16]) -@pytest.mark.parametrize("top_k", [2, 4]) -def test_trtllm_fp8_block_scale_monolithic_routing_replay_records_valid_experts( - num_tokens: int, - top_k: int, -) -> None: - """End-to-end: ``TrtLlmFp8ExpertsMonolithic`` (DeepSeekFp8 block-scale - path, DSV3 routing) captures valid expert IDs.""" - if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP: - pytest.skip( - f"DSV3 requires top_k <= n_group * topk_group " - f"({_DSV3_N_GROUP * _DSV3_TOPK_GROUP})" - ) - torch.manual_seed(0) - device = torch.device("cuda:0") - - num_experts = _DSV3_NUM_EXPERTS - hidden_size = 1024 - intermediate_size = 1024 - - experts, w13, w2 = _make_fp8_block_scale_monolithic_experts( - num_experts=num_experts, - top_k=top_k, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - device=device, - ) - assert experts.supports_routing_replay_capture() - - captured: list[torch.Tensor] = [] - experts.set_capture_fn(lambda r: captured.append(r.clone())) - - # Per-token / per-block hidden scales (ones is fine for the routing - # path; the GEMM output isn't being asserted on). - hidden_states = ( - torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 - ).to(torch.float8_e4m3fn) - hidden_states_scale = torch.ones( - num_tokens, hidden_size // 128, device=device, dtype=torch.float32 - ) - router_logits = torch.rand( - num_tokens, num_experts, device=device, dtype=torch.float32 - ) - routing_bias = _make_dsv3_routing_bias(num_experts, device) - - _ = _run_fp8_block_scale_monolithic( - experts, - hidden_states_fp8=hidden_states, - hidden_states_scale=hidden_states_scale, - w13=w13, - w2=w2, - router_logits=router_logits, - num_experts=num_experts, - routing_bias=routing_bias, - ) - - assert len(captured) == 1 - replay = captured[0] - assert replay.dtype == torch.int16 - assert replay.shape == (num_tokens, top_k) - assert (replay >= 0).all(), f"got out-of-range values: {replay}" - assert (replay < num_experts).all(), f"got out-of-range values: {replay}" - for t in range(num_tokens): - unique = replay[t].unique() - assert unique.numel() == top_k, ( - f"token {t}: expected {top_k} distinct experts, " - f"got {unique.numel()} ({replay[t].tolist()})" + def forward(router_logits, routing_replay_out): + num_tokens = len(router_logits) + hidden_states = torch.randn(num_tokens, HIDDEN, device=DEVICE) * 0.1 + experts.apply( + hidden_states=hidden_states.to(torch.float8_e4m3fn), + w1=w13, + w2=w2, + router_logits=router_logits, + activation=MoEActivation.SILU, + global_num_experts=NUM_EXPERTS, + expert_map=None, + # Per-block activation scales, (num_tokens, hidden / 128). + a1q_scale=torch.ones(num_tokens, HIDDEN // block, device=DEVICE), + apply_router_weight_on_input=False, + routing_replay_out=routing_replay_out, + **_routing_kwargs(routing_method), ) - -# ---------------------------------------------------------------------------- -# NVFP4 — vLLM's ``TrtLlmNvFp4ExpertsMonolithic`` -# ---------------------------------------------------------------------------- + return experts, forward -def _make_nvfp4_monolithic_experts( - num_experts: int, - top_k: int, - hidden_size: int, - intermediate_size: int, - device: torch.device, -) -> tuple[ - TrtLlmNvFp4ExpertsMonolithic, - torch.Tensor, # w13 (packed nvfp4 uint8) - torch.Tensor, # w13 block-scale (fp8) - torch.Tensor, # w2 (packed nvfp4 uint8) - torch.Tensor, # w2 block-scale (fp8) - torch.Tensor, # input global scale (per-tensor float32) -]: - """Set up ``TrtLlmNvFp4ExpertsMonolithic`` with NVFP4-quantized weights - and DSV3 routing. - - NVFP4 = per-block-of-16 fp4 with an fp8 scale, plus a per-tensor - "global" scale. We follow the layout in ``test_ocp_mx_moe.py`` / - ``flashinfer/tests/moe/test_trtllm_gen_routed_fused_moe.py``: - * weights: uint8 (packed fp4) ``(E, M, K//2)`` - * weight scales: fp8 ``(E, M, K//16)`` - * hidden states: uint8 (packed fp4) ``(N, K//2)`` - * hidden state scales: fp8 ``(N, K//16)`` - """ +def _nvfp4_kernel(top_k: int, routing_method: RoutingMethodType) -> Kernel: from flashinfer import fp4_quantize - block_size = 16 - parallel_cfg = FusedMoEParallelConfig.make_no_parallel() - moe_config = FusedMoEConfig( - num_experts=num_experts, - experts_per_token=top_k, - hidden_dim=hidden_size, - intermediate_size=intermediate_size, - num_local_experts=num_experts, - num_logical_experts=num_experts, - moe_parallel_config=parallel_cfg, - in_dtype=torch.bfloat16, - activation=MoEActivation.SILU, - device=device, - routing_method=RoutingMethodType.DeepSeekV3, - max_num_tokens=16, - ) + block = 16 + global_scale = torch.tensor(1.0, device=DEVICE) - gemm1 = torch.randn( - num_experts, - 2 * intermediate_size, - hidden_size, - device=device, - dtype=torch.bfloat16, - ) - gemm2 = torch.randn( - num_experts, - hidden_size, - intermediate_size, - device=device, - dtype=torch.bfloat16, - ) - # Per-tensor weight scaling factor (used to build ``g1_alphas`` / - # ``g2_alphas`` below). - w_global_scale = torch.tensor(1.0, device=device) - # Per-tensor input scaling factor. - a_global_scale = torch.tensor(1.0, device=device) + def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Packed fp4 values and their fp8 per-16 block scales, (E, M, K/16).""" + q, scale = fp4_quantize( + w, global_scale, block, sf_use_ue8m0=False, is_sf_swizzled_layout=False + ) + return q, scale.view(torch.float8_e4m3fn).reshape(*w.shape[:2], -1) - w13_q, w13_scale = fp4_quantize( - gemm1, - w_global_scale, - block_size, - sf_use_ue8m0=False, - is_sf_swizzled_layout=False, + w13, w13_scale = quantize( + torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN, device=DEVICE).bfloat16() ) - w13_scale = w13_scale.view(torch.float8_e4m3fn).reshape( - num_experts, 2 * intermediate_size, hidden_size // block_size + w2, w2_scale = quantize( + torch.randn(NUM_EXPERTS, HIDDEN, INTERMEDIATE, device=DEVICE).bfloat16() ) - w2_q, w2_scale = fp4_quantize( - gemm2, - w_global_scale, - block_size, - sf_use_ue8m0=False, - is_sf_swizzled_layout=False, - ) - w2_scale = w2_scale.view(torch.float8_e4m3fn).reshape( - num_experts, hidden_size, intermediate_size // block_size - ) - - # NVFP4 dq scale chain: g1_alphas = w1_scale_2 * a1_scale_2, - # g2_alphas = w2_scale_2 * a2_scale_2. The kernel multiplies by these. - g_alphas = torch.full((num_experts,), 1.0, device=device, dtype=torch.float32) - a2_gscale = torch.full((num_experts,), 1.0, device=device, dtype=torch.float32) - - quant_config = FusedMoEQuantConfig.make( - quant_dtype="nvfp4", - per_act_token_quant=False, - per_out_ch_quant=False, - block_shape=None, - w1_scale=w13_scale, - w2_scale=w2_scale, - g1_alphas=g_alphas, - g2_alphas=g_alphas, - a1_gscale=a_global_scale, - a2_gscale=a2_gscale, - ) - + ones = torch.ones(NUM_EXPERTS, device=DEVICE) experts = TrtLlmNvFp4ExpertsMonolithic( - moe_config=moe_config, quant_config=quant_config - ) - return experts, w13_q, w13_scale, w2_q, w2_scale, a_global_scale - - -def _run_nvfp4_monolithic( - experts: TrtLlmNvFp4ExpertsMonolithic, - hidden_states_q: torch.Tensor, - hidden_states_scale: torch.Tensor, - router_logits: torch.Tensor, - num_experts: int, - routing_bias: torch.Tensor, -) -> torch.Tensor: - """The monolithic NVFP4 apply expects packed fp4 hidden states + the - matching fp8 per-block scale stored in the ``a1q_scale`` slot.""" - # Stash the weight tensors on the experts in the locations the apply() - # implementation reads from (it pulls them from quant_config / scales - # already; w1/w2 come in as args). - return experts.apply( - hidden_states=hidden_states_q, - w1=experts._w13_packed, - w2=experts._w2_packed, - router_logits=router_logits, - activation=MoEActivation.SILU, - global_num_experts=num_experts, - expert_map=None, - a1q_scale=hidden_states_scale, - apply_router_weight_on_input=False, - num_expert_group=_DSV3_N_GROUP, - topk_group=_DSV3_TOPK_GROUP, - e_score_correction_bias=routing_bias, - routed_scaling_factor=1.0, + moe_config=_moe_config(top_k, routing_method), + quant_config=FusedMoEQuantConfig.make( + quant_dtype="nvfp4", + per_act_token_quant=False, + per_out_ch_quant=False, + block_shape=None, + w1_scale=w13_scale, + w2_scale=w2_scale, + g1_alphas=ones, + g2_alphas=ones, + a1_gscale=global_scale, + a2_gscale=ones, + ), ) + def forward(router_logits, routing_replay_out): + hidden_states = torch.randn(len(router_logits), HIDDEN, device=DEVICE) * 0.1 + # apply() reinterprets the packed uint8 activation scales itself. + hidden_q, hidden_scale = fp4_quantize( + hidden_states.bfloat16(), + global_scale, + block, + sf_use_ue8m0=False, + is_sf_swizzled_layout=False, + ) + experts.apply( + hidden_states=hidden_q, + w1=w13, + w2=w2, + router_logits=router_logits, + activation=MoEActivation.SILU, + global_num_experts=NUM_EXPERTS, + expert_map=None, + a1q_scale=hidden_scale, + apply_router_weight_on_input=False, + routing_replay_out=routing_replay_out, + **_routing_kwargs(routing_method), + ) -@pytest.mark.parametrize("num_tokens", [2, 7, 16]) + return experts, forward + + +KERNELS = {"bf16": _bf16_kernel, "fp8_block": _fp8_block_kernel, "nvfp4": _nvfp4_kernel} + + +@pytest.mark.parametrize("kernel", list(KERNELS)) +@pytest.mark.parametrize( + "routing_method", [RoutingMethodType.Renormalize, RoutingMethodType.DeepSeekV3] +) @pytest.mark.parametrize("top_k", [2, 4]) -def test_trtllm_nvfp4_monolithic_routing_replay_records_valid_experts( - num_tokens: int, - top_k: int, +@pytest.mark.parametrize("num_tokens", [2, 7, 16]) +def test_routing_replay_reports_the_routed_experts( + kernel: str, routing_method: RoutingMethodType, top_k: int, num_tokens: int ) -> None: - """End-to-end: ``TrtLlmNvFp4ExpertsMonolithic`` captures valid expert IDs - on the DSV3 routing path.""" - if top_k > _DSV3_N_GROUP * _DSV3_TOPK_GROUP: - pytest.skip( - f"DSV3 requires top_k <= n_group * topk_group " - f"({_DSV3_N_GROUP * _DSV3_TOPK_GROUP})" - ) - from flashinfer import fp4_quantize - torch.manual_seed(0) - device = torch.device("cuda:0") - - num_experts = _DSV3_NUM_EXPERTS - hidden_size = 1024 - intermediate_size = 1024 - block_size = 16 - - experts, w13_q, _w13_s, w2_q, _w2_s, a_gs = _make_nvfp4_monolithic_experts( - num_experts=num_experts, - top_k=top_k, - hidden_size=hidden_size, - intermediate_size=intermediate_size, - device=device, - ) - # The apply() reads w1/w2 from its args, but we keep them on the experts - # for convenience of the helper. - experts._w13_packed = w13_q - experts._w2_packed = w2_q - + experts, forward = KERNELS[kernel](top_k, routing_method) assert experts.supports_routing_replay_capture() - captured: list[torch.Tensor] = [] - experts.set_capture_fn(lambda r: captured.append(r.clone())) - - hidden_states = ( - torch.randn(num_tokens, hidden_size, device=device, dtype=torch.bfloat16) * 0.1 - ) - hidden_states_q, hidden_states_scale = fp4_quantize( - hidden_states, - a_gs, - block_size, - sf_use_ue8m0=False, - is_sf_swizzled_layout=False, - ) - # The vLLM apply() does the .view(fp8_e4m3fn).reshape itself, so leave - # ``hidden_states_scale`` in its native (uint8 packed) form. - router_logits = torch.rand( - num_tokens, num_experts, device=device, dtype=torch.float32 - ) - routing_bias = _make_dsv3_routing_bias(num_experts, device) - - _ = _run_nvfp4_monolithic( - experts, - hidden_states_q=hidden_states_q, - hidden_states_scale=hidden_states_scale, - router_logits=router_logits, - num_experts=num_experts, - routing_bias=routing_bias, + router_logits, planted = _planted_routing(num_tokens, top_k) + # One spare row: the kernel must write the batch's rows and nothing else. + routing_replay_out = torch.full( + (num_tokens + 1, top_k), -1, dtype=torch.int16, device=DEVICE ) - assert len(captured) == 1 - replay = captured[0] - assert replay.dtype == torch.int16 - assert replay.shape == (num_tokens, top_k) - assert (replay >= 0).all(), f"got out-of-range values: {replay}" - assert (replay < num_experts).all(), f"got out-of-range values: {replay}" - for t in range(num_tokens): - unique = replay[t].unique() - assert unique.numel() == top_k, ( - f"token {t}: expected {top_k} distinct experts, " - f"got {unique.numel()} ({replay[t].tolist()})" - ) + forward(router_logits, routing_replay_out) + + reported = routing_replay_out[:num_tokens].long().sort(dim=-1).values + assert torch.equal(reported, planted), (reported, planted) + assert (routing_replay_out[num_tokens:] == -1).all() diff --git a/tests/model_executor/test_routed_experts_capture.py b/tests/model_executor/test_routed_experts_capture.py index ff683ae44c..2d57236f8e 100644 --- a/tests/model_executor/test_routed_experts_capture.py +++ b/tests/model_executor/test_routed_experts_capture.py @@ -11,8 +11,16 @@ from vllm.config import VllmConfig from vllm.config.compilation import CompilationMode from vllm.distributed.eplb.eplb_state import EplbLayerState from vllm.model_executor.layers.fused_moe.config import RoutingMethodType +from vllm.model_executor.layers.fused_moe.experts.cpu_int4_moe import CPUExpertsInt4 +from vllm.model_executor.layers.fused_moe.modular_kernel import ( + FusedMoEKernelMonolithicImpl, +) +from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import ( + MoEPrepareAndFinalizeNoDPEPMonolithic, +) from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( RoutedExpertsCapturer, + RoutedExpertsSink, bind_routed_experts_capturer, ) from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter @@ -174,45 +182,139 @@ def test_public_binding_supports_direct_capture_source(): assert calls == [(3, topk_ids)] -def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch): - class DummyFusedMoE: - def __init__(self): - self.layer_id = 3 - self.router = _make_router() - # Use a concrete monolithic expert and override its capability - # instead of instantiating the abstract base class directly. - from vllm.model_executor.layers.fused_moe.experts.cpu_moe import ( - CPUExpertsFp8, - ) +_SINK_CONFIG = SimpleNamespace( + use_ep=False, + dp_size=1, + ep_size=1, + max_num_tokens=4, + experts_per_token=2, + device="cpu", +) - fused_experts = CPUExpertsFp8.__new__(CPUExpertsFp8) - self.routed_experts = types.SimpleNamespace( - quant_method=types.SimpleNamespace( - is_monolithic=True, - moe_kernel=types.SimpleNamespace( - impl=types.SimpleNamespace(fused_experts=fused_experts) - ), - ) - ) - self._quant_method = self.routed_experts.quant_method - self._quant_method.moe_kernel.impl.fused_experts = fused_experts - fused_experts.supports_routing_replay_capture = lambda: False - class DummyCapturer: - def capture(self, layer_id, topk_ids): - pass +class _ToyMonolithicExperts(CPUExpertsInt4): + """A monolithic kernel instance, as a weight reload rebuilds it: routes + token t to experts (t + offset, t + offset + 1).""" - dummy_module = DummyFusedMoE() + moe_config = _SINK_CONFIG + quant_config = None + + def __init__(self, offset: int = 0, supports_capture: bool = True): + self.offset = offset + self.supports_capture = supports_capture + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + def supports_routing_replay_capture(self) -> bool: + return self.supports_capture + + def apply(self, hidden_states, *args, routing_replay_out=None, **kwargs): + if routing_replay_out is not None: + t = torch.arange(hidden_states.shape[0]) + routing_replay_out[: len(t)] = torch.stack([t, t + 1], 1) + self.offset + return hidden_states + + +class _MonolithicMoE: + """The parts of a MoE layer that capture binding reads.""" + + layer_id = 5 + + def __init__(self, experts: _ToyMonolithicExperts): + self.router = _make_router() + kernel = SimpleNamespace(impl=SimpleNamespace(fused_experts=experts)) + self._quant_method = SimpleNamespace(is_monolithic=True, moe_kernel=kernel) + self.routed_experts = SimpleNamespace( + quant_method=self._quant_method, routing_sink=None + ) + + +def _bind_monolithic( + monkeypatch, experts: _ToyMonolithicExperts, capture +) -> _MonolithicMoE: import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer - monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE) + monkeypatch.setattr(fused_moe_layer, "MoERunner", _MonolithicMoE) + layer = _MonolithicMoE(experts) + bind_routed_experts_capturer( + SimpleNamespace(modules=lambda: [layer]), SimpleNamespace(capture=capture) + ) + return layer + +def test_public_binding_gives_monolithic_layers_a_sink(monkeypatch): + """A monolithic kernel routes internally, so binding gives the layer, which + outlives kernel rebuilds, a sink that captures under its layer id.""" + calls = [] + layer = _bind_monolithic( + monkeypatch, _ToyMonolithicExperts(), lambda *args: calls.append(args) + ) + ids = torch.tensor([[1, 2]]) + layer.routed_experts.routing_sink.capture_fn(ids) + assert calls == [(_MonolithicMoE.layer_id, ids)] + + +def test_public_binding_rejects_monolithic_without_replay_support(monkeypatch): with pytest.raises(ValueError, match="monolithic MoE kernel"): - bind_routed_experts_capturer( - types.SimpleNamespace(modules=lambda: [dummy_module]), DummyCapturer() + _bind_monolithic( + monkeypatch, _ToyMonolithicExperts(supports_capture=False), Mock() ) +@pytest.mark.parametrize( + ("use_ep", "dp_size", "ep_size", "rows"), + [(False, 1, 1, 4), (False, 2, 4, 8), (True, 2, 4, 16)], + ids=["single-rank", "dp", "ep"], +) +def test_routed_experts_sink_holds_a_dispatched_batch(use_ep, dp_size, ep_size, rows): + """One int16 buffer, allocated once, large enough for the batch the kernel + sees after a DP or EP gather.""" + config = SimpleNamespace( + **{ + **vars(_SINK_CONFIG), + "use_ep": use_ep, + "dp_size": dp_size, + "ep_size": ep_size, + } + ) + sink = RoutedExpertsSink(config, Mock()) + + assert sink.buffer.shape == (rows, 2) and sink.buffer.dtype == torch.int16 + + +def test_monolithic_capture_survives_kernel_rebuilds(): + """The bug in #59449: a weight reload rebuilds the kernel. Through the + production kernel path, the layer's sink captures what the current kernel + routed, before and after a rebuild, and a rebuilt kernel that cannot + capture is refused instead of leaving stale rows.""" + captured = [] + sink = RoutedExpertsSink(_SINK_CONFIG, lambda ids: captured.append(ids.tolist())) + + def forward(experts: _ToyMonolithicExperts) -> None: + kernel = FusedMoEKernelMonolithicImpl( + MoEPrepareAndFinalizeNoDPEPMonolithic(), experts + ) + kernel.apply( + torch.zeros(3, 8), + w1=None, + w2=None, + router_logits=torch.zeros(3, 16), + activation=None, + global_num_experts=16, + expert_map=None, + apply_router_weight_on_input=False, + routing_sink=sink, + ) + + for offset in (0, 7): # the kernel before and after a reload + forward(_ToyMonolithicExperts(offset)) + assert captured[-1] == [[t + offset, t + offset + 1] for t in range(3)] + with pytest.raises(ValueError, match="not supported"): + forward(_ToyMonolithicExperts(supports_capture=False)) + + def test_routed_experts_capturer_single_dp_no_metadata(): """dp_metadata is None: capture writes the full topk_ids rows.""" capturer = _capturer_with_buffer(dp_rank=0) diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py index 4776a908fb..5423a93b5f 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a16_moe.py @@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: routing_method = self.moe_config.routing_method if routing_method == RoutingMethodType.DeepseekV4: diff --git a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py index 407dc097b1..bc1a4af6c0 100644 --- a/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/aiter_mxfp4_w4a8_moe.py @@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: assert self.moe_config.intermediate_size_per_partition_unpadded is not None assert self.moe_config.hidden_dim_unpadded is not None diff --git a/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py b/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py index 5ee98a8c84..3f594dc0b3 100644 --- a/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/cpu_int4_moe.py @@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: """Apply the monolithic 4-bit INT MoE forward pass. @@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: Bias for expert scores routed_scaling_factor: Scaling factor for routing topk_group: Group size for topk + routing_replay_out: Unused; this kernel does not capture routing Returns: Output tensor after MoE computation diff --git a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py index 5bf5065858..d23edaa01a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/gpt_oss_triton_kernels_moe.py @@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: return triton_kernel_moe_forward( hidden_states=hidden_states, diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py index 54227d3cd1..782d6bddd5 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_bf16_moe.py @@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor | UnfinalizedMoEOutput: import flashinfer @@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol # dummy 0-token forward has to keep the finalized (empty) form. defer = self.moe_config.should_defer_moe_finalize(num_tokens) - routing_replay_out = self._maybe_make_routing_replay_buffer( - num_tokens=num_tokens, - device=hidden_states.device, - ) flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe( routing_logits=router_logits, routing_bias=e_score_correction_bias, @@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol num_tokens=num_tokens, top_k=self.topk, ) - self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens) return routed_output diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py index fb52445d84..532e25b745 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_fp8_moe.py @@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: import flashinfer from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout @@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit n_group = num_expert_group or 0 selected_topk_group = topk_group or 0 - routing_replay_out = self._maybe_make_routing_replay_buffer( - num_tokens=hidden_states.shape[0], - device=hidden_states.device, - ) - kwargs = dict( routing_logits=router_logits, routing_bias=e_score_correction_bias, @@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL: kwargs["activation_type"] = activation_type result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs) - self._maybe_dispatch_routing_replay( - routing_replay_out, num_tokens=hidden_states.shape[0] - ) return result def _apply_per_tensor( @@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: # Delay import for non-CUDA. import flashinfer @@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit else: assert not apply_router_weight_on_input - routing_replay_out = self._maybe_make_routing_replay_buffer( - num_tokens=hidden_states.shape[0], - device=hidden_states.device, - ) - out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe( routing_logits=router_logits, routing_bias=e_score_correction_bias, @@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config), routing_replay_out=routing_replay_out, ) - self._maybe_dispatch_routing_replay( - routing_replay_out, num_tokens=hidden_states.shape[0] - ) return out def apply( @@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: if self.quant_config.block_shape is not None: return self._apply_block_scale( @@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit e_score_correction_bias=e_score_correction_bias, routed_scaling_factor=routed_scaling_factor, topk_group=topk_group, + routing_replay_out=routing_replay_out, ) elif self.quant_config.is_per_tensor: return self._apply_per_tensor( @@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit num_expert_group=num_expert_group, e_score_correction_bias=e_score_correction_bias, routed_scaling_factor=routed_scaling_factor, + routing_replay_out=routing_replay_out, ) else: raise NotImplementedError( diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py index e1557a8f0d..6b62e8ba9c 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxfp4_moe.py @@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic( e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor | UnfinalizedMoEOutput: from flashinfer import trtllm_fp4_block_scale_moe @@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic( device=hidden_states.device, ) - routing_replay_out = self._maybe_make_routing_replay_buffer( - num_tokens=num_tokens, - device=hidden_states.device, - ) flashinfer_output = trtllm_fp4_block_scale_moe( routing_logits=router_logits, routing_bias=e_score_correction_bias, @@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic( top_k=self.topk, finalized_output=finalized_output, ) - self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens) return routed_output diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py index e3e9bc6f2e..a2f06b1be9 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_mxint4_moe.py @@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501 flashinfer_trtllm_mxint4_moe, @@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): assert self.w1_scale is not None assert self.w2_scale is not None - routing_replay_out = self._maybe_make_routing_replay_buffer( - num_tokens=hidden_states.shape[0], - device=hidden_states.device, - ) result = flashinfer_trtllm_mxint4_moe( x=hidden_states, router_logits=router_logits, @@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic): routing_method_type=self.routing_method, routing_replay_out=routing_replay_out, ) - self._maybe_dispatch_routing_replay( - routing_replay_out, num_tokens=hidden_states.shape[0] - ) return result diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py index 9dcfc6af3f..c9af86b43a 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic( e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor | UnfinalizedMoEOutput: import flashinfer @@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic( # dummy 0-token forward has to keep the finalized (empty) form. defer = self.moe_config.should_defer_moe_finalize(num_tokens) - routing_replay_out = self._maybe_make_routing_replay_buffer( - num_tokens=num_tokens, - device=hidden_states.device, - ) # Invoke kernel. # NOTE: Activation padding and output # truncation are handled by the MoE runner's @@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic( num_tokens=num_tokens, top_k=self.topk, ) - self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens) return routed_output diff --git a/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py b/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py index 0076b65966..b98fb226bb 100644 --- a/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/zentorch_moe.py @@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor: if apply_router_weight_on_input: raise NotImplementedError( diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index f563c819e4..9bde03795c 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, ) from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput +from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( + RoutedExpertsSink, +) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, SharedExpertsOrder, @@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts): def is_monolithic() -> bool: return True - routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None - _routing_replay_buffer: torch.Tensor | None = None - def supports_routing_replay_capture(self) -> bool: """Whether this expert supports routing replay capture. @@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts): """ return False - def set_capture_fn( - self, - capture_fn: Callable[[torch.Tensor], None] | None, - ) -> None: - self.routing_replay_capture_fn = capture_fn - if capture_fn is None: - self._routing_replay_buffer = None - return - # Allocate for per-rank batches gathered across the DP or EP group. - dispatch_group_size = ( - self.moe_config.ep_size - if self.moe_config.use_ep - else self.moe_config.dp_size - ) - max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size - self._routing_replay_buffer = torch.empty( - (max_num_replay_tokens, self.moe_config.experts_per_token), - dtype=torch.int16, - device=self.moe_config.device, - ) - - def _maybe_make_routing_replay_buffer( - self, - num_tokens: int, - device: torch.device, - ) -> torch.Tensor | None: - if self.routing_replay_capture_fn is None: - return None - buf = self._routing_replay_buffer - assert buf is not None - if buf.shape[0] < num_tokens or buf.device != device: - raise ValueError( - "Routing replay buffer was initialized for " - f"{buf.shape[0]} tokens on {buf.device}, but the kernel " - f"received {num_tokens} tokens on {device}." - ) - return buf - - def _maybe_dispatch_routing_replay( - self, - routing_replay_out: torch.Tensor | None, - num_tokens: int, - ) -> None: - if routing_replay_out is None or self.routing_replay_capture_fn is None: - return - self.routing_replay_capture_fn(routing_replay_out[:num_tokens]) - def apply( self, hidden_states: torch.Tensor, @@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts): e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + routing_replay_out: torch.Tensor | None = None, ) -> torch.Tensor | UnfinalizedMoEOutput: """Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed to the topk_ids and topk_weights. This is useful for kernels with fused router and fused_experts (e.g. FLASHINFER_TRTLLM). + + Kernels that ``supports_routing_replay_capture`` write each token's + routed expert ids into the leading rows of ``routing_replay_out``. """ raise NotImplementedError @@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl: e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + *, + routing_sink: RoutedExpertsSink | None, ) -> torch.Tensor | UnfinalizedMoEOutput: """Same as forward(), except uses router_logits as opposed to the topk_ids and topk_weights. This is used for kernels @@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl: defer_input_quant=self.fused_experts.expects_unquantized_inputs, ) + routing_replay_out = None + if routing_sink is not None: + # Checked per call: a weight reload may rebuild a different kernel. + if not self.fused_experts.supports_routing_replay_capture(): + raise ValueError( + "Routed-experts capture is not supported with monolithic MoE " + f"kernel {type(self.fused_experts).__name__}." + ) + routing_replay_out = routing_sink.buffer[: len(a1q)] fused_out = self.fused_experts.apply( hidden_states=a1q, w1=w1, @@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl: e_score_correction_bias=e_score_correction_bias, routed_scaling_factor=routed_scaling_factor, topk_group=topk_group, + routing_replay_out=routing_replay_out, ) + if routing_sink is not None: + routing_sink.capture_fn(routing_replay_out) if isinstance(fused_out, UnfinalizedMoEOutput): if not self.prepare_finalize.supports_deferred_moe_finalize(): @@ -1698,6 +1669,8 @@ class FusedMoEKernel: e_score_correction_bias: torch.Tensor | None = None, routed_scaling_factor: float | None = None, topk_group: int | None = None, + *, + routing_sink: RoutedExpertsSink | None, ) -> torch.Tensor | UnfinalizedMoEOutput: assert isinstance(self.impl, FusedMoEKernelMonolithicImpl) return self.impl.apply( @@ -1713,6 +1686,7 @@ class FusedMoEKernel: e_score_correction_bias=e_score_correction_bias, routed_scaling_factor=routed_scaling_factor, topk_group=topk_group, + routing_sink=routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index de08d9aa54..f3c9522df1 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput +from vllm.model_executor.layers.fused_moe.routed_experts_capturer import ( + RoutedExpertsSink, +) from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( UnquantizedFusedMoEMethod, ) @@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer): self.swiglu_alpha = swiglu_alpha self.swiglu_beta = swiglu_beta self.e_score_correction_bias = e_score_correction_bias + # Set by bind_routed_experts_capturer for monolithic kernels. + self.routing_sink: RoutedExpertsSink | None = None self.apply_router_weight_on_input = apply_router_weight_on_input # End random parameters self._loaded_expert_biases: set[str] = set() diff --git a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py index c49d614549..5d111fa635 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts_capturer.py @@ -15,6 +15,7 @@ import torch from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_tp_group from vllm.forward_context import get_forward_context +from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig from vllm.platforms import current_platform logger = logging.getLogger(__name__) @@ -186,6 +187,23 @@ class RoutedExpertsCapturer: return self.device_buffer[:num_tokens].to(self.output_dtype) +class RoutedExpertsSink: + """Layer-owned buffer and callback for the expert ids a monolithic kernel + routes to; it outlives kernel rebuilds on weight reload.""" + + def __init__( + self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None] + ) -> None: + # Sized for per-rank batches gathered across the DP or EP group. + group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size + self.buffer = torch.empty( + (moe_config.max_num_tokens * group_size, moe_config.experts_per_token), + dtype=torch.int16, + device=moe_config.device, + ) + self.capture_fn = capture_fn + + def bind_routed_experts_capturer( model: torch.nn.Module, capturer: RoutedExpertsCapturer, @@ -219,7 +237,9 @@ def bind_routed_experts_capturer( "Routed-experts capture is not supported with monolithic " f"MoE kernel {type(fused_experts).__name__}." ) - fused_experts.set_capture_fn(capture_fn) + module.routed_experts.routing_sink = RoutedExpertsSink( + fused_experts.moe_config, capture_fn + ) num_bound += 1 elif isinstance(module.router, BaseRouter): module.router.set_capture_fn(capture_fn) diff --git a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py index 17481d4a91..e9a401cf45 100644 --- a/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py +++ b/vllm/model_executor/layers/fused_moe/unquantized_fused_moe_method.py @@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) diff --git a/vllm/model_executor/layers/quantization/auto_awq.py b/vllm/model_executor/layers/quantization/auto_awq.py index 0c3e980551..e0eccc708e 100644 --- a/vllm/model_executor/layers/quantization/auto_awq.py +++ b/vllm/model_executor/layers/quantization/auto_awq.py @@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) diff --git a/vllm/model_executor/layers/quantization/auto_gptq.py b/vllm/model_executor/layers/quantization/auto_gptq.py index ed430f7445..37260ac5c8 100644 --- a/vllm/model_executor/layers/quantization/auto_gptq.py +++ b/vllm/model_executor/layers/quantization/auto_gptq.py @@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py index 51b7d0460a..a6f2879d87 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_nvfp4.py @@ -293,6 +293,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py index 0abae681f0..f3952afe0d 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_int8.py @@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py index 82f3ada446..c20b217010 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_fp8.py @@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index 1f6e87a672..75dfc5fa8f 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py index b5326e1875..90f7cb2cf3 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_mxfp8.py @@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py index 8bbaaecd64..fdd37d1203 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16.py @@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 37bb9c6840..1c1d18614e 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index ae6ab3d50f..272eb457bb 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( @@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( @@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/moe_wna16.py b/vllm/model_executor/layers/quantization/moe_wna16.py index a8b97abc86..1faf2a88e8 100644 --- a/vllm/model_executor/layers/quantization/moe_wna16.py +++ b/vllm/model_executor/layers/quantization/moe_wna16.py @@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) @staticmethod diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index f9c7f37ff3..7342b4d630 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -472,6 +472,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) @@ -900,4 +901,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) diff --git a/vllm/model_executor/layers/quantization/online/moe_base.py b/vllm/model_executor/layers/quantization/online/moe_base.py index 414c4aba5b..6e21bc9092 100644 --- a/vllm/model_executor/layers/quantization/online/moe_base.py +++ b/vllm/model_executor/layers/quantization/online/moe_base.py @@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, ) def apply( diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index f84a9b1e00..262f9bdb0a 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1980,6 +1980,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): topk_group=layer.topk_group, e_score_correction_bias=layer.e_score_correction_bias, routed_scaling_factor=layer.routed_scaling_factor, + routing_sink=layer.routing_sink, )