[K3 Perf] Fuse MXFP4 top-k finalization into latent-tail, ~5% E2E latency reduction (#53152)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wentao Ye
2026-08-21 13:29:59 -04:00
committed by GitHub
co-authored by Yongye Zhu OpenAI Codex
parent fe76112ff2
commit 7a2fdbaac4
17 changed files with 720 additions and 139 deletions
+21 -3
View File
@@ -1321,6 +1321,8 @@ class FusedMoEConfig:
# consumer; read through `use_deferred_moe_finalize`, which applies the
# guards. Kernels without the capability ignore it. Default False.
defer_moe_finalize: bool = False
# Optional consumer capacity for deferred finalize. Negative means unbounded.
defer_moe_finalize_max_num_tokens: int = -1
# SwiGLU clamp limit. When set, backends that do not implement the clamp
# are filtered out by `FusedMoEExperts.is_supported_config` so the oracle
@@ -1447,9 +1449,25 @@ class FusedMoEConfig:
``defer_moe_finalize`` is set after construction, like
``skip_final_all_reduce``.
"""
# Without TP there is no all-reduce for the top-k reduction to fuse
# into, so the deferred form buys nothing.
return self.defer_moe_finalize and self.tp_size > 1
# The consumer fuses a TP all-reduce. Other parallel modes require a
# combine or reduce-scatter after the experts and cannot defer it.
return (
self.defer_moe_finalize
and self.tp_size > 1
and self.dp_size == 1
and self.ep_size == 1
and self.pcp_size == 1
and not self.is_sequence_parallel
)
def should_defer_moe_finalize(self, num_tokens: int) -> bool:
"""Return whether this invocation may defer the top-k reduction."""
max_num_tokens = self.defer_moe_finalize_max_num_tokens
return (
self.use_deferred_moe_finalize
and num_tokens > 0
and (max_num_tokens < 0 or num_tokens <= max_num_tokens)
)
@property
def use_deepep_ht_kernels(self):
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.moe_output import (
UnfinalizedMoEOutput,
convert_flashinfer_moe_output,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
@@ -278,13 +279,13 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
num_tokens = hidden_states.shape[0]
# The runner divides by the token count on the host, so an idle rank's
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.use_deferred_moe_finalize and num_tokens > 0
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
out = flashinfer.fused_moe.trtllm_bf16_moe(
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=hidden_states,
@@ -304,17 +305,11 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
routing_replay_out=routing_replay_out,
do_finalize=not defer,
)
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=not defer,
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
if defer:
# flashinfer returns a flat permute map; the protocol wants
# [num_tokens, top_k] so consumers can read top_k from its shape.
return UnfinalizedMoEOutput(
gemm2_permuted=out[0],
expert_weights=out[1],
expanded_idx_to_permuted_idx=out[2]
.to(torch.int32)
.view(num_tokens, self.topk),
)
# do_finalize=True yields the finalized states (a bare tensor on some
# FlashInfer versions, a single-element list on others).
return out[0] if isinstance(out, (list, tuple)) else out
return routed_output
@@ -11,6 +11,10 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import (
UnfinalizedMoEOutput,
convert_flashinfer_moe_output,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
@@ -215,7 +219,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
from flashinfer import trtllm_fp4_block_scale_moe
if a1q_scale is not None:
@@ -225,18 +229,22 @@ class TrtLlmMxfp4ExpertsMonolithic(
assert hidden_states.dtype == torch.bfloat16
x_quant = hidden_states
x_scale = None
output = torch.empty(
*hidden_states.shape[:-1],
self.hidden_dim_unpadded,
dtype=torch.bfloat16,
device=hidden_states.device,
)
num_tokens = hidden_states.shape[0]
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
finalized_output = None
if not defer:
finalized_output = torch.empty(
*hidden_states.shape[:-1],
self.hidden_dim_unpadded,
dtype=torch.bfloat16,
device=hidden_states.device,
)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
num_tokens=num_tokens,
device=hidden_states.device,
)
trtllm_fp4_block_scale_moe(
flashinfer_output = trtllm_fp4_block_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=x_quant,
@@ -262,16 +270,21 @@ class TrtLlmMxfp4ExpertsMonolithic(
local_num_experts=self.local_num_experts,
routed_scaling_factor=routed_scaling_factor,
routing_method_type=self.routing_method_type,
do_finalize=True,
do_finalize=not defer,
activation_type=self._flashinfer_activation_type(activation),
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
output=output,
output=finalized_output,
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=not defer,
num_tokens=num_tokens,
top_k=self.topk,
finalized_output=finalized_output,
)
return output
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModular):
@@ -15,6 +15,7 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.moe_output import (
UnfinalizedMoEOutput,
convert_flashinfer_moe_output,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
@@ -551,7 +552,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
num_tokens = hidden_states.shape[0]
# The runner divides by the token count on the host, so an idle rank's
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.use_deferred_moe_finalize and num_tokens > 0
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
@@ -560,7 +561,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
result = flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
flashinfer_output = flashinfer.fused_moe.trtllm_fp4_block_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
hidden_states=hidden_states,
@@ -594,15 +595,11 @@ class TrtLlmNvFp4ExpertsMonolithic(
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=not defer,
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
if defer:
# flashinfer returns a flat permute map; the protocol wants
# [num_tokens, top_k] so consumers can read top_k from its shape.
return UnfinalizedMoEOutput(
gemm2_permuted=result[0],
expert_weights=result[1],
expanded_idx_to_permuted_idx=result[2]
.to(torch.int32)
.view(num_tokens, self.topk),
)
return result[0]
return routed_output
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
@@ -161,7 +162,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
shared_experts_input: Input for shared experts (if any)
Returns:
Output tensor from routed experts
Output tensor from routed experts.
"""
raise NotImplementedError
@@ -171,7 +172,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Apply the MoE operation using monolithic kernels.
@@ -181,6 +182,6 @@ class FusedMoEMethodBase(QuantizeMethodBase):
router_logits: Router logits (routing done internally)
Returns:
Output tensor from routed experts
Finalized routed states or a deferred-finalize output.
"""
raise NotImplementedError
@@ -22,6 +22,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
SharedExpertsOrder,
@@ -425,6 +426,9 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
described above for the monolithic case.
"""
def supports_deferred_moe_finalize(self) -> bool:
return False
@abstractmethod
def prepare(
self,
@@ -1078,7 +1082,7 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Same as apply(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is useful for kernels
@@ -1548,7 +1552,7 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
@@ -1579,9 +1583,14 @@ class FusedMoEKernelMonolithicImpl:
topk_group=topk_group,
)
output = self.prepare_finalize.finalize(fused_out)
return output
if isinstance(fused_out, UnfinalizedMoEOutput):
if not self.prepare_finalize.supports_deferred_moe_finalize():
raise RuntimeError(
f"{type(self.prepare_finalize).__name__} cannot pass through "
"a deferred MoE output."
)
return fused_out
return self.prepare_finalize.finalize(fused_out)
@final
@@ -1664,6 +1673,12 @@ class FusedMoEKernel:
"""
return self.prepare_finalize.output_is_reduced()
def supports_deferred_moe_finalize(self) -> bool:
return (
isinstance(self.prepare_finalize, FusedMoEPrepareAndFinalizeMonolithic)
and self.prepare_finalize.supports_deferred_moe_finalize()
)
def apply_monolithic(
self,
hidden_states: torch.Tensor,
@@ -1679,7 +1694,7 @@ class FusedMoEKernel:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
hidden_states=hidden_states,
@@ -33,6 +33,96 @@ class UnfinalizedMoEOutput:
expanded_idx_to_permuted_idx: torch.Tensor
def convert_flashinfer_moe_output(
flashinfer_output: torch.Tensor | list[torch.Tensor],
*,
do_finalize: bool,
num_tokens: int,
top_k: int,
finalized_output: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Normalize the two FlashInfer TRTLLM MoE return layouts.
Args:
flashinfer_output: Tensor returned by the FlashInfer BF16 wrapper's
legacy finalized path, or its mode-dependent tensor list.
do_finalize: Whether FlashInfer ran its top-k finalize step.
num_tokens: Number of input tokens.
top_k: Number of routed experts per token.
finalized_output: Optional destination passed to FlashInfer's ``output``
argument.
Returns:
A finalized tensor or the structured deferred-finalize output.
Raises:
ValueError: If FlashInfer returns an unexpected layout.
"""
if num_tokens < 0:
raise ValueError("num_tokens must be non-negative.")
if top_k <= 0:
raise ValueError("top_k must be positive.")
if do_finalize:
if isinstance(flashinfer_output, torch.Tensor):
returned_output = flashinfer_output
elif len(flashinfer_output) == 1 and isinstance(
flashinfer_output[0], torch.Tensor
):
returned_output = flashinfer_output[0]
else:
raise ValueError(
"Finalized FlashInfer MoE output must contain exactly one tensor."
)
if finalized_output is not None and (
returned_output.shape != finalized_output.shape
or returned_output.dtype != finalized_output.dtype
or returned_output.device != finalized_output.device
or returned_output.data_ptr() != finalized_output.data_ptr()
):
raise ValueError(
"Finalized FlashInfer MoE output must alias the provided destination."
)
return returned_output
if finalized_output is not None:
raise ValueError("Deferred FlashInfer MoE output cannot have a destination.")
if isinstance(flashinfer_output, torch.Tensor) or len(flashinfer_output) != 3:
raise ValueError(
"Deferred FlashInfer MoE output must contain GEMM2 output, routing "
"weights, and a permutation map."
)
gemm2_permuted, expert_weights, expanded_idx = flashinfer_output
if gemm2_permuted.ndim != 2:
raise ValueError("Deferred FlashInfer GEMM2 output must be rank-2.")
if expanded_idx.dtype != torch.int32:
raise ValueError("Deferred FlashInfer permutation map must use int32.")
expected_routes = num_tokens * top_k
if expert_weights.numel() != expected_routes:
raise ValueError(
"Deferred FlashInfer routing weights must contain "
f"num_tokens * top_k ({expected_routes}) elements."
)
if expanded_idx.numel() != expected_routes:
raise ValueError(
"Deferred FlashInfer permutation map must contain "
f"num_tokens * top_k ({expected_routes}) elements."
)
if not (gemm2_permuted.device == expert_weights.device == expanded_idx.device):
raise ValueError("Deferred FlashInfer outputs must share a device.")
if not all(
output.is_contiguous()
for output in (gemm2_permuted, expert_weights, expanded_idx)
):
raise ValueError("Deferred FlashInfer outputs must be contiguous.")
return UnfinalizedMoEOutput(
gemm2_permuted=gemm2_permuted,
expert_weights=expert_weights.view(num_tokens, top_k),
expanded_idx_to_permuted_idx=expanded_idx.view(num_tokens, top_k),
)
@dataclass
class MoEOutput:
"""A MoE layer's output with its final reduction still open.
@@ -98,6 +98,9 @@ class MoEPrepareAndFinalizeNoDPEPModular(mk.FusedMoEPrepareAndFinalizeModular):
class MoEPrepareAndFinalizeNoDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMonolithic):
def supports_deferred_moe_finalize(self) -> bool:
return True
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.expert_map_manager import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -1216,7 +1217,7 @@ class RoutedExperts(PluggableLayer):
shared_experts_input: Input for shared experts (if any)
Returns:
Output tensor from routed experts
Output tensor from routed experts.
"""
assert not self.quant_method.is_monolithic
@@ -1235,7 +1236,7 @@ class RoutedExperts(PluggableLayer):
x: torch.Tensor,
router_logits: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""
Execute routed experts using the quantization method's apply function.
@@ -1250,7 +1251,7 @@ class RoutedExperts(PluggableLayer):
input_ids: input ids for DeepSeek V4
Returns:
Output tensor from routed experts
Finalized routed states or a deferred-finalize output.
"""
assert self.quant_method.is_monolithic
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Iterable
from contextlib import nullcontext
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, cast
import torch
import torch.nn.functional as F
@@ -28,6 +28,7 @@ from vllm.model_executor.layers.fused_moe.config import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts import (
RoutedExperts,
)
@@ -121,11 +122,14 @@ def _moe_forward(
hidden_dim_unpadded: int,
) -> torch.Tensor:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
input_ids,
return cast(
torch.Tensor,
layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
input_ids,
),
)
@@ -155,11 +159,14 @@ def _moe_forward_shared(
hidden_dim_unpadded: int,
) -> tuple[torch.Tensor, torch.Tensor]:
layer = get_layer_from_name(_resolve_layer_name(layer_name))
return layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
input_ids,
return cast(
tuple[torch.Tensor, torch.Tensor],
layer._forward_impl(
hidden_states,
router_logits,
shared_experts_input,
input_ids,
),
)
@@ -207,8 +214,10 @@ direct_register_custom_op(
def _unpack(
result: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
) -> tuple[torch.Tensor | None, torch.Tensor]:
result: torch.Tensor
| UnfinalizedMoEOutput
| tuple[torch.Tensor, torch.Tensor | UnfinalizedMoEOutput],
) -> tuple[torch.Tensor | None, torch.Tensor | UnfinalizedMoEOutput]:
if isinstance(result, tuple):
return result
else:
@@ -573,7 +582,7 @@ class MoERunner(MoERunnerInterface):
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
input_ids: torch.Tensor | None = None,
) -> tuple[torch.Tensor | None, torch.Tensor]:
) -> tuple[torch.Tensor | None, torch.Tensor | UnfinalizedMoEOutput]:
"""Run expert routing and the fused MoE kernel via the quant method.
Orchestrates shared expert execution (before/after), expert selection
@@ -733,6 +742,7 @@ class MoERunner(MoERunnerInterface):
# Extract outputs from result
shared_output, fused_output = _unpack(result)
fused_output = cast(torch.Tensor, fused_output)
if og_hidden_dim_pre_xform is not None:
fused_output = fused_output[..., :og_hidden_dim_pre_xform]
@@ -808,18 +818,25 @@ class MoERunner(MoERunnerInterface):
def _maybe_combine(
self,
shared_output: torch.Tensor | None,
hidden_states: torch.Tensor,
) -> torch.Tensor | tuple[torch.Tensor | None, torch.Tensor]:
hidden_states: torch.Tensor | UnfinalizedMoEOutput,
) -> (
torch.Tensor
| UnfinalizedMoEOutput
| tuple[torch.Tensor | None, torch.Tensor | UnfinalizedMoEOutput]
):
if self.do_naive_dispatch_combine:
hidden_states = get_ep_group().combine(
hidden_states, self.moe_config.is_sequence_parallel
cast(torch.Tensor, hidden_states),
self.moe_config.is_sequence_parallel,
)
if (
self.moe_config.pcp_size > 1
and not self.moe_config.moe_parallel_config.use_all2all_kernels
):
hidden_states = get_pcp_group().reduce_scatter(hidden_states, dim=0)
hidden_states = get_pcp_group().reduce_scatter(
cast(torch.Tensor, hidden_states), dim=0
)
if self.shared_experts is not None:
assert shared_output is not None
@@ -833,7 +850,11 @@ class MoERunner(MoERunnerInterface):
router_logits: torch.Tensor,
shared_experts_input: torch.Tensor | None,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
) -> (
torch.Tensor
| UnfinalizedMoEOutput
| tuple[torch.Tensor, torch.Tensor | UnfinalizedMoEOutput]
):
"""Entry point called by the custom op to run the MoE computation.
Handles pre-dispatch setup (gate application, external shared expert
@@ -844,7 +865,8 @@ class MoERunner(MoERunnerInterface):
- fused MoE kernel execution
- shared expert computation.
Returns a single tensor of combined fused and shared output (if present).
Returns routed output, optionally paired with shared-expert output. A
fused consumer may request the routed output in deferred-finalize form.
"""
# TODO(bnell): this can be removed after MK migration is complete.
self.routed_experts._ensure_moe_quant_config_init()
@@ -17,6 +17,7 @@ from vllm.model_executor.layers.fused_moe import modular_kernel as mk
from vllm.model_executor.layers.fused_moe.config import (
mxfp4_w4a16_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
@@ -451,7 +452,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(
@@ -865,7 +866,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
x: torch.Tensor,
router_logits: torch.Tensor,
input_ids: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply_monolithic(