[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-25 15:38:29 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 24928a4d62
commit 378504a544
26 changed files with 652 additions and 117 deletions
+57 -13
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from dataclasses import dataclass, field
from enum import IntEnum
from typing import Union
@@ -1290,13 +1290,12 @@ class FusedMoEConfig:
# Only honored on the non-reduced (late-AR) TP path. Default False.
skip_final_all_reduce: bool = False
# When True, experts that can stop after GEMM2 are allowed to hand back an
# UnfinalizedMoEOutput instead of finalized states, leaving the top-k
# reduction to fuse into the consumer. Set by layers that have such a
# consumer; read through `use_deferred_moe_finalize`, which applies the
# guards. Kernels without the capability ignore it. Default False.
defer_moe_finalize: bool = False
# Optional consumer capacity for deferred finalize. Negative means unbounded.
# Requested through `defer_moe_finalize()`, read through
# `should_defer_moe_finalize()`.
_defer_moe_finalize: bool = field(default=False, init=False)
# Most tokens a deferred call covers: the consumer's capacity, lowered by
# experts that would split a larger call across kernel launches. Negative
# means unbounded.
defer_moe_finalize_max_num_tokens: int = -1
# SwiGLU clamp limit. When set, backends that do not implement the clamp
@@ -1422,21 +1421,57 @@ class FusedMoEConfig:
def use_deferred_moe_finalize(self) -> bool:
"""Whether experts may return an unfinalized output on this deployment.
Evaluated on read rather than in ``__post_init__`` because
``defer_moe_finalize`` is set after construction, like
``skip_final_all_reduce``.
Evaluated on read rather than in ``__post_init__`` because deferral is
requested after construction, like ``skip_final_all_reduce``.
"""
# The consumer fuses a TP all-reduce. Other parallel modes require a
# combine or reduce-scatter after the experts and cannot defer it.
# combine or reduce-scatter after the experts and cannot defer it, and
# the consumer has no way to strip hidden-dim padding from GEMM2 rows.
return (
self.defer_moe_finalize
self._defer_moe_finalize
and self.tp_size > 1
and self.dp_size == 1
and self.ep_size == 1
and self.pcp_size == 1
and not self.is_sequence_parallel
and self.hidden_dim == self.hidden_dim_unpadded
)
def defer_moe_finalize(self, max_num_tokens: int = -1) -> None:
"""Ask the experts to leave the top-k reduction to the layer's consumer.
A layer whose consumer can fuse the top-k reduction (e.g. into its TP
all-reduce) calls this once while the model is built, and only when its
experts are TRTLLM-Gen ones that can stop after GEMM2 (the
``do_finalize=False`` path). Other experts ignore the request and
always finalize, which ``should_defer_moe_finalize`` can't see, so the
layer checks the quant method's ``experts_cls`` first, as Kimi-K3 does.
From then on:
- The experts return an ``UnfinalizedMoEOutput`` instead of finalized
states for every call ``should_defer_moe_finalize`` accepts.
- ``should_defer_moe_finalize(num_tokens)`` is the answer for a call:
it requires a TP-only deployment without hidden-dim padding
(``use_deferred_moe_finalize``), a non-empty call and at most
``defer_moe_finalize_max_num_tokens`` tokens. The cap starts at
``max_num_tokens`` and only ever goes down: experts that would split a
larger call across kernel launches lower it to their single-launch
size when they are built, since each launch permutes into its own
buffer.
- The model asks ``should_defer_moe_finalize`` before each call and takes
the matching path. A deferred call runs the runner's ``_forward_impl``
directly, since the MoE custom op returns tensors only, and the
consumer then owns the top-k reduction, the shared-expert add and the
all-reduce.
Args:
max_num_tokens: Most tokens per call the consumer can take in
deferred form. Negative means no limit of its own.
"""
self._defer_moe_finalize = True
self.limit_deferred_moe_finalize(max_num_tokens)
def should_defer_moe_finalize(self, num_tokens: int) -> bool:
"""Return whether this invocation may defer the top-k reduction."""
max_num_tokens = self.defer_moe_finalize_max_num_tokens
@@ -1446,6 +1481,15 @@ class FusedMoEConfig:
and (max_num_tokens < 0 or num_tokens <= max_num_tokens)
)
def limit_deferred_moe_finalize(self, max_num_tokens: int) -> None:
"""Finalize calls above ``max_num_tokens`` even when deferring.
Negative means no limit, and leaves the current one in place.
"""
current = self.defer_moe_finalize_max_num_tokens
if max_num_tokens >= 0 and (current < 0 or max_num_tokens < current):
self.defer_moe_finalize_max_num_tokens = max_num_tokens
@property
def use_deepep_ht_kernels(self):
return self.moe_parallel_config.use_deepep_ht_kernels
@@ -12,6 +12,10 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import (
UnfinalizedMoEOutput,
convert_flashinfer_moe_output,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
@@ -273,7 +277,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
) -> UnfinalizedMoEOutput | None:
import flashinfer
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
@@ -297,7 +301,9 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
weight_layout = WeightLayout.BlockMajorK
hidden_states_scale = prepare_deepseek_fp8_x_sf(hidden_states, a1q_scale)
flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe(
num_tokens = hidden_states.shape[0]
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
flashinfer_output = flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
hidden_states=hidden_states,
@@ -321,9 +327,20 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
use_shuffled_weight=use_shuffled_weight,
weight_layout=weight_layout,
fp8_quantization_type=fp8_quant_type,
output=output,
do_finalize=not defer,
output=None if defer else output,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
if not defer:
return None
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=False,
num_tokens=num_tokens,
top_k=topk_ids.size(1),
)
assert isinstance(routed_output, UnfinalizedMoEOutput)
return routed_output
class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolithic):
@@ -289,6 +289,18 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
Moved from trtllm_moe.py.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
**kwargs,
):
super().__init__(moe_config, quant_config, **kwargs)
# Each launch permutes into its own buffer, so a chunked call finalizes.
moe_config.limit_deferred_moe_finalize(
self._max_supported_tokens(self.topk, moe_config.num_experts)
)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
@@ -343,7 +355,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
def _invoke_kernel(
self,
output: torch.Tensor,
output: torch.Tensor | None,
x_quant: torch.Tensor,
x_scale: torch.Tensor | None,
topk_ids: torch.Tensor,
@@ -355,10 +367,11 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
local_num_experts: int,
local_expert_offset: int,
topk: int,
) -> None:
) -> UnfinalizedMoEOutput | None:
"""Finalize into ``output``, or stop after GEMM2 when it is None."""
from flashinfer import trtllm_fp4_block_scale_routed_moe
trtllm_fp4_block_scale_routed_moe(
flashinfer_output = trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
hidden_states=x_quant,
@@ -386,12 +399,22 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
# Modular kernel receives pre-routed tokens, so routing is already
# done. Use Renormalize as a safe default the TRTLLM kernel supports.
routing_method_type=RoutingMethodType.Renormalize,
do_finalize=True,
do_finalize=output is not None,
enable_pdl=True,
activation_type=self._flashinfer_activation_type(activation),
output=output,
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
)
if output is not None:
return None
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=False,
num_tokens=x_quant.shape[0],
top_k=topk,
)
assert isinstance(routed_output, UnfinalizedMoEOutput)
return routed_output
def apply(
self,
@@ -410,7 +433,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
) -> UnfinalizedMoEOutput | None:
topk_ids = topk_ids.to(dtype=torch.int32)
topk = topk_ids.size(-1)
@@ -428,13 +451,17 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
assert self.w1_scale is not None
assert self.w2_scale is not None
# Chunk tokens so the batched-GEMM grid stays within CUDA limits.
# Chunk tokens so the batched-GEMM grid stays within CUDA limits. Each
# launch permutes into its own buffer, so only a run that fits in one
# launch can leave the top-k reduction to a deferring consumer.
M = x_quant.size(0)
chunk_size = self._max_supported_tokens(topk, global_num_experts)
defer = chunk_size >= M and self.moe_config.should_defer_moe_finalize(M)
unfinalized: UnfinalizedMoEOutput | None = None
for start in range(0, M, chunk_size):
end = min(start + chunk_size, M)
self._invoke_kernel(
output[start:end],
unfinalized = self._invoke_kernel(
None if defer else output[start:end],
x_quant[start:end],
None if x_scale is None else x_scale[start:end],
topk_ids[start:end],
@@ -448,4 +475,4 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
topk,
)
return output
return unfinalized
@@ -302,6 +302,16 @@ class TrtLlmNvFp4ExpertsBase:
class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModular):
"""Modular version of the implementation (just the experts)."""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
per_token_activation: bool = False,
):
super().__init__(moe_config, quant_config, per_token_activation)
# Each launch permutes into its own buffer, so a chunked call finalizes.
moe_config.limit_deferred_moe_finalize(self._get_chunk_size())
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
"""The modular implementation supports all parallel configs."""
@@ -333,7 +343,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
def _invoke_kernel(
self,
output: torch.Tensor,
output: torch.Tensor | None,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
@@ -342,7 +352,8 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
activation: MoEActivation,
global_num_experts: int,
a1q_scale: torch.Tensor | None,
):
) -> UnfinalizedMoEOutput | None:
"""Finalize into ``output``, or stop after GEMM2 when it is None."""
import flashinfer
assert self.quant_config.w1_scale is not None
@@ -359,7 +370,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
output1_scale_gate_scalar = self.quant_config.g1_alphas
# Invoke kernel.
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
flashinfer_output = flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
hidden_states=hidden_states,
@@ -387,7 +398,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
local_num_experts=self.local_num_experts,
routed_scaling_factor=None,
routing_method_type=1, # not used
do_finalize=True,
do_finalize=output is not None,
activation_type=activation_to_flashinfer_int(activation),
per_token_scale=per_token_scale,
output=output,
@@ -395,6 +406,16 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
fi_moe_largest_bucket(self.moe_config), self._get_chunk_size()
),
)
if output is not None:
return None
routed_output = convert_flashinfer_moe_output(
flashinfer_output,
do_finalize=False,
num_tokens=hidden_states.shape[0],
top_k=self.topk,
)
assert isinstance(routed_output, UnfinalizedMoEOutput)
return routed_output
def apply(
self,
@@ -413,7 +434,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
):
) -> UnfinalizedMoEOutput | None:
assert self._supports_activation(activation)
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
assert a1q_scale is not None or self.per_token_activation
@@ -425,8 +446,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
chunk_size = self._get_chunk_size()
if chunk_size >= M:
self._invoke_kernel(
output,
# Each launch permutes into its own buffer, so only a run that fits
# in one launch can leave the top-k reduction to a deferring
# consumer.
defer = self.moe_config.should_defer_moe_finalize(M)
return self._invoke_kernel(
None if defer else output,
hidden_states,
w1,
w2,
@@ -450,6 +475,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
global_num_experts,
None if a1q_scale is None else a1q_scale[start:end],
)
return None
class TrtLlmNvFp4ExpertsMonolithic(
@@ -148,7 +148,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None",
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Apply the MoE operation using modular kernels.
Args:
@@ -160,7 +160,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
shared_experts: The shared experts module, if any
Returns:
Output tensor from routed experts.
Finalized routed states or a deferred-finalize output.
"""
raise NotImplementedError
@@ -238,6 +238,15 @@ class FusedMoEPrepareAndFinalize(ABC):
"""
return False
def supports_deferred_moe_finalize(self) -> bool:
"""Whether ``finalize`` can be skipped for a deferring consumer.
An implementation opts in only if everything it does in ``finalize``
-- the top-k reduction, and any combine or reduce-scatter -- is work
the consumer takes over.
"""
return False
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
@@ -404,9 +413,6 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
described above for the monolithic case.
"""
def supports_deferred_moe_finalize(self) -> bool:
return False
@abstractmethod
def prepare(
self,
@@ -895,10 +901,14 @@ class FusedMoEExpertsModular(FusedMoEExperts):
workspace2: torch.Tensor,
expert_tokens_meta: ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
) -> UnfinalizedMoEOutput | None:
"""This function computes the intermediate result of a Mixture of Experts
(MoE) layer using two sets of weights, w1 and w2.
Writes into `output` and returns None, unless the implementation stopped
after GEMM2 and left the top-k reduction to a fused consumer, in which
case `output` is untouched and the unfinalized result is returned.
Args:
output: (torch.Tensor): The unweighted, unreduced output tensor.
hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
@@ -1270,7 +1280,7 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input: bool,
expert_tokens_meta: ExpertTokensMetadata | None,
output_alias: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
_, M_full, N, K, top_k = self.fused_experts.moe_problem_size(
a1q, w1, w2, topk_ids
)
@@ -1318,7 +1328,7 @@ class FusedMoEKernelModularImpl:
elif use_output_alias:
fused_out = output_alias
self.fused_experts.apply(
unfinalized = self.fused_experts.apply(
output=fused_out,
hidden_states=a1q,
w1=w1,
@@ -1336,6 +1346,11 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input=apply_router_weight_on_input,
)
# Experts that stopped after GEMM2 wrote nothing into `fused_out`; the
# top-k reduction they left open belongs to whoever takes this.
if isinstance(unfinalized, UnfinalizedMoEOutput):
return unfinalized
return fused_out
def _finalize(
@@ -1426,7 +1441,7 @@ class FusedMoEKernelModularImpl:
apply_router_weight_on_input: bool = False,
shared_experts: SharedExperts | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""This function computes a Mixture of Experts (MoE) layer using two sets
of weights, w1 and w2, and top-k gating mechanism.
@@ -1453,7 +1468,9 @@ class FusedMoEKernelModularImpl:
hidden_states before latent projection.
Returns:
torch.Tensor: The output tensor after applying the MoE layer.
torch.Tensor: The output tensor after applying the MoE layer, or
the unfinalized output when the experts left the top-k reduction to
a fused consumer.
"""
output = torch.empty_like(hidden_states)
@@ -1498,6 +1515,17 @@ class FusedMoEKernelModularImpl:
if lora_ctx is not None:
lora_ctx.original_hidden_states = None
if isinstance(fused_out, UnfinalizedMoEOutput):
# Nothing below can run on an unfinalized output: finalize is the
# local top-k reduction (and, for DP/EP, the combine) that the
# consumer takes over.
if not self.prepare_finalize.supports_deferred_moe_finalize():
raise RuntimeError(
f"{type(self.prepare_finalize).__name__} cannot pass through "
"a deferred MoE output."
)
return fused_out
return self._finalize(
output,
fused_out,
@@ -1653,10 +1681,7 @@ class FusedMoEKernel:
return self.prepare_finalize.output_is_reduced()
def supports_deferred_moe_finalize(self) -> bool:
return (
isinstance(self.prepare_finalize, FusedMoEPrepareAndFinalizeMonolithic)
and self.prepare_finalize.supports_deferred_moe_finalize()
)
return self.prepare_finalize.supports_deferred_moe_finalize()
def apply_monolithic(
self,
@@ -1703,7 +1728,7 @@ class FusedMoEKernel:
apply_router_weight_on_input: bool,
shared_experts: SharedExperts | None = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelModularImpl)
return self.impl.apply(
hidden_states=hidden_states,
@@ -9,7 +9,7 @@ import torch
@dataclass
class UnfinalizedMoEOutput:
"""Unfinalized output of a monolithic MoE kernel.
"""Unfinalized output of a MoE kernel.
Kernels that can stop after GEMM2 (the TRTLLM-Gen ``do_finalize=False`` path)
hand back their permuted, unweighted output plus the routing weights and the
@@ -24,10 +24,11 @@ class UnfinalizedMoEOutput:
# [num_permuted_rows, hidden_dim] permuted, unweighted GEMM2 output.
gemm2_permuted: torch.Tensor
# [num_tokens, top_k] routing weights, in the activation dtype -- consumers
# index them as such, and a buffer typed wider than its contents reads as
# garbage. These already carry routed_scaling_factor for routing methods that
# fold it in.
# [num_tokens, top_k] routing weights, in the activation dtype when the
# kernel routed itself and in the router's dtype for modular experts --
# consumers index them as such, and a buffer typed wider than its contents
# reads as garbage. These already carry routed_scaling_factor for routing
# methods that fold it in.
expert_weights: torch.Tensor
# [num_tokens, top_k] int32 permute map; -1 = expert not local to this rank.
expanded_idx_to_permuted_idx: torch.Tensor
@@ -38,6 +38,11 @@ def _quantize_input(
class MoEPrepareAndFinalizeNoDPEPModular(mk.FusedMoEPrepareAndFinalizeModular):
def supports_deferred_moe_finalize(self) -> bool:
# finalize() here is the top-k weight-and-reduce and nothing else,
# which is exactly the work a deferring consumer takes over.
return True
@property
def activation_format(self) -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@@ -1224,7 +1224,7 @@ class RoutedExperts(PluggableLayer):
topk_ids: torch.Tensor,
shared_experts: "SharedExperts | None" = None,
shared_experts_input: torch.Tensor | None = None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Execute routed experts using the quantization method's apply function.
This is called by the runner after router selection (for modular kernels)
@@ -1239,7 +1239,7 @@ class RoutedExperts(PluggableLayer):
shared_experts_input: Input for shared experts (if any)
Returns:
Output tensor from routed experts.
Finalized routed states or a deferred-finalize output.
"""
assert not self.quant_method.is_monolithic
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
Mxfp4MoeBackend,
@@ -227,7 +228,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -303,7 +303,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -375,7 +376,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -188,7 +189,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe import (
UnquantizedFusedMoEMethod,
)
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -827,7 +828,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
Mxfp4MoeBackend,
@@ -228,7 +229,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe import (
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
convert_to_fp8_moe_kernel_format,
make_fp8_moe_kernel,
@@ -179,7 +180,7 @@ class INCMxfp8MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,
@@ -690,7 +690,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -1102,7 +1102,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -1471,7 +1471,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -433,7 +433,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -860,7 +860,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -166,7 +166,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert not self.is_monolithic
assert self.moe_kernel is not None
return self.moe_kernel.apply(
@@ -976,7 +976,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
@@ -1935,7 +1935,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
hidden_states=x,
@@ -2182,7 +2182,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
topk_ids: torch.Tensor,
shared_experts: SharedExperts | None,
shared_experts_input: torch.Tensor | None,
) -> torch.Tensor:
) -> torch.Tensor | UnfinalizedMoEOutput:
assert self.moe_kernel is not None
return self.moe_kernel.apply(
x,