[MoE] Defer the TRTLLM-Gen top-k finalize on the modular path (#58635)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
24928a4d62
commit
378504a544
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from enum import IntEnum
|
||||
from typing import Union
|
||||
|
||||
@@ -1290,13 +1290,12 @@ class FusedMoEConfig:
|
||||
# Only honored on the non-reduced (late-AR) TP path. Default False.
|
||||
skip_final_all_reduce: bool = False
|
||||
|
||||
# When True, experts that can stop after GEMM2 are allowed to hand back an
|
||||
# UnfinalizedMoEOutput instead of finalized states, leaving the top-k
|
||||
# reduction to fuse into the consumer. Set by layers that have such a
|
||||
# consumer; read through `use_deferred_moe_finalize`, which applies the
|
||||
# guards. Kernels without the capability ignore it. Default False.
|
||||
defer_moe_finalize: bool = False
|
||||
# Optional consumer capacity for deferred finalize. Negative means unbounded.
|
||||
# Requested through `defer_moe_finalize()`, read through
|
||||
# `should_defer_moe_finalize()`.
|
||||
_defer_moe_finalize: bool = field(default=False, init=False)
|
||||
# Most tokens a deferred call covers: the consumer's capacity, lowered by
|
||||
# experts that would split a larger call across kernel launches. Negative
|
||||
# means unbounded.
|
||||
defer_moe_finalize_max_num_tokens: int = -1
|
||||
|
||||
# SwiGLU clamp limit. When set, backends that do not implement the clamp
|
||||
@@ -1422,21 +1421,57 @@ class FusedMoEConfig:
|
||||
def use_deferred_moe_finalize(self) -> bool:
|
||||
"""Whether experts may return an unfinalized output on this deployment.
|
||||
|
||||
Evaluated on read rather than in ``__post_init__`` because
|
||||
``defer_moe_finalize`` is set after construction, like
|
||||
``skip_final_all_reduce``.
|
||||
Evaluated on read rather than in ``__post_init__`` because deferral is
|
||||
requested after construction, like ``skip_final_all_reduce``.
|
||||
"""
|
||||
# The consumer fuses a TP all-reduce. Other parallel modes require a
|
||||
# combine or reduce-scatter after the experts and cannot defer it.
|
||||
# combine or reduce-scatter after the experts and cannot defer it, and
|
||||
# the consumer has no way to strip hidden-dim padding from GEMM2 rows.
|
||||
return (
|
||||
self.defer_moe_finalize
|
||||
self._defer_moe_finalize
|
||||
and self.tp_size > 1
|
||||
and self.dp_size == 1
|
||||
and self.ep_size == 1
|
||||
and self.pcp_size == 1
|
||||
and not self.is_sequence_parallel
|
||||
and self.hidden_dim == self.hidden_dim_unpadded
|
||||
)
|
||||
|
||||
def defer_moe_finalize(self, max_num_tokens: int = -1) -> None:
|
||||
"""Ask the experts to leave the top-k reduction to the layer's consumer.
|
||||
|
||||
A layer whose consumer can fuse the top-k reduction (e.g. into its TP
|
||||
all-reduce) calls this once while the model is built, and only when its
|
||||
experts are TRTLLM-Gen ones that can stop after GEMM2 (the
|
||||
``do_finalize=False`` path). Other experts ignore the request and
|
||||
always finalize, which ``should_defer_moe_finalize`` can't see, so the
|
||||
layer checks the quant method's ``experts_cls`` first, as Kimi-K3 does.
|
||||
From then on:
|
||||
|
||||
- The experts return an ``UnfinalizedMoEOutput`` instead of finalized
|
||||
states for every call ``should_defer_moe_finalize`` accepts.
|
||||
- ``should_defer_moe_finalize(num_tokens)`` is the answer for a call:
|
||||
it requires a TP-only deployment without hidden-dim padding
|
||||
(``use_deferred_moe_finalize``), a non-empty call and at most
|
||||
``defer_moe_finalize_max_num_tokens`` tokens. The cap starts at
|
||||
``max_num_tokens`` and only ever goes down: experts that would split a
|
||||
larger call across kernel launches lower it to their single-launch
|
||||
size when they are built, since each launch permutes into its own
|
||||
buffer.
|
||||
- The model asks ``should_defer_moe_finalize`` before each call and takes
|
||||
the matching path. A deferred call runs the runner's ``_forward_impl``
|
||||
directly, since the MoE custom op returns tensors only, and the
|
||||
consumer then owns the top-k reduction, the shared-expert add and the
|
||||
all-reduce.
|
||||
|
||||
Args:
|
||||
max_num_tokens: Most tokens per call the consumer can take in
|
||||
deferred form. Negative means no limit of its own.
|
||||
|
||||
"""
|
||||
self._defer_moe_finalize = True
|
||||
self.limit_deferred_moe_finalize(max_num_tokens)
|
||||
|
||||
def should_defer_moe_finalize(self, num_tokens: int) -> bool:
|
||||
"""Return whether this invocation may defer the top-k reduction."""
|
||||
max_num_tokens = self.defer_moe_finalize_max_num_tokens
|
||||
@@ -1446,6 +1481,15 @@ class FusedMoEConfig:
|
||||
and (max_num_tokens < 0 or num_tokens <= max_num_tokens)
|
||||
)
|
||||
|
||||
def limit_deferred_moe_finalize(self, max_num_tokens: int) -> None:
|
||||
"""Finalize calls above ``max_num_tokens`` even when deferring.
|
||||
|
||||
Negative means no limit, and leaves the current one in place.
|
||||
"""
|
||||
current = self.defer_moe_finalize_max_num_tokens
|
||||
if max_num_tokens >= 0 and (current < 0 or max_num_tokens < current):
|
||||
self.defer_moe_finalize_max_num_tokens = max_num_tokens
|
||||
|
||||
@property
|
||||
def use_deepep_ht_kernels(self):
|
||||
return self.moe_parallel_config.use_deepep_ht_kernels
|
||||
|
||||
@@ -12,6 +12,10 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEQuantConfig,
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import (
|
||||
UnfinalizedMoEOutput,
|
||||
convert_flashinfer_moe_output,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
@@ -273,7 +277,7 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
import flashinfer
|
||||
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
|
||||
|
||||
@@ -297,7 +301,9 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
weight_layout = WeightLayout.BlockMajorK
|
||||
hidden_states_scale = prepare_deepseek_fp8_x_sf(hidden_states, a1q_scale)
|
||||
|
||||
flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe(
|
||||
num_tokens = hidden_states.shape[0]
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
flashinfer_output = flashinfer.fused_moe.trtllm_fp8_block_scale_routed_moe(
|
||||
topk_ids=(topk_ids, topk_weights),
|
||||
routing_bias=None,
|
||||
hidden_states=hidden_states,
|
||||
@@ -321,9 +327,20 @@ class TrtLlmFp8ExpertsModular(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
use_shuffled_weight=use_shuffled_weight,
|
||||
weight_layout=weight_layout,
|
||||
fp8_quantization_type=fp8_quant_type,
|
||||
output=output,
|
||||
do_finalize=not defer,
|
||||
output=None if defer else output,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
if not defer:
|
||||
return None
|
||||
routed_output = convert_flashinfer_moe_output(
|
||||
flashinfer_output,
|
||||
do_finalize=False,
|
||||
num_tokens=num_tokens,
|
||||
top_k=topk_ids.size(1),
|
||||
)
|
||||
assert isinstance(routed_output, UnfinalizedMoEOutput)
|
||||
return routed_output
|
||||
|
||||
|
||||
class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolithic):
|
||||
|
||||
@@ -289,6 +289,18 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
Moved from trtllm_moe.py.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(moe_config, quant_config, **kwargs)
|
||||
# Each launch permutes into its own buffer, so a chunked call finalizes.
|
||||
moe_config.limit_deferred_moe_finalize(
|
||||
self._max_supported_tokens(self.topk, moe_config.num_experts)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
@@ -343,7 +355,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
def _invoke_kernel(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
output: torch.Tensor | None,
|
||||
x_quant: torch.Tensor,
|
||||
x_scale: torch.Tensor | None,
|
||||
topk_ids: torch.Tensor,
|
||||
@@ -355,10 +367,11 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
local_num_experts: int,
|
||||
local_expert_offset: int,
|
||||
topk: int,
|
||||
) -> None:
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
"""Finalize into ``output``, or stop after GEMM2 when it is None."""
|
||||
from flashinfer import trtllm_fp4_block_scale_routed_moe
|
||||
|
||||
trtllm_fp4_block_scale_routed_moe(
|
||||
flashinfer_output = trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=(topk_ids, topk_weights),
|
||||
routing_bias=None,
|
||||
hidden_states=x_quant,
|
||||
@@ -386,12 +399,22 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
# Modular kernel receives pre-routed tokens, so routing is already
|
||||
# done. Use Renormalize as a safe default the TRTLLM kernel supports.
|
||||
routing_method_type=RoutingMethodType.Renormalize,
|
||||
do_finalize=True,
|
||||
do_finalize=output is not None,
|
||||
enable_pdl=True,
|
||||
activation_type=self._flashinfer_activation_type(activation),
|
||||
output=output,
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
)
|
||||
if output is not None:
|
||||
return None
|
||||
routed_output = convert_flashinfer_moe_output(
|
||||
flashinfer_output,
|
||||
do_finalize=False,
|
||||
num_tokens=x_quant.shape[0],
|
||||
top_k=topk,
|
||||
)
|
||||
assert isinstance(routed_output, UnfinalizedMoEOutput)
|
||||
return routed_output
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -410,7 +433,7 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
topk_ids = topk_ids.to(dtype=torch.int32)
|
||||
|
||||
topk = topk_ids.size(-1)
|
||||
@@ -428,13 +451,17 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
# Chunk tokens so the batched-GEMM grid stays within CUDA limits.
|
||||
# Chunk tokens so the batched-GEMM grid stays within CUDA limits. Each
|
||||
# launch permutes into its own buffer, so only a run that fits in one
|
||||
# launch can leave the top-k reduction to a deferring consumer.
|
||||
M = x_quant.size(0)
|
||||
chunk_size = self._max_supported_tokens(topk, global_num_experts)
|
||||
defer = chunk_size >= M and self.moe_config.should_defer_moe_finalize(M)
|
||||
unfinalized: UnfinalizedMoEOutput | None = None
|
||||
for start in range(0, M, chunk_size):
|
||||
end = min(start + chunk_size, M)
|
||||
self._invoke_kernel(
|
||||
output[start:end],
|
||||
unfinalized = self._invoke_kernel(
|
||||
None if defer else output[start:end],
|
||||
x_quant[start:end],
|
||||
None if x_scale is None else x_scale[start:end],
|
||||
topk_ids[start:end],
|
||||
@@ -448,4 +475,4 @@ class TrtLlmMxfp4ExpertsModular(TrtLlmMxfp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
topk,
|
||||
)
|
||||
|
||||
return output
|
||||
return unfinalized
|
||||
|
||||
@@ -302,6 +302,16 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModular):
|
||||
"""Modular version of the implementation (just the experts)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
per_token_activation: bool = False,
|
||||
):
|
||||
super().__init__(moe_config, quant_config, per_token_activation)
|
||||
# Each launch permutes into its own buffer, so a chunked call finalizes.
|
||||
moe_config.limit_deferred_moe_finalize(self._get_chunk_size())
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
"""The modular implementation supports all parallel configs."""
|
||||
@@ -333,7 +343,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
|
||||
def _invoke_kernel(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
output: torch.Tensor | None,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
@@ -342,7 +352,8 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
):
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
"""Finalize into ``output``, or stop after GEMM2 when it is None."""
|
||||
import flashinfer
|
||||
|
||||
assert self.quant_config.w1_scale is not None
|
||||
@@ -359,7 +370,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
output1_scale_gate_scalar = self.quant_config.g1_alphas
|
||||
|
||||
# Invoke kernel.
|
||||
flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
|
||||
flashinfer_output = flashinfer.fused_moe.trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=(topk_ids, topk_weights),
|
||||
routing_bias=None,
|
||||
hidden_states=hidden_states,
|
||||
@@ -387,7 +398,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
local_num_experts=self.local_num_experts,
|
||||
routed_scaling_factor=None,
|
||||
routing_method_type=1, # not used
|
||||
do_finalize=True,
|
||||
do_finalize=output is not None,
|
||||
activation_type=activation_to_flashinfer_int(activation),
|
||||
per_token_scale=per_token_scale,
|
||||
output=output,
|
||||
@@ -395,6 +406,16 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
fi_moe_largest_bucket(self.moe_config), self._get_chunk_size()
|
||||
),
|
||||
)
|
||||
if output is not None:
|
||||
return None
|
||||
routed_output = convert_flashinfer_moe_output(
|
||||
flashinfer_output,
|
||||
do_finalize=False,
|
||||
num_tokens=hidden_states.shape[0],
|
||||
top_k=self.topk,
|
||||
)
|
||||
assert isinstance(routed_output, UnfinalizedMoEOutput)
|
||||
return routed_output
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -413,7 +434,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
assert self._supports_activation(activation)
|
||||
# Per-token defers input quant to _invoke_kernel, so a1q_scale is None.
|
||||
assert a1q_scale is not None or self.per_token_activation
|
||||
@@ -425,8 +446,12 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
chunk_size = self._get_chunk_size()
|
||||
|
||||
if chunk_size >= M:
|
||||
self._invoke_kernel(
|
||||
output,
|
||||
# Each launch permutes into its own buffer, so only a run that fits
|
||||
# in one launch can leave the top-k reduction to a deferring
|
||||
# consumer.
|
||||
defer = self.moe_config.should_defer_moe_finalize(M)
|
||||
return self._invoke_kernel(
|
||||
None if defer else output,
|
||||
hidden_states,
|
||||
w1,
|
||||
w2,
|
||||
@@ -450,6 +475,7 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula
|
||||
global_num_experts,
|
||||
None if a1q_scale is None else a1q_scale[start:end],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
class TrtLlmNvFp4ExpertsMonolithic(
|
||||
|
||||
@@ -148,7 +148,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: "SharedExperts | None",
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Apply the MoE operation using modular kernels.
|
||||
|
||||
Args:
|
||||
@@ -160,7 +160,7 @@ class FusedMoEMethodBase(QuantizeMethodBase):
|
||||
shared_experts: The shared experts module, if any
|
||||
|
||||
Returns:
|
||||
Output tensor from routed experts.
|
||||
Finalized routed states or a deferred-finalize output.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -238,6 +238,15 @@ class FusedMoEPrepareAndFinalize(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
"""Whether ``finalize`` can be skipped for a deferring consumer.
|
||||
|
||||
An implementation opts in only if everything it does in ``finalize``
|
||||
-- the top-k reduction, and any combine or reduce-scatter -- is work
|
||||
the consumer takes over.
|
||||
"""
|
||||
return False
|
||||
|
||||
|
||||
# TODO: pass FusedMoEParallelConfig in as ctor parameter?
|
||||
class FusedMoEPrepareAndFinalizeModular(FusedMoEPrepareAndFinalize):
|
||||
@@ -404,9 +413,6 @@ class FusedMoEPrepareAndFinalizeMonolithic(FusedMoEPrepareAndFinalize):
|
||||
described above for the monolithic case.
|
||||
"""
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
return False
|
||||
|
||||
@abstractmethod
|
||||
def prepare(
|
||||
self,
|
||||
@@ -895,10 +901,14 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
) -> UnfinalizedMoEOutput | None:
|
||||
"""This function computes the intermediate result of a Mixture of Experts
|
||||
(MoE) layer using two sets of weights, w1 and w2.
|
||||
|
||||
Writes into `output` and returns None, unless the implementation stopped
|
||||
after GEMM2 and left the top-k reduction to a fused consumer, in which
|
||||
case `output` is untouched and the unfinalized result is returned.
|
||||
|
||||
Args:
|
||||
output: (torch.Tensor): The unweighted, unreduced output tensor.
|
||||
hidden_states: (torch.Tensor): The (quantized) input tensor to the MoE
|
||||
@@ -1270,7 +1280,7 @@ class FusedMoEKernelModularImpl:
|
||||
apply_router_weight_on_input: bool,
|
||||
expert_tokens_meta: ExpertTokensMetadata | None,
|
||||
output_alias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
_, M_full, N, K, top_k = self.fused_experts.moe_problem_size(
|
||||
a1q, w1, w2, topk_ids
|
||||
)
|
||||
@@ -1318,7 +1328,7 @@ class FusedMoEKernelModularImpl:
|
||||
elif use_output_alias:
|
||||
fused_out = output_alias
|
||||
|
||||
self.fused_experts.apply(
|
||||
unfinalized = self.fused_experts.apply(
|
||||
output=fused_out,
|
||||
hidden_states=a1q,
|
||||
w1=w1,
|
||||
@@ -1336,6 +1346,11 @@ class FusedMoEKernelModularImpl:
|
||||
apply_router_weight_on_input=apply_router_weight_on_input,
|
||||
)
|
||||
|
||||
# Experts that stopped after GEMM2 wrote nothing into `fused_out`; the
|
||||
# top-k reduction they left open belongs to whoever takes this.
|
||||
if isinstance(unfinalized, UnfinalizedMoEOutput):
|
||||
return unfinalized
|
||||
|
||||
return fused_out
|
||||
|
||||
def _finalize(
|
||||
@@ -1426,7 +1441,7 @@ class FusedMoEKernelModularImpl:
|
||||
apply_router_weight_on_input: bool = False,
|
||||
shared_experts: SharedExperts | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""This function computes a Mixture of Experts (MoE) layer using two sets
|
||||
of weights, w1 and w2, and top-k gating mechanism.
|
||||
|
||||
@@ -1453,7 +1468,9 @@ class FusedMoEKernelModularImpl:
|
||||
hidden_states before latent projection.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: The output tensor after applying the MoE layer.
|
||||
torch.Tensor: The output tensor after applying the MoE layer, or
|
||||
the unfinalized output when the experts left the top-k reduction to
|
||||
a fused consumer.
|
||||
|
||||
"""
|
||||
output = torch.empty_like(hidden_states)
|
||||
@@ -1498,6 +1515,17 @@ class FusedMoEKernelModularImpl:
|
||||
if lora_ctx is not None:
|
||||
lora_ctx.original_hidden_states = None
|
||||
|
||||
if isinstance(fused_out, UnfinalizedMoEOutput):
|
||||
# Nothing below can run on an unfinalized output: finalize is the
|
||||
# local top-k reduction (and, for DP/EP, the combine) that the
|
||||
# consumer takes over.
|
||||
if not self.prepare_finalize.supports_deferred_moe_finalize():
|
||||
raise RuntimeError(
|
||||
f"{type(self.prepare_finalize).__name__} cannot pass through "
|
||||
"a deferred MoE output."
|
||||
)
|
||||
return fused_out
|
||||
|
||||
return self._finalize(
|
||||
output,
|
||||
fused_out,
|
||||
@@ -1653,10 +1681,7 @@ class FusedMoEKernel:
|
||||
return self.prepare_finalize.output_is_reduced()
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
return (
|
||||
isinstance(self.prepare_finalize, FusedMoEPrepareAndFinalizeMonolithic)
|
||||
and self.prepare_finalize.supports_deferred_moe_finalize()
|
||||
)
|
||||
return self.prepare_finalize.supports_deferred_moe_finalize()
|
||||
|
||||
def apply_monolithic(
|
||||
self,
|
||||
@@ -1703,7 +1728,7 @@ class FusedMoEKernel:
|
||||
apply_router_weight_on_input: bool,
|
||||
shared_experts: SharedExperts | None = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelModularImpl)
|
||||
return self.impl.apply(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -9,7 +9,7 @@ import torch
|
||||
|
||||
@dataclass
|
||||
class UnfinalizedMoEOutput:
|
||||
"""Unfinalized output of a monolithic MoE kernel.
|
||||
"""Unfinalized output of a MoE kernel.
|
||||
|
||||
Kernels that can stop after GEMM2 (the TRTLLM-Gen ``do_finalize=False`` path)
|
||||
hand back their permuted, unweighted output plus the routing weights and the
|
||||
@@ -24,10 +24,11 @@ class UnfinalizedMoEOutput:
|
||||
|
||||
# [num_permuted_rows, hidden_dim] permuted, unweighted GEMM2 output.
|
||||
gemm2_permuted: torch.Tensor
|
||||
# [num_tokens, top_k] routing weights, in the activation dtype -- consumers
|
||||
# index them as such, and a buffer typed wider than its contents reads as
|
||||
# garbage. These already carry routed_scaling_factor for routing methods that
|
||||
# fold it in.
|
||||
# [num_tokens, top_k] routing weights, in the activation dtype when the
|
||||
# kernel routed itself and in the router's dtype for modular experts --
|
||||
# consumers index them as such, and a buffer typed wider than its contents
|
||||
# reads as garbage. These already carry routed_scaling_factor for routing
|
||||
# methods that fold it in.
|
||||
expert_weights: torch.Tensor
|
||||
# [num_tokens, top_k] int32 permute map; -1 = expert not local to this rank.
|
||||
expanded_idx_to_permuted_idx: torch.Tensor
|
||||
|
||||
@@ -38,6 +38,11 @@ def _quantize_input(
|
||||
|
||||
|
||||
class MoEPrepareAndFinalizeNoDPEPModular(mk.FusedMoEPrepareAndFinalizeModular):
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
# finalize() here is the top-k weight-and-reduce and nothing else,
|
||||
# which is exactly the work a deferring consumer takes over.
|
||||
return True
|
||||
|
||||
@property
|
||||
def activation_format(self) -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@@ -1224,7 +1224,7 @@ class RoutedExperts(PluggableLayer):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: "SharedExperts | None" = None,
|
||||
shared_experts_input: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Execute routed experts using the quantization method's apply function.
|
||||
|
||||
This is called by the runner after router selection (for modular kernels)
|
||||
@@ -1239,7 +1239,7 @@ class RoutedExperts(PluggableLayer):
|
||||
shared_experts_input: Input for shared experts (if any)
|
||||
|
||||
Returns:
|
||||
Output tensor from routed experts.
|
||||
Finalized routed states or a deferred-finalize output.
|
||||
|
||||
"""
|
||||
assert not self.quant_method.is_monolithic
|
||||
|
||||
+2
-1
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
@@ -227,7 +228,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
+1
-1
@@ -303,7 +303,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
+2
-1
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
@@ -375,7 +376,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
+2
-1
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
@@ -188,7 +189,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
@@ -827,7 +828,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
@@ -228,7 +229,7 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
make_fp8_moe_kernel,
|
||||
@@ -179,7 +180,7 @@ class INCMxfp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
@@ -690,7 +690,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
@@ -1102,7 +1102,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
@@ -1471,7 +1471,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -433,7 +433,7 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
@@ -860,7 +860,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -166,7 +166,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert not self.is_monolithic
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
|
||||
@@ -976,7 +976,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
@@ -1935,7 +1935,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
@@ -2182,7 +2182,7 @@ class QuarkNvfp4MoEMethod(QuarkMoEMethod):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
x,
|
||||
|
||||
Reference in New Issue
Block a user