[MoE Refactor] Move the shared/fused expert output sum into MoERunnerBase (#35949)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -716,7 +716,7 @@ class MarlinExperts(MarlinExpertsBase):
|
||||
):
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
return fused_marlin_moe(
|
||||
fused_marlin_moe(
|
||||
hidden_states=hidden_states,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
|
||||
@@ -230,11 +230,18 @@ class FusedMoE(PluggableLayer):
|
||||
hidden_size: Input hidden state size of the transformer
|
||||
intermediate_size: Intermediate size of the experts
|
||||
params_dtype: Data type for the parameters.
|
||||
reduce_results: Whether to all_reduce on the output of the layer
|
||||
renormalize: Whether to renormalize the logits in the fused_moe kernel
|
||||
quant_config: Quantization configure.
|
||||
enable_eplb: Whether to enable expert parallelism load balancer.
|
||||
router_logits_dtype: Data type for router logits buffers.
|
||||
routed_scaling_factor: A scaling factor that is applied to the topk_weights
|
||||
by the router or the output of the layer depending
|
||||
on the value of `apply_routed_scale_to_output`
|
||||
apply_routed_scale_to_output: Determine whether or not `routed_scaling_factor`
|
||||
is applied to the topk_weights or to the experts
|
||||
output. It is applied to the experts output
|
||||
instead of the topk_weights when this feature is
|
||||
not supported by the router (or the experts).
|
||||
"""
|
||||
|
||||
# --8<-- [end:fused_moe]
|
||||
@@ -246,7 +253,6 @@ class FusedMoE(PluggableLayer):
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
params_dtype: torch.dtype | None = None,
|
||||
reduce_results: bool = False,
|
||||
renormalize: bool = True,
|
||||
use_grouped_topk: bool = False,
|
||||
num_expert_group: int | None = None,
|
||||
@@ -274,12 +280,12 @@ class FusedMoE(PluggableLayer):
|
||||
gate: torch.nn.Module | None = None,
|
||||
shared_experts: torch.nn.Module | None = None,
|
||||
routed_input_transform: torch.nn.Module | None = None,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
apply_routed_scale_to_output: bool = False,
|
||||
zero_expert_type: str | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self._routed_input_transform = routed_input_transform
|
||||
|
||||
if params_dtype is None:
|
||||
params_dtype = torch.get_default_dtype()
|
||||
self.params_dtype = params_dtype
|
||||
@@ -425,7 +431,6 @@ class FusedMoE(PluggableLayer):
|
||||
|
||||
assert intermediate_size % self.tp_size == 0
|
||||
intermediate_size_per_partition = intermediate_size // self.tp_size
|
||||
self.reduce_results = reduce_results
|
||||
self.renormalize = renormalize
|
||||
|
||||
# TODO(bnell): these attributes are only used by monolithic kernels.
|
||||
@@ -437,7 +442,14 @@ class FusedMoE(PluggableLayer):
|
||||
self.topk_group = topk_group
|
||||
self.custom_routing_function = custom_routing_function
|
||||
self.scoring_func = scoring_func
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
# When apply_routed_scale_to_output is True, we set the scaling factor
|
||||
# to 1.0 so it ends up being a nop. Applying the scale will be handled
|
||||
# by the runner in this case.
|
||||
# The member variable must be set in the same way as the router since
|
||||
# some quantization methods can access it.
|
||||
self.routed_scaling_factor = (
|
||||
routed_scaling_factor if not apply_routed_scale_to_output else 1.0
|
||||
)
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
# TODO(bnell): end attributes
|
||||
|
||||
@@ -456,7 +468,7 @@ class FusedMoE(PluggableLayer):
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=custom_routing_function,
|
||||
scoring_func=scoring_func,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routed_scaling_factor=self.routed_scaling_factor,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
num_fused_shared_experts=self.num_fused_shared_experts,
|
||||
enable_eplb=enable_eplb,
|
||||
@@ -578,12 +590,18 @@ class FusedMoE(PluggableLayer):
|
||||
layer_name=self.layer_name,
|
||||
moe_config=self.moe_config,
|
||||
router=self.router,
|
||||
routed_input_transform=self._routed_input_transform,
|
||||
gate=gate,
|
||||
shared_experts=shared_experts,
|
||||
quant_method=self.quant_method,
|
||||
reduce_results=self.reduce_results,
|
||||
enable_dbo=self.vllm_config.parallel_config.enable_dbo,
|
||||
routed_input_transform=routed_input_transform,
|
||||
routed_output_transform=routed_output_transform,
|
||||
# When apply_routed_scale_to_output is True, we allow
|
||||
# the scaling factor to be passed to the runner, otherwise
|
||||
# we pass 1.0 so it ends up being a nop.
|
||||
routed_scaling_factor=routed_scaling_factor
|
||||
if apply_routed_scale_to_output
|
||||
else 1.0,
|
||||
)
|
||||
|
||||
# TODO(bnell): This method is provided as a hook so vllm/lora/layers/fused_moe.py
|
||||
@@ -1514,32 +1532,11 @@ class FusedMoE(PluggableLayer):
|
||||
self.ensure_moe_quant_config_init()
|
||||
return self.quant_method.moe_quant_config
|
||||
|
||||
def must_reduce_shared_expert_outputs(self) -> bool:
|
||||
"""
|
||||
The shared_experts are typically computed using the RowParallelLinear
|
||||
layer. The result of this function is typically used as
|
||||
the reduce_results argument to the module.
|
||||
When just tensor-parallel is used, it is not required to reduce
|
||||
the shared_experts results immediately. Instead we reduce at the
|
||||
once at the end of the MoE op. (Refer to DeepSeekV2MoE module)
|
||||
With EP and all2all kernels - this is no longer viable as all
|
||||
GPU ranks in DP, produce the complete set of hidden_states.
|
||||
Therefore it is required that we reduce the shared_experts output
|
||||
early.
|
||||
"""
|
||||
return self.runner.must_reduce_shared_expert_outputs()
|
||||
|
||||
def maybe_all_reduce_tensor_model_parallel(self, final_hidden_states: torch.Tensor):
|
||||
"""
|
||||
Some combine kernels reduce across GPU ranks by default.
|
||||
"""
|
||||
return self.runner.maybe_all_reduce_tensor_model_parallel(final_hidden_states)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
return self.runner.forward(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
@@ -1613,7 +1610,6 @@ class FusedMoE(PluggableLayer):
|
||||
f"intermediate_size_per_partition={self.intermediate_size_per_partition}, " # noqa: E501
|
||||
f"tp_size={self.tp_size},\n"
|
||||
f"ep_size={self.ep_size}, "
|
||||
f"reduce_results={self.reduce_results}, "
|
||||
)
|
||||
|
||||
return s
|
||||
|
||||
@@ -1261,7 +1261,7 @@ class FusedMoEKernelModularImpl:
|
||||
topk_ids: torch.Tensor,
|
||||
apply_router_weight_on_input: bool,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
The _finalize method is a wrapper around self.prepare_finalize.finalize
|
||||
that handles DBO, async and shared expert overlap.
|
||||
|
||||
@@ -37,10 +37,6 @@ class DefaultMoERunner(MoERunnerBase):
|
||||
for different configurations (e.g., with/without shared experts, gates, etc.).
|
||||
"""
|
||||
|
||||
@property
|
||||
def reduce_results(self) -> bool:
|
||||
return self._reduce_results
|
||||
|
||||
@property
|
||||
def do_naive_dispatch_combine(self) -> bool:
|
||||
return (
|
||||
|
||||
@@ -26,18 +26,7 @@ class MoERunner(ABC):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def must_reduce_shared_expert_outputs(self) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def maybe_all_reduce_tensor_model_parallel(
|
||||
self,
|
||||
final_hidden_states: torch.Tensor,
|
||||
):
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
|
||||
@@ -81,7 +81,9 @@ def _resolve_layer_name(layer_name: str | LayerName) -> str:
|
||||
|
||||
# Note: _moe_forward and _moe_forward_shared should not contain any
|
||||
# implementation details, They should merely pass along control to
|
||||
# the runner's 'forward_dispatch' method.
|
||||
# the runner's '_forward_dispatch' method.
|
||||
# These functions should never be called directly since they do not
|
||||
# include all the functionality of the MoE layer.
|
||||
def _moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
@@ -89,7 +91,7 @@ def _moe_forward(
|
||||
layer_name: _layer_name_type,
|
||||
) -> torch.Tensor:
|
||||
layer = get_layer_from_name(_resolve_layer_name(layer_name))
|
||||
return layer.runner.forward_dispatch(
|
||||
return layer.runner._forward_dispatch(
|
||||
layer,
|
||||
hidden_states,
|
||||
router_logits,
|
||||
@@ -113,7 +115,7 @@ def _moe_forward_shared(
|
||||
layer_name: _layer_name_type,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
layer = get_layer_from_name(_resolve_layer_name(layer_name))
|
||||
return layer.runner.forward_dispatch(
|
||||
return layer.runner._forward_dispatch(
|
||||
layer,
|
||||
hidden_states,
|
||||
router_logits,
|
||||
@@ -143,7 +145,7 @@ def _moe_forward_shared_fake(
|
||||
direct_register_custom_op(
|
||||
op_name="moe_forward",
|
||||
op_func=_moe_forward,
|
||||
mutates_args=["hidden_states"], # is this still true?
|
||||
mutates_args=["hidden_states"],
|
||||
fake_impl=_moe_forward_fake,
|
||||
tags=(torch.Tag.needs_fixed_stride_order,),
|
||||
)
|
||||
@@ -157,6 +159,15 @@ direct_register_custom_op(
|
||||
)
|
||||
|
||||
|
||||
def _unpack(
|
||||
result: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
if isinstance(result, tuple):
|
||||
return result
|
||||
else:
|
||||
return (None, result)
|
||||
|
||||
|
||||
class MoERunnerBase(MoERunner):
|
||||
"""
|
||||
Abstract base class providing common functionality for MoE runner implementations.
|
||||
@@ -174,7 +185,6 @@ class MoERunnerBase(MoERunner):
|
||||
allowing flexibility in the actual MoE computation implementation.
|
||||
|
||||
Key abstract methods that subclasses must implement:
|
||||
- reduce_results: Determines whether results should be reduced across ranks
|
||||
- _forward_impl: The core MoE computation logic specific to each runner type
|
||||
"""
|
||||
|
||||
@@ -187,17 +197,23 @@ class MoERunnerBase(MoERunner):
|
||||
gate: torch.nn.Module | None,
|
||||
shared_experts: torch.nn.Module | None,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
reduce_results: bool,
|
||||
enable_dbo: bool,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
):
|
||||
super().__init__()
|
||||
self.moe_config = moe_config
|
||||
self.router = router
|
||||
self.routed_input_transform = routed_input_transform
|
||||
self.routed_output_transform = routed_output_transform
|
||||
self.routed_scaling_factor = routed_scaling_factor
|
||||
self.gate = gate
|
||||
self.quant_method = quant_method
|
||||
self._reduce_results = reduce_results
|
||||
self.enable_dbo = enable_dbo
|
||||
self._fused_output_is_reduced = (
|
||||
self.quant_method.moe_kernel is not None
|
||||
and self.quant_method.moe_kernel.output_is_reduced()
|
||||
)
|
||||
|
||||
self._shared_experts: SharedExperts | None = None
|
||||
if shared_experts is not None:
|
||||
@@ -209,7 +225,6 @@ class MoERunnerBase(MoERunner):
|
||||
# called, i.e. by a MK or by the MoERunner.
|
||||
# Once the MK can be created upfront, we can just pass in the proper
|
||||
# flags derived from the quant_method's MK.
|
||||
reduce_results=reduce_results,
|
||||
quant_method=quant_method,
|
||||
enable_dbo=enable_dbo,
|
||||
)
|
||||
@@ -217,7 +232,7 @@ class MoERunnerBase(MoERunner):
|
||||
# Needed for string -> FusedMoE layer lookup in custom ops.
|
||||
self.layer_name = layer_name
|
||||
|
||||
self.forward_entry = self._select_forward()
|
||||
self._forward_entry = self._select_forward()
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu() or current_platform.is_cpu():
|
||||
@@ -245,38 +260,6 @@ class MoERunnerBase(MoERunner):
|
||||
def is_internal_router(self) -> bool:
|
||||
return self.gate is not None
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def reduce_results(self) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def must_reduce_shared_expert_outputs(self) -> bool:
|
||||
"""
|
||||
The shared_experts are typically computed using the RowParallelLinear
|
||||
layer. The result of this function is typically used as
|
||||
the reduce_results argument to the module.
|
||||
When just tensor-parallel is used, it is not required to reduce
|
||||
the shared_experts results immediately. Instead we reduce at the
|
||||
once at the end of the MoE op. (Refer to DeepSeekV2MoE module)
|
||||
With EP and all2all kernels - this is no longer viable as all
|
||||
GPU ranks in DP, produce the complete set of hidden_states.
|
||||
Therefore it is required that we reduce the shared_experts output
|
||||
early.
|
||||
"""
|
||||
return (
|
||||
self.quant_method.moe_kernel is not None
|
||||
and self.quant_method.moe_kernel.output_is_reduced()
|
||||
)
|
||||
|
||||
def maybe_all_reduce_tensor_model_parallel(self, final_hidden_states: torch.Tensor):
|
||||
"""
|
||||
Some combine kernels reduce across GPU ranks by default.
|
||||
"""
|
||||
if self.must_reduce_shared_expert_outputs():
|
||||
return final_hidden_states
|
||||
else:
|
||||
return tensor_model_parallel_all_reduce(final_hidden_states)
|
||||
|
||||
def apply_routed_input_transform(
|
||||
self, hidden_states: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
@@ -286,10 +269,6 @@ class MoERunnerBase(MoERunner):
|
||||
is saved separately so shared experts get [S, hidden_size] while
|
||||
routed experts get the transformed [S, moe_latent_size].
|
||||
|
||||
TODO: For latent MoE bandwidth optimization, fc2_latent_proj could be
|
||||
moved inside SharedFusedMoE to all-reduce on the smaller latent
|
||||
dimension.
|
||||
|
||||
Returns (possibly transformed) hidden states and the input for shared
|
||||
experts (or None if there are no shared experts).
|
||||
"""
|
||||
@@ -306,33 +285,79 @@ class MoERunnerBase(MoERunner):
|
||||
hidden_states if self._shared_experts is not None else None,
|
||||
)
|
||||
|
||||
def _maybe_reduce_output(
|
||||
def apply_routed_output_transform(
|
||||
self,
|
||||
states: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
trunc_sizes: list[int],
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
def trunc(x: torch.Tensor, trunc_size: int) -> torch.Tensor:
|
||||
return x[..., :trunc_size]
|
||||
fused_output: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Apply transform to routed expert output (e.g., latent to full dim).
|
||||
|
||||
def reduce_and_trunc(x: torch.Tensor, trunc_size: int) -> torch.Tensor:
|
||||
return trunc(self.maybe_all_reduce_tensor_model_parallel(x), trunc_size)
|
||||
Used by latent MoE models (e.g., NemotronH) where routed experts
|
||||
operate in a compressed latent space and need projection back to
|
||||
the full hidden dimension before combining with shared expert output.
|
||||
"""
|
||||
if self.routed_output_transform is not None:
|
||||
r = self.routed_output_transform(fused_output)
|
||||
fused_output = r[0] if isinstance(r, tuple) else r
|
||||
return fused_output
|
||||
|
||||
def _maybe_apply_routed_scale_to_output(
|
||||
self,
|
||||
shared_output: torch.Tensor | None,
|
||||
fused_output: torch.Tensor,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Apply routed_scaling_factor to the output with FP16 overflow
|
||||
protection.
|
||||
|
||||
Scale the fused expert output by routed_scaling_factor. For FP16,
|
||||
avoid overflow by dividing shared_output by the scale instead
|
||||
(the decoder layer compensates with matching divisions).
|
||||
"""
|
||||
if self.routed_scaling_factor != 1.0:
|
||||
if fused_output.dtype != torch.float16:
|
||||
fused_output *= self.routed_scaling_factor
|
||||
elif shared_output is not None:
|
||||
shared_output *= 1.0 / self.routed_scaling_factor
|
||||
return shared_output, fused_output
|
||||
|
||||
def _maybe_reduce_shared_expert_output(
|
||||
self,
|
||||
shared_output: torch.Tensor | None,
|
||||
) -> torch.Tensor | None:
|
||||
"""All-reduce shared expert output when the combine kernel already
|
||||
reduced fused output.
|
||||
|
||||
This is the "early" all-reduce path. When the combine kernel produces
|
||||
already-reduced fused output, shared output must be reduced separately
|
||||
to match.
|
||||
"""
|
||||
if self._fused_output_is_reduced:
|
||||
assert shared_output is not None
|
||||
shared_output = tensor_model_parallel_all_reduce(shared_output)
|
||||
return shared_output
|
||||
|
||||
def _maybe_reduce_final_output(
|
||||
self,
|
||||
states: torch.Tensor,
|
||||
trunc_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Truncate padded dimensions and all-reduce the combined output.
|
||||
|
||||
This is the "late" all-reduce path. When neither fused nor shared
|
||||
output was individually reduced, the combined sum is all-reduced
|
||||
here. Skipped when sequence-parallel is active (SP handles its
|
||||
own reduction) or when the early path already reduced both outputs.
|
||||
"""
|
||||
# We don't need to reduce the final output if:
|
||||
# - We are not running with TP or DP
|
||||
# - The MK already reduced the fused output itself.
|
||||
if (
|
||||
not self.moe_config.is_sequence_parallel
|
||||
and self.reduce_results
|
||||
and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1)
|
||||
and not self._fused_output_is_reduced
|
||||
):
|
||||
func = reduce_and_trunc
|
||||
else:
|
||||
func = trunc
|
||||
states = tensor_model_parallel_all_reduce(states)
|
||||
|
||||
if isinstance(states, tuple):
|
||||
return tuple(
|
||||
[func(s, trunc_size) for s, trunc_size in zip(states, trunc_sizes)]
|
||||
)
|
||||
else:
|
||||
assert len(trunc_sizes) == 1
|
||||
return func(states, trunc_sizes[0])
|
||||
return states[..., :trunc_size]
|
||||
|
||||
def _encode_layer_name(self) -> str | LayerName:
|
||||
if _USE_LAYERNAME:
|
||||
@@ -349,7 +374,15 @@ class MoERunnerBase(MoERunner):
|
||||
self,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, list[int]]:
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Pad hidden_states to moe_config.hidden_dim and compute the
|
||||
original dimension for later truncation.
|
||||
|
||||
For latent MoE, the routed hidden_states may be smaller than
|
||||
hidden_dim. Padding ensures uniform tensor sizes through the
|
||||
fused MoE kernel. The returned trunc_size is used by
|
||||
_maybe_reduce_final_output to strip the padding from the result.
|
||||
"""
|
||||
shared_experts_hidden_dim = (
|
||||
shared_experts_input.shape[-1] if shared_experts_input is not None else 0
|
||||
)
|
||||
@@ -365,10 +398,10 @@ class MoERunnerBase(MoERunner):
|
||||
value=0.0,
|
||||
)
|
||||
|
||||
if self._shared_experts is not None:
|
||||
orig_hidden_dims = [shared_experts_hidden_dim, transformed_hidden_dim]
|
||||
if self.routed_output_transform is not None and shared_experts_hidden_dim > 0:
|
||||
orig_hidden_dims = shared_experts_hidden_dim
|
||||
else:
|
||||
orig_hidden_dims = [transformed_hidden_dim]
|
||||
orig_hidden_dims = transformed_hidden_dim
|
||||
|
||||
return hidden_states, orig_hidden_dims
|
||||
|
||||
@@ -388,6 +421,12 @@ class MoERunnerBase(MoERunner):
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor]:
|
||||
"""Run expert routing and the fused MoE kernel via the quant method.
|
||||
|
||||
Orchestrates shared expert execution (before/after), expert selection
|
||||
via the router, and the actual fused MoE computation. Returns
|
||||
(shared_expert_output, fused_expert_output).
|
||||
"""
|
||||
# Run this before quant_method to avoid inplace issues.
|
||||
# TODO(bnell): probably not needed anymore since inplace is
|
||||
# disabled when shared experts are present.
|
||||
@@ -428,6 +467,13 @@ class MoERunnerBase(MoERunner):
|
||||
)
|
||||
|
||||
def _sequence_parallel_context(self):
|
||||
"""Return a context manager for sequence-parallel token
|
||||
redistribution.
|
||||
|
||||
When sequence parallelism is active, returns a context that handles
|
||||
local size tracking for proper token scatter/gather. Otherwise
|
||||
returns a no-op context.
|
||||
"""
|
||||
ctx = get_forward_context()
|
||||
return (
|
||||
ctx.dp_metadata.sp_local_sizes(self.moe_config.sp_size)
|
||||
@@ -448,22 +494,25 @@ class MoERunnerBase(MoERunner):
|
||||
|
||||
def _maybe_add_zero_expert_output(
|
||||
self,
|
||||
result: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
result: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Add the zero expert's contribution to the final result.
|
||||
|
||||
When a ZeroExpertRouter is used, it computes a bias-like output
|
||||
from the "zero expert" that is added to the combined routed+shared
|
||||
expert output.
|
||||
"""
|
||||
if isinstance(self.router, ZeroExpertRouter):
|
||||
zero_expert_output = self.router.zero_expert_output
|
||||
assert zero_expert_output is not None
|
||||
if isinstance(result, tuple):
|
||||
result = (result[0], result[1] + zero_expert_output)
|
||||
else:
|
||||
result = result + zero_expert_output
|
||||
result = result + zero_expert_output
|
||||
return result
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
) -> torch.Tensor:
|
||||
"""Invoke the fused moe layer.
|
||||
|
||||
Input:
|
||||
@@ -472,55 +521,88 @@ class MoERunnerBase(MoERunner):
|
||||
|
||||
Output:
|
||||
- The new hidden_states.
|
||||
or
|
||||
- A tuple of (shared experts output, new hidden_states).
|
||||
|
||||
Calling sequence
|
||||
- forward
|
||||
- self.forward_entry (_moe_forward or _moe_forward_shared custom op)
|
||||
- forward_dispatch
|
||||
- self._forward_entry (_moe_forward or _moe_forward_shared custom op)
|
||||
- _forward_dispatch
|
||||
- _forward_impl
|
||||
|
||||
Note: The existence of _moe_forward and _moe_forward_shared custom ops are due
|
||||
to the following reasons:
|
||||
1. the chunking loop in ChunkingMoERunner._forward_impl cannot be compiled by
|
||||
torch.compile
|
||||
2. pytorch cannot handle union types in custom op signatures so _moe_forward
|
||||
and _moe_forward_shared must be split.
|
||||
2. pytorch cannot handle union types in custom op signatures so
|
||||
_moe_forward and _moe_forward_shared must be split.
|
||||
|
||||
If ChunkingMoERunner._forward_impl can be implemented via torch.scan we can
|
||||
potentially get rid of _moe_forward and _moe_forward_shared and collapse the
|
||||
whole sequence into the 'forward' method.
|
||||
"""
|
||||
|
||||
# Apply transform for routed experts (e.g., latent projection for latent MoE)
|
||||
# Apply transform for routed experts (e.g., latent projection
|
||||
# for latent MoE)
|
||||
hidden_states, shared_experts_input = self.apply_routed_input_transform(
|
||||
hidden_states
|
||||
)
|
||||
|
||||
hidden_states, og_hidden_dims = self._maybe_pad_hidden_states(
|
||||
hidden_states, og_hidden_dim = self._maybe_pad_hidden_states(
|
||||
shared_experts_input,
|
||||
hidden_states,
|
||||
)
|
||||
|
||||
fused_output = self.forward_entry(
|
||||
result = self._forward_entry(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
shared_experts_input,
|
||||
self._encode_layer_name(),
|
||||
)
|
||||
|
||||
result = self._maybe_reduce_output(fused_output, og_hidden_dims)
|
||||
#
|
||||
# Note: there are two all-reduce points below. They are mutually
|
||||
# exclusive, controlled by _fused_output_is_reduced
|
||||
# - When True: the combine kernel already reduced fused_output,
|
||||
# so we reduce shared_output here to match, then skip the
|
||||
# all-reduce in _maybe_reduce_final_output.
|
||||
# - When False: neither output is reduced yet, so we combine
|
||||
# them first and all-reduce the sum in _maybe_reduce_final_output.
|
||||
|
||||
# Extract outputs from result
|
||||
shared_output, fused_output = _unpack(result)
|
||||
|
||||
# If combine kernel already reduced fused, reduce shared to match.
|
||||
# See note above re: the two all-reduce points.
|
||||
shared_output = self._maybe_reduce_shared_expert_output(shared_output)
|
||||
|
||||
shared_output, fused_output = self._maybe_apply_routed_scale_to_output(
|
||||
shared_output, fused_output
|
||||
)
|
||||
|
||||
# Apply output transform (e.g. latent -> full dim)
|
||||
fused_output = self.apply_routed_output_transform(fused_output)
|
||||
|
||||
if shared_output is not None:
|
||||
result = shared_output + fused_output
|
||||
else:
|
||||
result = fused_output
|
||||
|
||||
result = self._maybe_reduce_final_output(result, og_hidden_dim)
|
||||
|
||||
return self._maybe_add_zero_expert_output(result)
|
||||
|
||||
def forward_dispatch(
|
||||
def _forward_dispatch(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Entry point called by the custom op to run the MoE computation.
|
||||
|
||||
Handles pre-dispatch setup (gate application, external shared expert
|
||||
triggering, quant config init) then delegates to _forward_impl within
|
||||
the sequence-parallel context.
|
||||
"""
|
||||
# TODO(bnell): this can be removed after MK migration is complete.
|
||||
layer.ensure_moe_quant_config_init()
|
||||
|
||||
@@ -549,4 +631,11 @@ class MoERunnerBase(MoERunner):
|
||||
router_logits: torch.Tensor,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Core MoE computation to be implemented by subclasses.
|
||||
|
||||
Performs expert routing, fused MoE kernel execution, and shared
|
||||
expert computation. Returns a single tensor (fused output only)
|
||||
or a tuple of (shared_output, fused_output) when shared experts
|
||||
are present.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -29,8 +29,9 @@ def create_moe_runner(
|
||||
gate: torch.nn.Module | None,
|
||||
shared_experts: SharedExperts | None,
|
||||
quant_method: FusedMoEMethodBase,
|
||||
reduce_results: bool,
|
||||
enable_dbo: bool,
|
||||
routed_output_transform: torch.nn.Module | None = None,
|
||||
routed_scaling_factor: float = 1.0,
|
||||
) -> MoERunner:
|
||||
return DefaultMoERunner(
|
||||
layer_name,
|
||||
@@ -40,6 +41,7 @@ def create_moe_runner(
|
||||
gate,
|
||||
shared_experts,
|
||||
quant_method,
|
||||
reduce_results,
|
||||
enable_dbo,
|
||||
routed_output_transform=routed_output_transform,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
)
|
||||
|
||||
@@ -5,10 +5,6 @@ from enum import IntEnum
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -48,7 +44,6 @@ class SharedExperts:
|
||||
layer: torch.nn.Module,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_method: QuantizeMethodBase,
|
||||
reduce_results: bool,
|
||||
enable_dbo: bool,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
@@ -68,7 +63,6 @@ class SharedExperts:
|
||||
self._layer = layer
|
||||
self._moe_config = moe_config
|
||||
self._quant_method = quant_method
|
||||
self._reduce_results = reduce_results
|
||||
|
||||
# Allow disabling of the separate shared experts stream for
|
||||
# debug purposes.
|
||||
@@ -139,18 +133,6 @@ class SharedExperts:
|
||||
|
||||
return output
|
||||
|
||||
def _maybe_reduce_shared_out(self, shared_out: torch.Tensor) -> torch.Tensor:
|
||||
# Reduce shared expert outputs if necessary, since the MLP
|
||||
# should have been created with reduce_results=False.
|
||||
if (
|
||||
self._reduce_results
|
||||
and self._quant_method.moe_kernel is not None
|
||||
and self._quant_method.moe_kernel.output_is_reduced()
|
||||
and get_tensor_model_parallel_world_size() > 1
|
||||
):
|
||||
shared_out = tensor_model_parallel_all_reduce(shared_out)
|
||||
return shared_out
|
||||
|
||||
@property
|
||||
def _output_idx(self) -> int:
|
||||
return dbo_current_ubatch_id() if self.enable_dbo else 0
|
||||
|
||||
@@ -18,12 +18,8 @@ class SharedFusedMoE(FusedMoE):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
result = super().forward(
|
||||
) -> torch.Tensor:
|
||||
return super().forward(
|
||||
hidden_states=hidden_states,
|
||||
router_logits=router_logits,
|
||||
)
|
||||
if self.shared_experts is None:
|
||||
return None, result
|
||||
else:
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user