Fix MLA padding and grouped topk routing in the Transformers modelling backend (#49982)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
bf2b45b5d6
commit
99de48e98f
@@ -23,6 +23,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
@@ -30,9 +31,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .utils import log_replacement
|
||||
|
||||
@@ -176,17 +178,31 @@ class MoEMixin(MixtureOfExperts):
|
||||
0,
|
||||
)
|
||||
|
||||
# Unused kwargs since we use custom_routing_function:
|
||||
# - `scoring_func` and `e_score_correction_bias` only used for grouped
|
||||
# topk routing inside vLLM and are non-trivial to infer
|
||||
# and hard code `use_grouped_topk=False`
|
||||
# - `renormalize` passed anyway because it's easy to infer
|
||||
# - `num_expert_group` and `topk_group` used for inferring expert
|
||||
# placement strategy in FusedMoE
|
||||
# - `apply_router_weight_on_input` is already applied in Transformers
|
||||
# Common kwargs
|
||||
renormalize = getattr(text_config, "norm_topk_prob", top_k > 1)
|
||||
|
||||
# Routed scaling factor kwargs
|
||||
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
|
||||
# aiter applies routed_scaling_factor internally
|
||||
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
|
||||
routed_scaling_factor_kwargs = dict(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scale_to_output=apply_routed_scale_to_output,
|
||||
)
|
||||
|
||||
# Dtype the router computes in, if it is not the activation dtype.
|
||||
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
|
||||
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
|
||||
|
||||
# Grouped topk routing kwargs
|
||||
num_expert_group = getattr(text_config, "n_group", None)
|
||||
topk_group = getattr(text_config, "topk_group", None)
|
||||
use_grouped_topk = num_expert_group is not None and topk_group is not None
|
||||
grouped_topk_routing_kwargs = dict(
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
)
|
||||
|
||||
# MoE activation function
|
||||
activation = "silu"
|
||||
@@ -211,6 +227,9 @@ class MoEMixin(MixtureOfExperts):
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_redundant_experts = num_redundant_experts
|
||||
|
||||
# Down projections of shared experts consumed by FusedMoE
|
||||
shared_down_projs: list[tuple[nn.Module, str]] = []
|
||||
|
||||
# Recursively fuse MoE layers
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
@@ -249,7 +268,6 @@ class MoEMixin(MixtureOfExperts):
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=renormalize,
|
||||
use_grouped_topk=False,
|
||||
quant_config=self.quant_config,
|
||||
prefix=qual_name,
|
||||
activation=activation,
|
||||
@@ -259,17 +277,55 @@ class MoEMixin(MixtureOfExperts):
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
fuser = MoEBlockFuser.match(moe_block, experts_name)
|
||||
if self.num_expert_groups <= 1 and fuser is not None:
|
||||
# _maybe_apply_routed_scale_to_output edge case. Transformers
|
||||
# decoder layers do not compensate for dividing by scaling factor.
|
||||
reaches_fp16_trick = (
|
||||
routed_scaling_factor != 1.0
|
||||
and apply_routed_scale_to_output
|
||||
and self.model_config.dtype == torch.float16
|
||||
and fuser is not None
|
||||
and fuser.shared_name is not None
|
||||
)
|
||||
if reaches_fp16_trick:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert "
|
||||
"output, which only fp16 overflow protection expects the "
|
||||
"decoder layer to compensate for. Falling back to routing "
|
||||
"in Transformers; run in bfloat16 to fuse it.",
|
||||
moe_block_cls,
|
||||
)
|
||||
if fuser is not None and not reaches_fp16_trick:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in FusedMoE.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
hf_shared = shared_experts.shared_experts
|
||||
glu_fuser = get_fuser(hf_shared)
|
||||
down_name = getattr(glu_fuser, "down_name", None)
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=fuser.scoring_func,
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
gate=fuser.gate(moe_block, prefix),
|
||||
shared_experts=fuser.shared_experts(moe_block, prefix),
|
||||
gate=gate,
|
||||
shared_experts=shared_experts,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
fuser.rewrite_forward(moe_block)
|
||||
routed = "gate + experts"
|
||||
if fuser.shared_name:
|
||||
@@ -333,3 +389,7 @@ class MoEMixin(MixtureOfExperts):
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
# Continue with the replacement of layers in Base
|
||||
super().recursive_replace()
|
||||
# GLUFuser likely fused shared_experts. The down projection after the GLU
|
||||
# normally immediately reduces but we want FusedMoE to handle the reduction.
|
||||
for hf_shared, down_name in shared_down_projs:
|
||||
hf_shared.get_submodule(down_name).reduce_results = False
|
||||
|
||||
Reference in New Issue
Block a user