Make the Transformers modeling backend as fast as native vLLM (#47187)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-06 16:59:14 +01:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5ad11172b7
commit 5bce653e09
23 changed files with 2679 additions and 160 deletions
+83 -46
View File
@@ -26,9 +26,11 @@ import torch.nn as nn
from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import maybe_prefix
from vllm.utils.torch_utils import direct_register_custom_op
@@ -37,6 +39,8 @@ from .utils import log_replacement
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class TransformersMoEState:
@@ -142,14 +146,15 @@ class MoEMixin(MixtureOfExperts):
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for mlp in self.mlp_layers:
mlp.n_local_physical_experts = num_local_physical_experts
mlp.n_physical_experts = num_physical_experts
mlp.n_redundant_experts = self.num_redundant_experts
mlp.experts.update_expert_map()
for moe_block in self.mlp_layers:
moe_block.n_local_physical_experts = num_local_physical_experts
moe_block.n_physical_experts = num_physical_experts
moe_block.n_redundant_experts = self.num_redundant_experts
moe_block.experts.update_expert_map()
def recursive_replace(self):
"""Initialize the MoE layers."""
experts_name = "experts"
text_config = self.text_config
# Positional arguments
@@ -217,10 +222,13 @@ class MoEMixin(MixtureOfExperts):
# down_proj = (num_experts, intermediate_size, hidden_size)
params = list(child_module.parameters())
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
if child_name == "experts" and (is_modulelist or is_3d):
if child_name == experts_name and (is_modulelist or is_3d):
# Alias for readability
mlp = module
moe_block = module
experts = child_module
# Class of the fused block (parent of gate/experts/shared)
moe_block_cls = type(moe_block).__name__
experts_cls = type(experts).__name__
# Do the experts have biases
has_bias = False
for experts_param_name, _ in experts.named_parameters():
@@ -230,64 +238,93 @@ class MoEMixin(MixtureOfExperts):
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
for mlp_param_name, _ in mlp.named_parameters():
if "shared_expert" in mlp_param_name:
for moe_block_param_name, _ in moe_block.named_parameters():
if "shared_expert" in moe_block_param_name:
self.num_shared_experts = 1
break
# Replace experts module with FusedMoE
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
fused_experts = FusedMoE(
kwargs: dict[str, Any] = dict(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
renormalize=renormalize,
# Hard coded because topk happens in Transformers
use_grouped_topk=False,
num_expert_group=num_expert_group,
topk_group=topk_group,
quant_config=self.quant_config,
prefix=qual_name,
activation=activation,
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
custom_routing_function=partial(
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersMoERunner,
routed_experts_cls=TransformersRoutedExperts,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
fuser = MoEBlockFuser.match(moe_block, experts_name)
if self.num_expert_groups <= 1 and fuser is not None:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in FusedMoE.
kwargs |= dict(
scoring_func=fuser.scoring_func,
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
gate=fuser.gate(moe_block, prefix),
shared_experts=fuser.shared_experts(moe_block, prefix),
)
fuser.rewrite_forward(moe_block)
routed = "gate + experts"
if fuser.shared_name:
routed += " + shared experts"
logger.info_once(
"Fused: %s (%s) -> FusedMoE (internal routing)",
routed,
moe_block_cls,
)
else:
# MoE block forward is unmodified.
# gate/router and shared expert (if any) runs in Transformers.
# We then smuggle the topk_ids in using a custom op.
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
custom_routing_function, moe_state=moe_state
),
runner_cls=TransformersMoERunner,
runner_args={"moe_state": moe_state},
)
logger.info_once(
"Fused: experts (%s) -> FusedMoE (external routing)",
experts_cls,
)
fused_experts = FusedMoE(**kwargs)
moe_block.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
self.mlp_layers.append(mlp)
self.mlp_layers.append(moe_block)
self.moe_layers.append(fused_experts)
else:
_recursive_replace(child_module, prefix=qual_name)