Make the Transformers modeling backend as fast as native vLLM (#47187)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5ad11172b7
commit
5bce653e09
@@ -26,9 +26,11 @@ import torch.nn as nn
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
@@ -37,6 +39,8 @@ from .utils import log_replacement
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformersMoEState:
|
||||
@@ -142,14 +146,15 @@ class MoEMixin(MixtureOfExperts):
|
||||
self.num_physical_experts = num_physical_experts
|
||||
self.num_local_physical_experts = num_local_physical_experts
|
||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||
for mlp in self.mlp_layers:
|
||||
mlp.n_local_physical_experts = num_local_physical_experts
|
||||
mlp.n_physical_experts = num_physical_experts
|
||||
mlp.n_redundant_experts = self.num_redundant_experts
|
||||
mlp.experts.update_expert_map()
|
||||
for moe_block in self.mlp_layers:
|
||||
moe_block.n_local_physical_experts = num_local_physical_experts
|
||||
moe_block.n_physical_experts = num_physical_experts
|
||||
moe_block.n_redundant_experts = self.num_redundant_experts
|
||||
moe_block.experts.update_expert_map()
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Initialize the MoE layers."""
|
||||
experts_name = "experts"
|
||||
text_config = self.text_config
|
||||
|
||||
# Positional arguments
|
||||
@@ -217,10 +222,13 @@ class MoEMixin(MixtureOfExperts):
|
||||
# down_proj = (num_experts, intermediate_size, hidden_size)
|
||||
params = list(child_module.parameters())
|
||||
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
|
||||
if child_name == "experts" and (is_modulelist or is_3d):
|
||||
if child_name == experts_name and (is_modulelist or is_3d):
|
||||
# Alias for readability
|
||||
mlp = module
|
||||
moe_block = module
|
||||
experts = child_module
|
||||
# Class of the fused block (parent of gate/experts/shared)
|
||||
moe_block_cls = type(moe_block).__name__
|
||||
experts_cls = type(experts).__name__
|
||||
# Do the experts have biases
|
||||
has_bias = False
|
||||
for experts_param_name, _ in experts.named_parameters():
|
||||
@@ -230,64 +238,93 @@ class MoEMixin(MixtureOfExperts):
|
||||
# If the config does not specify num_shared_experts, but
|
||||
# the model has shared experts, we assume there is one.
|
||||
if self.num_shared_experts == 0:
|
||||
for mlp_param_name, _ in mlp.named_parameters():
|
||||
if "shared_expert" in mlp_param_name:
|
||||
for moe_block_param_name, _ in moe_block.named_parameters():
|
||||
if "shared_expert" in moe_block_param_name:
|
||||
self.num_shared_experts = 1
|
||||
break
|
||||
|
||||
# Replace experts module with FusedMoE
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
def custom_routing_function(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
dist_group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
fused_experts = FusedMoE(
|
||||
kwargs: dict[str, Any] = dict(
|
||||
num_experts=num_experts,
|
||||
top_k=top_k,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=renormalize,
|
||||
# Hard coded because topk happens in Transformers
|
||||
use_grouped_topk=False,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
quant_config=self.quant_config,
|
||||
prefix=qual_name,
|
||||
activation=activation,
|
||||
enable_eplb=enable_eplb,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function,
|
||||
moe_state=moe_state,
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
mlp.experts = fused_experts
|
||||
fuser = MoEBlockFuser.match(moe_block, experts_name)
|
||||
if self.num_expert_groups <= 1 and fuser is not None:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in FusedMoE.
|
||||
kwargs |= dict(
|
||||
scoring_func=fuser.scoring_func,
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
gate=fuser.gate(moe_block, prefix),
|
||||
shared_experts=fuser.shared_experts(moe_block, prefix),
|
||||
)
|
||||
fuser.rewrite_forward(moe_block)
|
||||
routed = "gate + experts"
|
||||
if fuser.shared_name:
|
||||
routed += " + shared experts"
|
||||
logger.info_once(
|
||||
"Fused: %s (%s) -> FusedMoE (internal routing)",
|
||||
routed,
|
||||
moe_block_cls,
|
||||
)
|
||||
else:
|
||||
# MoE block forward is unmodified.
|
||||
# gate/router and shared expert (if any) runs in Transformers.
|
||||
# We then smuggle the topk_ids in using a custom op.
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
def custom_routing_function(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
):
|
||||
"""Return `topk_weights` from `gating_output` and the
|
||||
`topk_ids` we stored in the layer earlier."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
custom_routing_function, moe_state=moe_state
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
logger.info_once(
|
||||
"Fused: experts (%s) -> FusedMoE (external routing)",
|
||||
experts_cls,
|
||||
)
|
||||
fused_experts = FusedMoE(**kwargs)
|
||||
moe_block.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
# Update MixtureOfExperts mixin state
|
||||
self.mlp_layers.append(mlp)
|
||||
self.mlp_layers.append(moe_block)
|
||||
self.moe_layers.append(fused_experts)
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
Reference in New Issue
Block a user