[MoE Refactor] FusedMoE/MoERunner inversion refactor (#41184)
Signed-off-by: Bill Nell <bnell@redhat.com>
This commit is contained in:
@@ -16,6 +16,7 @@ from vllm.inputs import MultiModalDataDict
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
@@ -216,7 +217,7 @@ class AriaProjector(nn.Module):
|
||||
return out
|
||||
|
||||
|
||||
class AriaFusedMoE(FusedMoE):
|
||||
class AriaRoutedExperts(RoutedExperts):
|
||||
def weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
|
||||
) -> None:
|
||||
@@ -225,13 +226,14 @@ class AriaFusedMoE(FusedMoE):
|
||||
# up weights for each expert.
|
||||
# Note: Loading expert weights with quantization is not supported
|
||||
tp_rank = get_tensor_model_parallel_rank()
|
||||
tp_size = self.moe_config.tp_size
|
||||
if shard_id == "w13":
|
||||
# the shape of loaded_weight is
|
||||
# (num_experts, hidden_size, 2 * moe_intermediate_size)
|
||||
if self.tp_size > 1:
|
||||
if tp_size > 1:
|
||||
up, gate = loaded_weight.chunk(2, dim=-1)
|
||||
up_current_rank = up.chunk(self.tp_size, dim=-1)[tp_rank]
|
||||
gate_current_rank = gate.chunk(self.tp_size, dim=-1)[tp_rank]
|
||||
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
|
||||
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
|
||||
up_and_gate = torch.cat(
|
||||
[up_current_rank, gate_current_rank], dim=-1
|
||||
).transpose(1, 2)
|
||||
@@ -241,8 +243,8 @@ class AriaFusedMoE(FusedMoE):
|
||||
elif shard_id == "w2":
|
||||
# the shape of loaded_weight is
|
||||
# (num_experts, moe_intermediate_size, hidden_size)
|
||||
if self.tp_size > 1:
|
||||
down_current_rank = loaded_weight.chunk(self.tp_size, dim=1)[tp_rank]
|
||||
if tp_size > 1:
|
||||
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
|
||||
param.data.copy_(down_current_rank.transpose(1, 2))
|
||||
else:
|
||||
param.data.copy_(loaded_weight.transpose(1, 2))
|
||||
@@ -278,7 +280,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
bias=config.mlp_bias,
|
||||
)
|
||||
|
||||
self.experts = AriaFusedMoE(
|
||||
self.experts = FusedMoE(
|
||||
shared_experts=self.shared_experts,
|
||||
num_experts=config.moe_num_experts,
|
||||
top_k=config.moe_topk,
|
||||
@@ -286,6 +288,7 @@ class AriaTextMoELayer(nn.Module):
|
||||
intermediate_size=config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
routed_experts_cls=AriaRoutedExperts,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -351,8 +354,8 @@ class AriaTextModel(LlamaModel, SupportsQuant):
|
||||
(".qkv_proj", ".v_proj", "v"),
|
||||
(".gate_up_proj", ".gate_proj", 0),
|
||||
(".gate_up_proj", ".up_proj", 1),
|
||||
("experts.w13_weight", "experts.fc1.weight", "w13"),
|
||||
("experts.w2_weight", "experts.fc2.weight", "w2"),
|
||||
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
|
||||
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
|
||||
]
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
|
||||
Reference in New Issue
Block a user