Remove more unnecessary load_weights methods (#47058)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -127,7 +127,7 @@ def FusedMoE(
|
||||
num_redundant_experts: int = 0,
|
||||
has_bias: bool = False,
|
||||
is_sequence_parallel: bool = False,
|
||||
expert_mapping: list[tuple[str, str, int, str]] | None = None,
|
||||
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
|
||||
n_shared_experts: int | None = None,
|
||||
router_logits_dtype: torch.dtype | None = None,
|
||||
gate: torch.nn.Module | None = None,
|
||||
@@ -356,7 +356,9 @@ def FusedMoE(
|
||||
moe_config,
|
||||
quant_config,
|
||||
expert_map_manager=expert_map_manager,
|
||||
expert_mapping=expert_mapping,
|
||||
ckpt_gate_proj_name=ckpt_names[0],
|
||||
ckpt_down_proj_name=ckpt_names[1],
|
||||
ckpt_up_proj_name=ckpt_names[2],
|
||||
# Extra params that are needed by quant_methods, pass along for now
|
||||
# Prefer getting these from other sources, e.g. moe_config or
|
||||
# router object
|
||||
|
||||
@@ -58,7 +58,9 @@ class RoutedExperts(PluggableLayer):
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
expert_map_manager: ExpertMapManager,
|
||||
expert_mapping: list[tuple[str, str, int, str]] | None = None,
|
||||
ckpt_gate_proj_name: str = "gate_proj",
|
||||
ckpt_down_proj_name: str = "down_proj",
|
||||
ckpt_up_proj_name: str = "up_proj",
|
||||
#
|
||||
# Extra params that are needed by quant_methods, pass along for now
|
||||
# Prefer getting these from other sources, e.g. moe_config or
|
||||
@@ -81,7 +83,9 @@ class RoutedExperts(PluggableLayer):
|
||||
self.layer_name = layer_name
|
||||
self.moe_config = moe_config
|
||||
self.quant_config = quant_config
|
||||
self.expert_mapping = expert_mapping
|
||||
self.ckpt_gate_proj_name = ckpt_gate_proj_name
|
||||
self.ckpt_down_proj_name = ckpt_down_proj_name
|
||||
self.ckpt_up_proj_name = ckpt_up_proj_name
|
||||
self.expert_map_manager = expert_map_manager
|
||||
self.hidden_size = moe_config.hidden_dim
|
||||
self.global_num_experts = moe_config.num_experts
|
||||
@@ -167,6 +171,8 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
self.quant_method.create_weights(layer=self, **moe_quant_params)
|
||||
|
||||
self.lora_base_layer_prefix = ""
|
||||
|
||||
# TODO(bnell): Temporary hack. Get rid of this.
|
||||
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
|
||||
self.quant_method = quant_method
|
||||
@@ -863,26 +869,33 @@ class RoutedExperts(PluggableLayer):
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
if (expert_mapping := self.expert_mapping) is None:
|
||||
raise ValueError(
|
||||
"`self.expert_mapping` must be provided to "
|
||||
"load weights using `self.load_weights`."
|
||||
)
|
||||
expert_mapping = self.get_expert_mapping(include_fused=True)
|
||||
unpadded_hidden = self.moe_config.hidden_dim_unpadded
|
||||
for expert_name, loaded_weight in weights:
|
||||
qual_name = f"{self.layer_name}.{expert_name}"
|
||||
# Fused expert weights can be identified by their 3D tensors
|
||||
is_fused = loaded_weight.dim() == 3
|
||||
matched = False
|
||||
for param_name, weight_name, expert_id, shard_id in expert_mapping:
|
||||
if weight_name not in qual_name:
|
||||
if matched and is_fused:
|
||||
break
|
||||
continue
|
||||
matched = True
|
||||
weight_name = qual_name.replace(weight_name, param_name)
|
||||
param_name = weight_name.removeprefix(f"{self.layer_name}.")
|
||||
param = getattr(self, param_name)
|
||||
# Fused expert weights can be identified by their 3D tensors
|
||||
if loaded_weight.dim() == 3:
|
||||
# Repurpose expert_id as shard_idx for deconcatenating w1 and w3
|
||||
if is_fused:
|
||||
if shard_id in {"w1", "w3"}:
|
||||
shard_idx = expert_id
|
||||
experts_shard = loaded_weight.chunk(2, dim=1)[shard_idx]
|
||||
if loaded_weight.shape[-1] != unpadded_hidden:
|
||||
# [..., hidden, intermediate] -> [..., intermediate, hidden]
|
||||
loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
# Repurpose expert_id for deconcatenating w1 and w3
|
||||
experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
|
||||
else:
|
||||
if loaded_weight.shape[-2] != unpadded_hidden:
|
||||
# [..., intermediate, hidden] -> [..., hidden, intermediate]
|
||||
loaded_weight = loaded_weight.transpose(-1, -2)
|
||||
experts_shard = loaded_weight
|
||||
start = 0
|
||||
else:
|
||||
@@ -912,6 +925,27 @@ class RoutedExperts(PluggableLayer):
|
||||
)
|
||||
yield param_name
|
||||
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name: str | None = None,
|
||||
ckpt_down_proj_name: str | None = None,
|
||||
ckpt_up_proj_name: str | None = None,
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
moe_config = self.moe_config
|
||||
num_fused_shared_experts = self.expert_map_manager.num_fused_shared_experts
|
||||
num_redundant_experts = moe_config.num_experts - moe_config.num_logical_experts
|
||||
return self.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name or self.ckpt_down_proj_name,
|
||||
ckpt_up_proj_name or self.ckpt_up_proj_name,
|
||||
num_experts=moe_config.num_logical_experts + num_fused_shared_experts,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
routed_experts_prefix="",
|
||||
lora_base_layer_prefix=self.lora_base_layer_prefix,
|
||||
include_fused=include_fused,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def make_expert_params_mapping(
|
||||
model: torch.nn.Module,
|
||||
@@ -921,6 +955,36 @@ class RoutedExperts(PluggableLayer):
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""Build the expert mapping, detecting the LoRA `base_layer.` prefix by
|
||||
scanning `model`'s parameters.
|
||||
|
||||
Legacy entry point for models that still hand-roll `load_weights`; the
|
||||
`RoutedExperts` weight loader uses `get_expert_mapping` /
|
||||
`build_expert_params_mapping` instead (which take the prefix directly).
|
||||
See `build_expert_params_mapping` for the returned tuple format.
|
||||
"""
|
||||
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
|
||||
return RoutedExperts.build_expert_params_mapping(
|
||||
ckpt_gate_proj_name,
|
||||
ckpt_down_proj_name,
|
||||
ckpt_up_proj_name,
|
||||
num_experts,
|
||||
num_redundant_experts,
|
||||
routed_experts_prefix,
|
||||
"base_layer." if has_base_layer else "",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def build_expert_params_mapping(
|
||||
ckpt_gate_proj_name: str,
|
||||
ckpt_down_proj_name: str,
|
||||
ckpt_up_proj_name: str,
|
||||
num_experts: int,
|
||||
num_redundant_experts: int = 0,
|
||||
routed_experts_prefix: str = "routed_experts",
|
||||
lora_base_layer_prefix: str = "",
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
"""
|
||||
Create expert parameter mapping for weight loading with redundant experts.
|
||||
@@ -929,12 +993,13 @@ class RoutedExperts(PluggableLayer):
|
||||
when loading weights with EPLB redundant experts.
|
||||
|
||||
Args:
|
||||
model: The model containing the MoE layer
|
||||
ckpt_gate_proj_name: Name of gate projection in checkpoint
|
||||
ckpt_down_proj_name: Name of down projection in checkpoint
|
||||
ckpt_up_proj_name: Name of up projection in checkpoint
|
||||
num_experts: Number of logical (non-redundant) experts
|
||||
num_redundant_experts: Number of redundant experts
|
||||
lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer
|
||||
include_fused: Prepend the fused pre-fused-checkpoint entries
|
||||
|
||||
Returns:
|
||||
List of tuples (param_name, weight_name, expert_id, shard_id)
|
||||
@@ -956,22 +1021,39 @@ class RoutedExperts(PluggableLayer):
|
||||
)
|
||||
)
|
||||
|
||||
base_layer = (
|
||||
"base_layer."
|
||||
if any(".base_layer." in name for name, _ in model.named_parameters())
|
||||
else ""
|
||||
)
|
||||
|
||||
if routed_experts_prefix != "":
|
||||
routed_experts_prefix = f"{routed_experts_prefix}."
|
||||
|
||||
return [
|
||||
w13 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w13_"
|
||||
w2 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w2_"
|
||||
|
||||
fused_mapping = []
|
||||
if include_fused:
|
||||
gate_up = None
|
||||
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
|
||||
gate_up = "gate_up_proj"
|
||||
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
|
||||
gate_up = "w13"
|
||||
else:
|
||||
logger.warning(
|
||||
"Unexpected gate/up projection names: %s, %s. "
|
||||
"Fused gate/up mapping will be skipped.",
|
||||
ckpt_gate_proj_name,
|
||||
ckpt_up_proj_name,
|
||||
)
|
||||
if gate_up is not None:
|
||||
fused_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
|
||||
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
|
||||
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
|
||||
]
|
||||
|
||||
per_expert_mapping = [
|
||||
# (param_name, weight_name, expert_id, shard_id)
|
||||
(
|
||||
f"experts.{routed_experts_prefix}{base_layer}w13_"
|
||||
if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name]
|
||||
else f"experts.{routed_experts_prefix}{base_layer}w2_",
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{base_layer}",
|
||||
w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2,
|
||||
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}",
|
||||
expert_id,
|
||||
shard_id,
|
||||
)
|
||||
@@ -983,6 +1065,8 @@ class RoutedExperts(PluggableLayer):
|
||||
]
|
||||
]
|
||||
|
||||
return fused_mapping + per_expert_mapping
|
||||
|
||||
def get_expert_weights(self) -> Iterable[torch.Tensor]:
|
||||
def _maybe_make_contiguous(
|
||||
name: str, p: torch.nn.Parameter
|
||||
|
||||
@@ -292,6 +292,11 @@ class MoERunner(MoERunnerInterface):
|
||||
# For smuggling this layer into the fused moe custom op
|
||||
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
|
||||
|
||||
def load_weights(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[str]:
|
||||
return self.routed_experts.load_weights(weights)
|
||||
|
||||
def _select_forward(self) -> Callable:
|
||||
if current_platform.is_tpu() or current_platform.is_cpu():
|
||||
# TODO: Once the OOM issue for the TPU backend is resolved, we
|
||||
|
||||
Reference in New Issue
Block a user