Remove more unnecessary load_weights methods (#47058)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-30 15:22:16 +01:00
committed by GitHub
parent 0fc2512094
commit 1ab9522935
63 changed files with 1024 additions and 3774 deletions
@@ -127,7 +127,7 @@ def FusedMoE(
num_redundant_experts: int = 0,
has_bias: bool = False,
is_sequence_parallel: bool = False,
expert_mapping: list[tuple[str, str, int, str]] | None = None,
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
n_shared_experts: int | None = None,
router_logits_dtype: torch.dtype | None = None,
gate: torch.nn.Module | None = None,
@@ -356,7 +356,9 @@ def FusedMoE(
moe_config,
quant_config,
expert_map_manager=expert_map_manager,
expert_mapping=expert_mapping,
ckpt_gate_proj_name=ckpt_names[0],
ckpt_down_proj_name=ckpt_names[1],
ckpt_up_proj_name=ckpt_names[2],
# Extra params that are needed by quant_methods, pass along for now
# Prefer getting these from other sources, e.g. moe_config or
# router object
@@ -58,7 +58,9 @@ class RoutedExperts(PluggableLayer):
moe_config: FusedMoEConfig,
quant_config: QuantizationConfig | None,
expert_map_manager: ExpertMapManager,
expert_mapping: list[tuple[str, str, int, str]] | None = None,
ckpt_gate_proj_name: str = "gate_proj",
ckpt_down_proj_name: str = "down_proj",
ckpt_up_proj_name: str = "up_proj",
#
# Extra params that are needed by quant_methods, pass along for now
# Prefer getting these from other sources, e.g. moe_config or
@@ -81,7 +83,9 @@ class RoutedExperts(PluggableLayer):
self.layer_name = layer_name
self.moe_config = moe_config
self.quant_config = quant_config
self.expert_mapping = expert_mapping
self.ckpt_gate_proj_name = ckpt_gate_proj_name
self.ckpt_down_proj_name = ckpt_down_proj_name
self.ckpt_up_proj_name = ckpt_up_proj_name
self.expert_map_manager = expert_map_manager
self.hidden_size = moe_config.hidden_dim
self.global_num_experts = moe_config.num_experts
@@ -167,6 +171,8 @@ class RoutedExperts(PluggableLayer):
self.quant_method.create_weights(layer=self, **moe_quant_params)
self.lora_base_layer_prefix = ""
# TODO(bnell): Temporary hack. Get rid of this.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self.quant_method = quant_method
@@ -863,26 +869,33 @@ class RoutedExperts(PluggableLayer):
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
if (expert_mapping := self.expert_mapping) is None:
raise ValueError(
"`self.expert_mapping` must be provided to "
"load weights using `self.load_weights`."
)
expert_mapping = self.get_expert_mapping(include_fused=True)
unpadded_hidden = self.moe_config.hidden_dim_unpadded
for expert_name, loaded_weight in weights:
qual_name = f"{self.layer_name}.{expert_name}"
# Fused expert weights can be identified by their 3D tensors
is_fused = loaded_weight.dim() == 3
matched = False
for param_name, weight_name, expert_id, shard_id in expert_mapping:
if weight_name not in qual_name:
if matched and is_fused:
break
continue
matched = True
weight_name = qual_name.replace(weight_name, param_name)
param_name = weight_name.removeprefix(f"{self.layer_name}.")
param = getattr(self, param_name)
# Fused expert weights can be identified by their 3D tensors
if loaded_weight.dim() == 3:
# Repurpose expert_id as shard_idx for deconcatenating w1 and w3
if is_fused:
if shard_id in {"w1", "w3"}:
shard_idx = expert_id
experts_shard = loaded_weight.chunk(2, dim=1)[shard_idx]
if loaded_weight.shape[-1] != unpadded_hidden:
# [..., hidden, intermediate] -> [..., intermediate, hidden]
loaded_weight = loaded_weight.transpose(-1, -2)
# Repurpose expert_id for deconcatenating w1 and w3
experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
else:
if loaded_weight.shape[-2] != unpadded_hidden:
# [..., intermediate, hidden] -> [..., hidden, intermediate]
loaded_weight = loaded_weight.transpose(-1, -2)
experts_shard = loaded_weight
start = 0
else:
@@ -912,6 +925,27 @@ class RoutedExperts(PluggableLayer):
)
yield param_name
def get_expert_mapping(
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
moe_config = self.moe_config
num_fused_shared_experts = self.expert_map_manager.num_fused_shared_experts
num_redundant_experts = moe_config.num_experts - moe_config.num_logical_experts
return self.build_expert_params_mapping(
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
ckpt_down_proj_name or self.ckpt_down_proj_name,
ckpt_up_proj_name or self.ckpt_up_proj_name,
num_experts=moe_config.num_logical_experts + num_fused_shared_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
include_fused=include_fused,
)
@staticmethod
def make_expert_params_mapping(
model: torch.nn.Module,
@@ -921,6 +955,36 @@ class RoutedExperts(PluggableLayer):
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
) -> list[tuple[str, str, int, str]]:
"""Build the expert mapping, detecting the LoRA `base_layer.` prefix by
scanning `model`'s parameters.
Legacy entry point for models that still hand-roll `load_weights`; the
`RoutedExperts` weight loader uses `get_expert_mapping` /
`build_expert_params_mapping` instead (which take the prefix directly).
See `build_expert_params_mapping` for the returned tuple format.
"""
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
return RoutedExperts.build_expert_params_mapping(
ckpt_gate_proj_name,
ckpt_down_proj_name,
ckpt_up_proj_name,
num_experts,
num_redundant_experts,
routed_experts_prefix,
"base_layer." if has_base_layer else "",
)
@staticmethod
def build_expert_params_mapping(
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
lora_base_layer_prefix: str = "",
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
"""
Create expert parameter mapping for weight loading with redundant experts.
@@ -929,12 +993,13 @@ class RoutedExperts(PluggableLayer):
when loading weights with EPLB redundant experts.
Args:
model: The model containing the MoE layer
ckpt_gate_proj_name: Name of gate projection in checkpoint
ckpt_down_proj_name: Name of down projection in checkpoint
ckpt_up_proj_name: Name of up projection in checkpoint
num_experts: Number of logical (non-redundant) experts
num_redundant_experts: Number of redundant experts
lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer
include_fused: Prepend the fused pre-fused-checkpoint entries
Returns:
List of tuples (param_name, weight_name, expert_id, shard_id)
@@ -956,22 +1021,39 @@ class RoutedExperts(PluggableLayer):
)
)
base_layer = (
"base_layer."
if any(".base_layer." in name for name, _ in model.named_parameters())
else ""
)
if routed_experts_prefix != "":
routed_experts_prefix = f"{routed_experts_prefix}."
return [
w13 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w13_"
w2 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w2_"
fused_mapping = []
if include_fused:
gate_up = None
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
gate_up = "gate_up_proj"
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
gate_up = "w13"
else:
logger.warning(
"Unexpected gate/up projection names: %s, %s. "
"Fused gate/up mapping will be skipped.",
ckpt_gate_proj_name,
ckpt_up_proj_name,
)
if gate_up is not None:
fused_mapping = [
# (param_name, weight_name, expert_id, shard_id)
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
]
per_expert_mapping = [
# (param_name, weight_name, expert_id, shard_id)
(
f"experts.{routed_experts_prefix}{base_layer}w13_"
if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name]
else f"experts.{routed_experts_prefix}{base_layer}w2_",
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{base_layer}",
w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2,
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}",
expert_id,
shard_id,
)
@@ -983,6 +1065,8 @@ class RoutedExperts(PluggableLayer):
]
]
return fused_mapping + per_expert_mapping
def get_expert_weights(self) -> Iterable[torch.Tensor]:
def _maybe_make_contiguous(
name: str, p: torch.nn.Parameter
@@ -292,6 +292,11 @@ class MoERunner(MoERunnerInterface):
# For smuggling this layer into the fused moe custom op
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
def _select_forward(self) -> Callable:
if current_platform.is_tpu() or current_platform.is_cpu():
# TODO: Once the OOM issue for the TPU backend is resolved, we