Remove even more unnecessary load_weights methods (#48496)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-18 08:40:27 +00:00
committed by GitHub
parent d96aee0951
commit c233d90aa8
44 changed files with 636 additions and 2073 deletions
+35 -182
View File
@@ -32,10 +32,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -49,14 +46,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -74,45 +70,17 @@ def _zero_mean_tensor(t: torch.Tensor) -> torch.Tensor:
return t - t.mean()
def _rename_and_normalize_weights(
def _normalize_expert_bias(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterator[tuple[str, torch.Tensor]]:
"""
Translate HuggingFace Param2MoE weight names to vLLM internal names
and zero-mean the expert-bias tensor so the router stays balanced.
"""Zero-mean the MoE router's per-expert score bias for load balance.
Mapping table (HF → vLLM):
model.word_embeddings.* → model.embed_tokens.*
*.attention.query_key_value.* → *.self_attn.qkv_proj.*
*.attention.dense.* → *.self_attn.o_proj.*
*.attention.query_layernorm.* → *.self_attn.q_layernorm.*
*.attention.key_layernorm.* → *.self_attn.k_layernorm.*
*.mlp.gate.expert_bias → *.mlp.gate.e_score_correction_bias
(also zero-meant for load balance)
The rename to ``e_score_correction_bias`` is done by the mapper; only the
tensor adjustment lives here, since a WeightsMapper cannot transform data.
"""
for name, w in weights:
# Embedding table
name = name.replace("model.word_embeddings.", "model.embed_tokens.")
# Fused QKV projection (HF: query_key_value → vLLM: qkv_proj)
name = name.replace(".attention.query_key_value.", ".self_attn.qkv_proj.")
# Output projection (HF: dense → vLLM: o_proj)
name = name.replace(".attention.dense.", ".self_attn.o_proj.")
# Per-head query norm
name = name.replace(".attention.query_layernorm.", ".self_attn.q_layernorm.")
# Per-head key norm
name = name.replace(".attention.key_layernorm.", ".self_attn.k_layernorm.")
# Catch any remaining .attention. → .self_attn. prefixes
# (e.g. future bias params on the projection layers)
name = name.replace(".attention.", ".self_attn.")
# Expert-score bias: rename + zero-mean
if name.endswith(".mlp.gate.expert_bias"):
name = name.replace(
".mlp.gate.expert_bias",
".mlp.gate.e_score_correction_bias",
)
if _is_expert_bias_name(name):
w = _zero_mean_tensor(w)
yield name, w
@@ -123,7 +91,8 @@ class Param2MoEAttention(nn.Module):
Notable differences from a vanilla GQA layer:
* The checkpoint fuses Q, K, V into a single ``query_key_value`` weight.
vLLM receives it already renamed to ``qkv_proj`` by the weight-name
translator and splits it during ``load_weights``.
translator and loads it directly; ``QKVParallelLinear`` splits the
fused ``[Q|K|V]`` tensor internally.
* Optional per-head RMS norms on Q and K (``use_qk_norm=True``).
"""
@@ -472,6 +441,28 @@ class Param2MoEDecoderLayer(nn.Module):
class Param2MoEModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"word_embeddings.": "embed_tokens.",
# Fused query_key_value is renamed to qkv_proj and loaded directly;
# QKVParallelLinear splits the fused [Q|K|V] tensor internally.
".attention.query_key_value.": ".self_attn.qkv_proj.",
".attention.dense.": ".self_attn.o_proj.",
".attention.query_layernorm.": ".self_attn.q_layernorm.",
".attention.key_layernorm.": ".self_attn.k_layernorm.",
# Catch-all for any remaining .attention. names (must come last).
".attention.": ".self_attn.",
".mlp.gate.expert_bias": ".mlp.gate.e_score_correction_bias",
},
orig_to_new_stacked={
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
},
)
def __init__(
self,
*,
@@ -558,147 +549,9 @@ class Param2MoEModel(nn.Module):
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
"""
Custom weight loader for the inner Param2MoEModel.
Receives weights that have already been renamed/normalised by the
outer model and whose ``model.`` prefix has been stripped by
``AutoWeightsLoader``. Handles:
1. Fused QKV split (query_key_value → qkv_proj q/k/v shards).
2. gate_proj + up_proj → gate_up_proj stacking (dense + shared-exp).
3. Routed-expert weights via the fused-MoE mapping.
4. All remaining weights via their default loader.
"""
config = self.config
num_heads: int = config.num_attention_heads
num_kv_heads: int = config.num_key_value_heads
head_dim: int = config.head_dim or (config.hidden_size // num_heads)
q_split = num_heads * head_dim
kv_split = num_kv_heads * head_dim
stacked_params_mapping = [
# (vllm_param_name, ckpt_weight_name, shard_id)
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
# ------------------------------------------------------------------
# 1. Fused QKV: split into q / k / v shards for QKVParallelLinear
# ------------------------------------------------------------------
if name.endswith(".self_attn.qkv_proj.weight"):
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
q_w = loaded_weight[:q_split, :]
k_w = loaded_weight[q_split : q_split + kv_split, :]
v_w = loaded_weight[q_split + kv_split :, :]
weight_loader(param, q_w, "q")
weight_loader(param, k_w, "k")
weight_loader(param, v_w, "v")
loaded_params.add(name)
continue
# ------------------------------------------------------------------
# 2. gate_proj / up_proj → gate_up_proj (dense MLP + shared-exp.)
# ------------------------------------------------------------------
matched_stacked = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name: # routed experts handled below
continue
new_name = name.replace(weight_name, param_name)
if new_name.endswith(".bias") and new_name not in params_dict:
continue
if new_name not in params_dict:
continue
if is_pp_missing_parameter(new_name, self):
continue
param = params_dict[new_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(new_name)
matched_stacked = True
break
if matched_stacked:
continue
# ------------------------------------------------------------------
# 3. Routed expert weights → fused-MoE kernel layout
# ------------------------------------------------------------------
matched_expert = False
for (
param_name,
weight_name,
expert_id,
shard_id,
) in expert_params_mapping:
if weight_name not in name:
continue
new_name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(new_name, self):
continue
if new_name not in params_dict:
continue
param = params_dict[new_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
loaded_params.add(new_name)
matched_expert = True
break
if matched_expert:
continue
# ------------------------------------------------------------------
# 4. All other weights: direct load (layernorms, embed_tokens, …)
# ------------------------------------------------------------------
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
try:
weight_loader(param, loaded_weight)
except Exception as e:
raise RuntimeError(
f"[param2moe] Failed to load weight '{name}' "
f"with shape {tuple(loaded_weight.shape)} "
f"into param type {type(param).__name__}: {e}"
) from e
loaded_params.add(name)
return loaded_params
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
loader = AutoWeightsLoader(self)
return loader.load_weights(
_normalize_expert_bias(weights), mapper=self.hf_to_vllm_mapper
)
@@ -859,4 +712,4 @@ class Param2MoEForCausalLM(
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(_rename_and_normalize_weights(weights))
return loader.load_weights(weights)