Remove even more unnecessary load_weights methods (#48496)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-18 08:40:27 +00:00
committed by GitHub
parent d96aee0951
commit c233d90aa8
44 changed files with 636 additions and 2073 deletions
+1 -13
View File
@@ -79,6 +79,7 @@ from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
@@ -1143,16 +1144,3 @@ class AXK1ForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_spec_layer_idx_from_weight_name(
config: AXK1Config, weight_name: str
) -> int | None:
if config.num_nextn_predict_layers and config.num_nextn_predict_layers > 0:
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if weight_name.startswith(
f"model.layers.{layer_idx + i}."
) or weight_name.startswith(f"layers.{layer_idx + i}."):
return layer_idx + i
return None
+22 -128
View File
@@ -11,21 +11,13 @@ from transformers.models.aria.processing_aria import AriaProcessor
from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.distributed import get_tensor_model_parallel_rank
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
RoutedExperts,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
@@ -51,7 +43,6 @@ from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
from .utils import (
AutoWeightsLoader,
WeightsMapper,
is_pp_missing_parameter,
maybe_prefix,
)
@@ -94,34 +85,18 @@ class AriaVisionTransformer(Idefics3VisionTransformer, SupportsQuant):
# Identity layer
self.post_layernorm = nn.Identity()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# NOTE: post_layernorm is not used in Aria
if "post_layernorm" in name:
continue
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# NOTE: post_layernorm is not used in Aria.
loader = AutoWeightsLoader(self, skip_substrs=["post_layernorm"])
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class AriaProjectorMLP(nn.Module):
@@ -217,39 +192,6 @@ class AriaProjector(nn.Module):
return out
class AriaRoutedExperts(RoutedExperts):
def weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str
) -> None:
# Override the weight_loader to handle the expert weights in the Aria
# model, which are already packed with experts, and merge the gate and
# up weights for each expert.
# Note: Loading expert weights with quantization is not supported
tp_rank = get_tensor_model_parallel_rank()
tp_size = self.moe_config.tp_size
if shard_id == "w13":
# the shape of loaded_weight is
# (num_experts, hidden_size, 2 * moe_intermediate_size)
if tp_size > 1:
up, gate = loaded_weight.chunk(2, dim=-1)
up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank]
gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank]
up_and_gate = torch.cat(
[up_current_rank, gate_current_rank], dim=-1
).transpose(1, 2)
param.data.copy_(up_and_gate)
else:
param.data.copy_(loaded_weight.transpose(1, 2))
elif shard_id == "w2":
# the shape of loaded_weight is
# (num_experts, moe_intermediate_size, hidden_size)
if tp_size > 1:
down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank]
param.data.copy_(down_current_rank.transpose(1, 2))
else:
param.data.copy_(loaded_weight.transpose(1, 2))
class AriaTextMoELayer(nn.Module):
"""
Mixture of Experts (MoE) Layer for the AriaMoE model.
@@ -288,7 +230,6 @@ class AriaTextMoELayer(nn.Module):
intermediate_size=config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.experts",
routed_experts_cls=AriaRoutedExperts,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -335,70 +276,23 @@ class AriaTextModel(LlamaModel, SupportsQuant):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
"experts.routed_experts.w13_weight": ["experts.fc1.weight"],
"experts.routed_experts.w2_weight": ["experts.fc2.weight"],
}
# Aria packs all experts into single (transposed) fc1/fc2 tensors, which is
# exactly the pre-fused checkpoint layout FusedMoE self-loads once fc1/fc2
# are renamed to the fused gate_up_proj/down_proj names.
hf_to_vllm_mapper = LlamaModel.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_substr={
"experts.fc1": "experts.gate_up_proj",
"experts.fc2": "experts.down_proj",
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(
vllm_config=vllm_config, prefix=prefix, layer_type=AriaTextDecoderLayer
)
# Adapted from LlamaModel.load_weights with the modification of adding
# the expert weights mapping to `stacked_params_mapping`
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"),
("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class AriaProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
+24 -92
View File
@@ -41,10 +41,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -58,14 +55,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -379,6 +375,16 @@ class BailingMoeBlock(nn.Module):
@support_torch_compile
class BailingMoeModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
)
def __init__(
self,
*,
@@ -468,89 +474,9 @@ class BailingMoeModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if (
hasattr(self.config, "norm_head")
and self.config.norm_head
and "lm_head.weight" in name
):
loaded_weight = F.normalize(loaded_weight, dim=0, p=2, eps=1e-7)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
@@ -622,15 +548,21 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def _normalize_lm_head(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
norm_head = getattr(self.config, "norm_head", False)
for name, loaded_weight in weights:
if norm_head and "lm_head.weight" in name:
loaded_weight = F.normalize(loaded_weight, dim=0, p=2, eps=1e-7)
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
return loader.load_weights(self._normalize_lm_head(weights))
class BailingMoeV2ForCausalLM(BailingMoeForCausalLM):
+14 -29
View File
@@ -47,13 +47,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsPP, SupportsQuant
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -297,36 +295,23 @@ class BloomModel(nn.Module):
hidden_states = self.ln_f(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
def _repack_qkv(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
# BLOOM's fused QKV is laid out as (num_heads * 3 * head_size) on its
# output dim (0), while vLLM expects (3 * num_heads * head_size).
num_heads = self.config.num_attention_heads
for name, loaded_weight in weights:
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
if "query_key_value" in name:
# NOTE: BLOOM's fused QKV's output_dim has the shape of
# (num_heads * 3 * head_size), while the
# required shape is (3 * num_heads * head_size).
# Thus, we need weight conversion.
output_dim = getattr(param, "output_dim", None)
num_heads = self.config.num_attention_heads
if output_dim is not None:
loaded_weight_shape = loaded_weight.shape
loaded_weight = loaded_weight.view(
loaded_weight_shape[:output_dim]
+ (num_heads, 3, -1)
+ loaded_weight_shape[output_dim + 1 :]
)
loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1)
loaded_weight = loaded_weight.reshape(loaded_weight_shape)
shape = loaded_weight.shape
loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:])
loaded_weight = loaded_weight.transpose(0, 1)
loaded_weight = loaded_weight.reshape(shape)
yield name, loaded_weight
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(self._repack_qkv(weights))
class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant):
+1 -3
View File
@@ -16,9 +16,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
+3 -10
View File
@@ -22,9 +22,9 @@ from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.attention import MMEncoderAttention
from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .clip import CLIPEncoder, CLIPVisionEmbeddings
from .utils import AutoWeightsLoader
class MLPBlock(nn.Module):
@@ -671,12 +671,5 @@ class DeepCLIPVisionTransformer(nn.Module):
return encoder_outputs
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+5 -2
View File
@@ -33,9 +33,12 @@ from .deepseek_v2 import (
DeepseekV2MixtureOfExperts,
DeepseekV2MoE,
_try_load_fp8_indexer_wk,
get_spec_layer_idx_from_weight_name,
)
from .utils import get_pp_missing_layer_names, maybe_prefix
from .utils import (
get_pp_missing_layer_names,
get_spec_layer_idx_from_weight_name,
maybe_prefix,
)
def _restore_full_token_layout_if_needed(
+1 -18
View File
@@ -108,6 +108,7 @@ from .interfaces import (
from .utils import (
PPMissingLayer,
get_pp_missing_layer_names,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
@@ -1918,21 +1919,3 @@ class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM):
class GlmMoeDsaForCausalLM(DeepseekV2ForCausalLM):
pass
# Compatibility with
# https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/blob/main/configuration_deepseek.py
def get_spec_layer_idx_from_weight_name(
config: DeepseekV2Config | DeepseekV3Config, weight_name: str
) -> int | None:
if (
hasattr(config, "num_nextn_predict_layers")
and config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if weight_name.startswith(
f"model.layers.{layer_idx + i}."
) or weight_name.startswith(f"layers.{layer_idx + i}."):
return layer_idx + i
return None
+31 -136
View File
@@ -23,8 +23,7 @@
# limitations under the License.
"""Inference-only ErineMoE model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
from typing import Any
@@ -42,11 +41,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -61,10 +56,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.config import set_default_rope_theta
@@ -72,8 +63,8 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -407,6 +398,19 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
@support_torch_compile
class Ernie4_5_MoeModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -486,132 +490,26 @@ class Ernie4_5_MoeModel(nn.Module):
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.moe_num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
def _preprocess(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
for name, loaded_weight in weights:
if self.config.tie_word_embeddings and name.endswith("lm_head.weight"):
continue
# MTP will be supported soon.
if "mtp" in name:
continue
# moe_statics.e_score_correction_bias is stored with a leading
# singleton dim and under a different module name.
if "e_score_correction_bias" in name:
name = name.replace("moe_statics", "gate")
loaded_weight = loaded_weight.squeeze(0)
yield name, loaded_weight
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
# Do not modify `name` since the loop may continue here
# Instead, create a new variable
name_mapped = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name_mapped, self):
continue
# Skip loading extra bias for GPTQ models.
if (
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
) and name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_substrs=["mtp"],
ignore_unexpected_suffixes=[".bias", "_bias"],
)
return loader.load_weights(
self._preprocess(weights), mapper=self.hf_to_vllm_mapper
)
class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExperts):
@@ -741,6 +639,3 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+17 -32
View File
@@ -30,7 +30,6 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
@@ -53,7 +52,12 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.import_utils import _has_module
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
from .utils import (
AutoWeightsLoader,
WeightsMapper,
init_vllm_registered_model,
maybe_prefix,
)
class _SinusoidsPositionEmbedding(nn.Module):
@@ -79,6 +83,14 @@ class _SinusoidsPositionEmbedding(nn.Module):
class FunAudioChatAudioAttention(nn.Module):
"""Multi-headed attention used inside the continuous audio tower."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(self, config: Any):
super().__init__()
self.embed_dim = int(config.d_model)
@@ -123,42 +135,15 @@ class FunAudioChatAudioAttention(nn.Module):
bias=True,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
with torch.no_grad():
if self.qkv_proj.bias is not None:
# HF FunAudioChat uses bias=False for k_proj. Ensure the missing
# shard starts as zeros, while allowing q/v shards to load.
self.qkv_proj.bias.zero_()
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, shard_name, shard_id in stacked_params_mapping:
if shard_name not in name:
continue
name = name.replace(shard_name, param_name)
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def forward(
self,
+22 -57
View File
@@ -44,18 +44,14 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata
from .interfaces import SupportsQuant
from .utils import (
AutoWeightsLoader,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
)
@@ -788,6 +784,25 @@ class Gemma3nCrossDecoder(nn.Module):
enable_if=lambda vllm_config: not vllm_config.cache_config.kv_sharing_fast_prefill
)
class Gemma3nTextModel(nn.Module, SupportsQuant):
# Decoder layers, altup_unembed_projections and norm live on the text
# model; every other submodule lives under self_decoder.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"embed_tokens.": "self_decoder.embed_tokens.",
"embed_tokens_per_layer.": "self_decoder.embed_tokens_per_layer.",
"per_layer_model_projection.": "self_decoder.per_layer_model_projection.",
"per_layer_projection_norm.": "self_decoder.per_layer_projection_norm.",
"altup_projections.": "self_decoder.altup_projections.",
},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
@@ -1036,58 +1051,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
return self.norm(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# decoder layer weights, altup_unembed_projections and rmsnorm
# are initialized in text model, others are in self decoder
if (
not name.startswith("layers")
and not name.startswith("altup_unembed_projections")
and not name.startswith("norm")
):
name = f"self_decoder.{name}"
for param_name, shard_name, shard_id in stacked_params_mapping:
if shard_name not in name:
continue
# Avoid spurious match with ".up_proj".
if "altup_projections" in name:
continue
name = name.replace(shard_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Gemma3nForCausalLM(nn.Module):
-13
View File
@@ -303,16 +303,3 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
]
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights)
def get_spec_layer_idx_from_weight_name(
config: Glm4Config, weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if f"layers.{layer_idx + i}." in weight_name:
return layer_idx + i
return None
+24 -208
View File
@@ -24,8 +24,7 @@
"""Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model
compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
import torch
@@ -43,10 +42,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -60,20 +56,18 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_fuse_shared_experts,
maybe_prefix,
skip_spec_layers,
)
logger = init_logger(__name__)
@@ -410,6 +404,19 @@ class Glm4MoeDecoderLayer(nn.Module):
}
)
class Glm4MoeModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -480,189 +487,14 @@ class Glm4MoeModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
# FSE widens the mapping by n_shared_experts slots; see deepseek_v2.py.
num_experts = self.config.n_routed_experts
if (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and self.config.n_shared_experts
):
num_experts += self.config.n_shared_experts
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
rocm_aiter_moe_shared_expert_enabled = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
weights = maybe_fuse_shared_experts(
skip_spec_layers(weights, self.config),
n_routed_experts=self.config.n_routed_experts,
n_shared_experts=self.config.n_shared_experts or 1,
)
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is not None:
continue
is_fusion_moe_shared_experts_layer = (
rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name)
)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
if is_fusion_moe_shared_experts_layer:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
# FSE: split a widened mlp.shared_experts tensor into
# n_shared_experts chunks; see deepseek_v2.py for details.
num_chunks = 1
split_dim = 0
chunk_size = 0
if is_fusion_moe_shared_experts_layer:
num_chunks = getattr(self.config, "n_shared_experts", 1) or 1
split_dim = (
1
if ("down_proj.weight" in name and loaded_weight.ndim > 1)
else 0
)
total = loaded_weight.shape[split_dim]
if total % num_chunks != 0:
raise ValueError(
f"FSE shared-expert weight {name} has dim "
f"{total} along axis {split_dim} which is not "
f"divisible by n_shared_experts={num_chunks}."
)
chunk_size = total // num_chunks
for j in range(num_chunks):
chunk_name = name
weight_to_load = loaded_weight
if is_fusion_moe_shared_experts_layer:
chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
if loaded_weight.ndim == 1:
weight_to_load = loaded_weight[chunk_slice]
elif split_dim == 0:
weight_to_load = loaded_weight[chunk_slice, :]
else:
weight_to_load = loaded_weight[:, chunk_slice]
# Synthesize an expert-style name for expert mapping.
chunk_name = name.replace(
"mlp.shared_experts",
f"mlp.experts.{self.config.n_routed_experts + j}",
)
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in chunk_name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
# Do not modify `name` since the loop may continue here
# Instead, create a new variable
name_mapped = chunk_name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success
# or not here since otherwise we may skip experts
# with other available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
if not is_fusion_moe_shared_experts_layer:
name = name_mapped
else:
loaded_params.add(name_mapped)
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
if name is not None and not is_fusion_moe_shared_experts_layer:
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Glm4MixtureOfExperts(MixtureOfExperts):
@@ -776,19 +608,3 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, Glm4MixtureOfExper
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
def get_spec_layer_idx_from_weight_name(
config: Glm4MoeConfig, weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if f"layers.{layer_idx + i}." in weight_name:
return layer_idx + i
return None
+1 -13
View File
@@ -70,6 +70,7 @@ from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
@@ -628,16 +629,3 @@ class Glm4MoeLiteForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_spec_layer_idx_from_weight_name(
config: "Glm4MoeLiteConfig", weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if f"layers.{layer_idx + i}." in weight_name:
return layer_idx + i
return None
@@ -54,10 +54,9 @@ from .glm4_moe_lite import (
Glm4MixtureOfExperts,
Glm4MoeLite,
Glm4MoeLiteDecoderLayer,
get_spec_layer_idx_from_weight_name,
)
from .interfaces import SupportsPP
from .utils import maybe_prefix
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
class SharedHead(nn.Module):
+1 -2
View File
@@ -51,9 +51,8 @@ from .glm4_moe import (
Glm4MixtureOfExperts,
Glm4MoE,
Glm4MoeDecoderLayer,
get_spec_layer_idx_from_weight_name,
)
from .utils import maybe_prefix
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
class SharedHead(nn.Module):
+2 -1
View File
@@ -41,13 +41,14 @@ from vllm.model_executor.model_loader.weight_utils import (
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from .glm4 import Glm4DecoderLayer, get_spec_layer_idx_from_weight_name
from .glm4 import Glm4DecoderLayer
from .glm4_moe_lite_mtp import (
Glm4MoeLiteMultiTokenPredictor,
SharedHead,
)
from .interfaces import SupportsPP
from .utils import (
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
maybe_prefix,
)
+16 -38
View File
@@ -65,7 +65,12 @@ from .interfaces import (
SupportsPP,
SupportsTranscription,
)
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
from .utils import (
AutoWeightsLoader,
WeightsMapper,
init_vllm_registered_model,
maybe_prefix,
)
from .whisper import ISO639_1_SUPPORTED_LANGS, _create_fake_bias_for_k_proj
@@ -395,6 +400,14 @@ class GlmAsrEncoder(nn.Module):
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config,
@@ -496,44 +509,9 @@ class GlmAsrEncoder(nn.Module):
return _GlmAsrEncoderOutput(last_hidden_state=hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Custom weight loading to handle q_proj/k_proj/v_proj -> qkv_proj mapping."""
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight")
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Default weight loading for non-stacked params
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class GlmAsrFeatureInputs(TensorSchema):
+17 -26
View File
@@ -47,13 +47,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsCrossEncoding, SupportsPP
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -241,32 +239,25 @@ class GPT2Model(nn.Module):
hidden_states = self.ln_f(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
def _transpose_conv1d(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
# HF's GPT-2 uses Conv1D instead of Linear, so its 2D weights are
# stored transposed relative to what vLLM expects.
# Note(zhuohan): the logic below might break quantized models.
for name, loaded_weight in weights:
if ".attn.bias" in name or ".attn.masked_bias" in name:
# Skip attention mask.
# NOTE: "c_attn.bias" should not be skipped.
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
# The HF's GPT-2 implementation uses Conv1D instead of Linear.
# Because of this, we need to transpose the weights.
# Note(zhuohan): the logic below might break quantized models.
for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]:
if conv1d_weight_name not in name:
continue
if not name.endswith(".weight"):
continue
if name.endswith(".weight") and any(
proj in name for proj in ("c_attn", "c_proj", "c_fc")
):
loaded_weight = loaded_weight.t()
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# Skip attention mask buffers; NOTE: "c_attn.bias" must not be skipped.
loader = AutoWeightsLoader(
self, skip_substrs=[".attn.bias", ".attn.masked_bias"]
)
return loader.load_weights(self._transpose_conv1d(weights))
class GPT2LMHeadModel(nn.Module, SupportsPP):
+16 -38
View File
@@ -43,13 +43,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsPP
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -249,45 +247,25 @@ class GPTNeoXModel(nn.Module):
hidden_states = self.final_layer_norm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _repack_qkv(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
# GPT-NeoX's fused QKV is laid out as (num_heads * 3 * head_size) on
# its output dim (0), while vLLM expects (3 * num_heads * head_size).
num_heads = self.config.num_attention_heads
for name, loaded_weight in weights:
if (
"attention.bias" in name
or "attention.masked_bias" in name
or "rotary_emb.inv_freq" in name
):
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using OpenRLHF may include
# these tensors in the checkpoint. Skip them.
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
if "query_key_value" in name:
# NOTE: GPT-NeoX's fused QKV's output_dim has the shape of
# (num_heads * 3 * head_size), while the
# required shape is (3 * num_heads * head_size).
# Thus, we need weight conversion.
output_dim = getattr(param, "output_dim", None)
num_heads = self.config.num_attention_heads
if output_dim is not None:
loaded_weight_shape = loaded_weight.shape
loaded_weight = loaded_weight.view(
loaded_weight_shape[:output_dim]
+ (num_heads, 3, -1)
+ loaded_weight_shape[output_dim + 1 :]
)
loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1)
loaded_weight = loaded_weight.reshape(loaded_weight_shape)
shape = loaded_weight.shape
loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:])
loaded_weight = loaded_weight.transpose(0, 1)
loaded_weight = loaded_weight.reshape(shape)
yield name, loaded_weight
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self, skip_substrs=["attention.bias", "attention.masked_bias"]
)
return loader.load_weights(self._repack_qkv(weights))
class GPTNeoXForCausalLM(nn.Module, SupportsPP):
+1 -15
View File
@@ -73,6 +73,7 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
@@ -650,21 +651,6 @@ class HYV3Model(nn.Module, MixtureOfExperts):
return loaded_params
def get_spec_layer_idx_from_weight_name(
config: PretrainedConfig, weight_name: str
) -> int | None:
# HYV3MTP is enabled only when num_nextn_predict_layers is greater than 1
if (
hasattr(config, "num_nextn_predict_layers")
and config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
return layer_idx + i
return None
class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
+6 -2
View File
@@ -51,8 +51,12 @@ from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm.v1.sample.sampler import Sampler
from .hy_v3 import HYV3DecoderLayer, get_spec_layer_idx_from_weight_name
from .utils import is_pp_missing_parameter, maybe_prefix
from .hy_v3 import HYV3DecoderLayer
from .utils import (
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
maybe_prefix,
)
def _is_moe(config: PretrainedConfig) -> bool:
+3 -9
View File
@@ -36,8 +36,8 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .utils import AutoWeightsLoader
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
NORM2FN = {
@@ -445,11 +445,5 @@ class InternVisionModel(nn.Module):
return encoder_outputs
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+4 -9
View File
@@ -20,7 +20,8 @@ from vllm.model_executor.layers.conv import Conv2dLayer
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .utils import AutoWeightsLoader
NORM2FN = {
"rms_norm": RMSNorm,
@@ -433,11 +434,5 @@ class InternS1VisionModel(nn.Module):
return encoder_outputs
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
+9 -90
View File
@@ -14,10 +14,7 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.distributed.parallel_state import get_pp_group
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -38,7 +35,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.llama import LlamaMLP as JambaMLP
from vllm.sequence import IntermediateTensors
@@ -52,7 +48,6 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
WeightsMapper,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -378,87 +373,6 @@ class JambaModel(nn.Module):
hidden_states, _ = self.final_layernorm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for (
param_name,
weight_name,
expert_id,
shard_id,
) in expert_params_mapping:
if weight_name not in name:
continue
if is_pp_missing_parameter(name, self):
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class JambaForCausalLM(
nn.Module,
@@ -470,6 +384,14 @@ class JambaForCausalLM(
):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={".self_attn.": ".", ".A_log": ".A"},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".feed_forward.gate_proj": (".feed_forward.gate_up_proj", 0),
".feed_forward.up_proj": (".feed_forward.gate_up_proj", 1),
},
)
packed_modules_mapping = {
"qkv_proj": [
@@ -577,9 +499,6 @@ class JambaForCausalLM(
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
class JambaForSequenceClassification(JambaForCausalLM):
is_pooling_model = True
+1 -13
View File
@@ -52,6 +52,7 @@ from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_layers,
maybe_prefix,
@@ -643,16 +644,3 @@ class KimiLinearForCausalLM(
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
return loader.load_weights(weights)
def get_spec_layer_idx_from_weight_name(
config: KimiLinearConfig, weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
return layer_idx + i
return None
+39 -168
View File
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Laguna model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
import torch
@@ -20,10 +19,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -38,10 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
SupportsEagle3,
@@ -51,8 +43,8 @@ from vllm.model_executor.models.interfaces import (
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -575,6 +567,14 @@ class LagunaDecoderLayer(nn.Module):
@support_torch_compile
class LagunaModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -675,167 +675,41 @@ class LagunaModel(nn.Module, EagleModelMixin):
return hidden_states, aux_hidden_states
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
"""Get expert parameter mapping for weight loading.
Returns mapping tuples of (param_name, weight_name, expert_id, shard_id)
that handle both weights and quantization scales.
"""
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
# gate_proj and up_proj are loaded as separate Linears (see
# LagunaMLP) so no merge entry is needed here.
]
# Suffixes to skip for GPTQ/modelopt models if param doesn't exist
ignore_suffixes = (
".bias",
"_bias",
".k_scale",
"_k_scale",
".v_scale",
"_v_scale",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
def _slice_sink(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
for name, loaded_weight in weights:
# Handle attention sinks (distributed across ranks). Derive the
# per-rank slice from the parameter's own shape so per-layer
# variations in head count are handled correctly.
if "sink" in name:
param = params_dict.get(name)
if param is not None:
layer_heads_per_rank = param.shape[0]
layer_head_start = tp_rank * layer_heads_per_rank
narrow_weight = loaded_weight.narrow(
0, layer_head_start, layer_heads_per_rank
)
param.data.copy_(narrow_weight)
loaded_params.add(name)
continue
heads_per_rank = loaded_weight.shape[0] // tp_size
loaded_weight = loaded_weight.narrow(
0, tp_rank * heads_per_rank, heads_per_rank
)
yield name, loaded_weight
# Handle stacked params (QKV, gate_up for
# non-expert layers and shared_expert)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
# Skip expert weights - handled below via expert_params_mapping
if "mlp.experts" in name and "shared_expert" not in name:
continue
name = name.replace(weight_name, param_name)
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
# Remap FP8 kv_scale names for backwards compatibility
if name.endswith("scale"):
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name)
break
else:
# Try expert params mapping (handles weights + quantization scales)
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Mark as expert weight so we skip regular loading below
is_expert_weight = True
# Create mapped name without modifying original
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
if (
name_mapped.endswith(ignore_suffixes)
and name_mapped not in params_dict
):
continue
if name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
# Use return_success to handle expert parallelism correctly
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_params.add(name_mapped)
break
else:
# Expert weight not mapped to this rank - skip
if is_expert_weight:
continue
# Remap kv_scale names before the ignore_suffixes filter:
# the suffix list includes .k_scale/.v_scale, so filtering
# first drops the checkpoint key before remap can rewrite
# it to the .attn.* name that exists in params_dict.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=[
".bias",
"_bias",
".k_scale",
"_k_scale",
".v_scale",
"_v_scale",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
],
)
return loader.load_weights(
self._slice_sink(weights), mapper=self.hf_to_vllm_mapper
)
class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
@@ -892,9 +766,6 @@ class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3):
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
+20 -107
View File
@@ -15,10 +15,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -40,10 +37,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_moe_expert_param_name,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig
@@ -60,7 +53,6 @@ from .utils import (
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -167,6 +159,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
scoring_func="sigmoid",
e_score_correction_bias=self.gate.e_score_correction_bias,
routed_scaling_factor=self.routed_scaling_factor,
ckpt_names=("w1", "w2", "w3"),
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -402,6 +395,22 @@ class Lfm2MoeShortConvDecoderLayer(nn.Module):
@support_torch_compile
class Lfm2MoeModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
".conv.": ".short_conv.",
"expert_bias": "gate.e_score_correction_bias",
},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# Scoped (with trailing dots) to the dense MLP so routed experts.*.w1/w3
# (loaded by FusedMoE) are left untouched and .w1 does not match inside .w13
".feed_forward.w1.": (".feed_forward.w13.", 0),
".feed_forward.w3.": (".feed_forward.w13.", 1),
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -487,102 +496,9 @@ class Lfm2MoeModel(nn.Module):
hidden_states, _ = self.embedding_norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".w13", ".w1", 0),
(".w13", ".w3", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if "expert_bias" in name:
name = name.replace("expert_bias", "gate.e_score_correction_bias")
if ".conv." in name:
name = name.replace(".conv.", ".short_conv.", 1)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
# Use segment-boundary matching (trailing dot) to prevent
# e.g. ".w1" from matching inside ".w13" in pre-fused keys.
if weight_name + "." not in name:
continue
if ("feed_forward.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name + ".", param_name + ".")
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
name = maybe_remap_moe_expert_param_name(name, params_dict)
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self, ignore_unexpected_suffixes=[".bias", "_bias"])
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Lfm2MoeForCausalLM(
@@ -763,6 +679,3 @@ class Lfm2MoeForCausalLM(
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+1 -16
View File
@@ -46,7 +46,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -76,7 +75,6 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
WeightsMapper,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -888,20 +886,7 @@ class MolmoModel(nn.Module, SupportsQuant):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return AutoWeightsLoader(self).load_weights(weights)
def _lowest_multiple(x: int, k: int) -> int:
+1 -16
View File
@@ -51,7 +51,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -89,7 +88,6 @@ from .utils import (
WeightsMapper,
_merge_multimodal_embeddings,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -1221,20 +1219,7 @@ class Molmo2TextModel(nn.Module, SupportsQuant):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return AutoWeightsLoader(self).load_weights(weights)
def get_patches_grid_size(
+11 -101
View File
@@ -18,8 +18,7 @@
# limitations under the License.
"""Inference-only NemotronH model."""
import typing
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Iterable, Mapping
from itertools import islice
import torch
@@ -59,10 +58,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
HasInnerState,
@@ -78,7 +73,6 @@ from vllm.model_executor.models.interfaces import (
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
WeightsMapper,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -222,6 +216,7 @@ class NemotronHMoE(nn.Module):
intermediate_size=config.moe_intermediate_size,
renormalize=config.norm_topk_prob,
quant_config=quant_config,
ckpt_names=("up_proj", "down_proj", ""),
use_grouped_topk=True,
num_expert_group=config.n_group,
topk_group=config.topk_group,
@@ -680,9 +675,12 @@ class NemotronHModel(nn.Module, EagleModelMixin):
return max_experts
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Consumed by `get_moe_expert_mapping` (bitsandbytes / LoRA); the main
# weight load is self-served by each `RoutedExperts` layer. Sized to the
# MAX expert count so heterogeneous puzzle models load every expert.
if self.has_moe:
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = fused_moe_make_expert_params_mapping(
return fused_moe_make_expert_params_mapping(
# - FusedMoe.w1 (aka gate_proj) should be up_proj since that's
# what the activation is applied to
# - FusedMoe.w3 (aka up_proj) should be ignored since we're
@@ -694,102 +692,9 @@ class NemotronHModel(nn.Module, EagleModelMixin):
num_experts=self._get_max_n_routed_experts(),
num_redundant_experts=getattr(self, "num_redundant_experts", 0),
)
return expert_params_mapping
return []
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
expert_params_mapping = self.get_expert_mapping()
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "scale" in name or "zero_point" in name:
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
# Skip MTP/spec decode layers early (before stacked params mapping)
if name.startswith("mtp."):
continue
# load stacked params
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
# load other params
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
# Do not modify `name` since the loop may continue here
# Instead, create a new variable
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class NemotronHForCausalLM(
nn.Module,
@@ -809,6 +714,11 @@ class NemotronHForCausalLM(
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={"backbone": "model"},
orig_to_new_substr={"A_log": "A", "embeddings": "embed_tokens"},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
},
)
packed_modules_mapping = {
+20 -77
View File
@@ -63,16 +63,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
)
from vllm.sequence import IntermediateTensors
from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -302,6 +299,23 @@ class OlmoHybridDecoderLayer(nn.Module):
@support_torch_compile
class OlmoHybridModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
".linear_attn.q_proj": (".linear_attn.in_proj_qkvg", 0),
".linear_attn.k_proj": (".linear_attn.in_proj_qkvg", 1),
".linear_attn.v_proj": (".linear_attn.in_proj_qkvg", 2),
".linear_attn.g_proj": (".linear_attn.in_proj_qkvg", 3),
".q_conv1d": (".conv1d", 0),
".k_conv1d": (".conv1d", 1),
".v_conv1d": (".conv1d", 2),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.config = vllm_config.model_config.hf_config
@@ -359,79 +373,8 @@ class OlmoHybridModel(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
linear_attn_stacked_params_mapping = [
("in_proj_qkvg", "q_proj", 0),
("in_proj_qkvg", "k_proj", 1),
("in_proj_qkvg", "v_proj", 2),
("in_proj_qkvg", "g_proj", 3),
("conv1d", "q_conv1d", 0),
("conv1d", "k_conv1d", 1),
("conv1d", "v_conv1d", 2),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if is_pp_missing_parameter(name, self):
continue
handled = False
if "linear_attn" in name:
for (
param_name,
weight_name,
shard_id,
) in linear_attn_stacked_params_mapping:
if weight_name not in name:
continue
mapped_name = name.replace(weight_name, param_name)
if mapped_name.endswith(".bias") and (
mapped_name not in params_dict
):
continue
if mapped_name not in params_dict:
continue
param = params_dict[mapped_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
name = mapped_name
handled = True
break
else:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
handled = True
break
if not handled:
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class OlmoHybridForCausalLM(
+39 -152
View File
@@ -21,8 +21,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from typing import Any
import torch
@@ -44,10 +43,7 @@ from vllm.model_executor.layers.attention import (
Attention,
StaticSinkAttention,
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -64,10 +60,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.interfaces import (
MixtureOfExperts,
SupportsLoRA,
@@ -76,8 +68,8 @@ from vllm.model_executor.models.interfaces import (
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -1056,153 +1048,48 @@ class OpenPanguModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_attn_mlp_weight(
self,
attn_mlp_replace_mapping: list[tuple[str, str, int]],
params_dict: dict[str, Any],
weight_name: str,
loaded_weight: torch.Tensor,
loaded_params: set[str],
) -> bool:
for param_name, origin_name, shard_id in attn_mlp_replace_mapping:
if origin_name not in weight_name or (
("mlp.experts." in weight_name) and weight_name not in params_dict
):
continue
weight_name_mapped = weight_name.replace(origin_name, param_name)
if (
param_name == "fused_qkv_a_proj"
and weight_name_mapped not in params_dict
):
continue
else:
weight_name = weight_name_mapped
if weight_name.endswith(".bias") and weight_name not in params_dict:
continue
if is_pp_missing_parameter(weight_name, self):
continue
param = params_dict[weight_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(weight_name)
return True
return False
def load_expert_weight(
self,
expert_merge_mapping: list[tuple[str, str, int, str]],
params_dict: dict[str, Any],
weight_name: str,
loaded_weight: torch.Tensor,
loaded_params: set[str],
flag_dict: dict[str, bool],
) -> bool:
for mapping in expert_merge_mapping:
param_name, origin_name, expert_id, shard_id = mapping
if origin_name not in weight_name:
continue
flag_dict["is_expert_weight"] = True
weight_name_mapped = weight_name.replace(origin_name, param_name)
if is_pp_missing_parameter(weight_name_mapped, self):
continue
param = params_dict[weight_name_mapped]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
success = weight_loader(
param,
loaded_weight,
weight_name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
weight_name = weight_name_mapped
loaded_params.add(weight_name_mapped)
return True
return False
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
attn_mlp_replace_mapping = [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".fused_qkv_a_proj", ".q_a_proj", 0),
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
has_experts = hasattr(self.config, "n_routed_experts")
if has_experts:
expert_merge_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.n_routed_experts,
num_redundant_experts=self.num_redundant_experts,
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _filter_spec_layers(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
# Skip the MTP (spec-decode) layers that follow the main model's layers.
num_mtp = getattr(self.config, "num_nextn_predict_layers", 0)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if self.config.tie_word_embeddings and "lm_head.weight" in name:
continue
if (
"layers" in name
and hasattr(self.config, "num_nextn_predict_layers")
and (self.config.num_nextn_predict_layers > 0)
):
if "layers" in name and num_mtp > 0:
layer_idx = int(name.split("layers.")[-1].split(".")[0])
mtp_idx = layer_idx - self.config.num_hidden_layers
if mtp_idx >= 0 and mtp_idx < self.config.num_nextn_predict_layers:
continue # skip spec decode layers for main model
flag_dict = {"is_expert_weight": False}
if (
self.load_attn_mlp_weight(
attn_mlp_replace_mapping,
params_dict,
name,
loaded_weight,
loaded_params,
)
or has_experts
and self.load_expert_weight(
expert_merge_mapping,
params_dict,
name,
loaded_weight,
loaded_params,
flag_dict,
)
):
continue
else:
if flag_dict["is_expert_weight"]:
if 0 <= mtp_idx < num_mtp:
continue
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name.endswith("e_score_correction_bias"):
name = name.replace(
"e_score_correction_bias", "gate.e_score_correction_bias"
)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
stacked: dict[str, tuple[str, int | str]] = {
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
# The attention layout is config-dependent: MLA with a fused low-rank
# projection, MLA without it, or standard qkv. Only add the stacked
# entries whose fused target actually exists as a parameter.
param_names = list(dict(self.named_parameters()))
if any(".fused_qkv_a_proj." in n for n in param_names):
stacked[".q_a_proj"] = (".fused_qkv_a_proj", 0)
stacked[".kv_a_proj_with_mqa"] = (".fused_qkv_a_proj", 1)
if any(".qkv_proj." in n for n in param_names):
stacked[".q_proj"] = (".qkv_proj", "q")
stacked[".k_proj"] = (".qkv_proj", "k")
stacked[".v_proj"] = (".qkv_proj", "v")
mapper = WeightsMapper(
orig_to_new_substr={
"e_score_correction_bias": "gate.e_score_correction_bias",
},
orig_to_new_stacked=stacked,
)
loader = AutoWeightsLoader(self)
loaded = loader.load_weights(self._filter_spec_layers(weights), mapper=mapper)
self.post_weight_load()
return loaded_params
return loaded
def post_weight_load(self) -> None:
for name, module in self.named_modules():
+35 -182
View File
@@ -32,10 +32,7 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -49,14 +46,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -74,45 +70,17 @@ def _zero_mean_tensor(t: torch.Tensor) -> torch.Tensor:
return t - t.mean()
def _rename_and_normalize_weights(
def _normalize_expert_bias(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterator[tuple[str, torch.Tensor]]:
"""
Translate HuggingFace Param2MoE weight names to vLLM internal names
and zero-mean the expert-bias tensor so the router stays balanced.
"""Zero-mean the MoE router's per-expert score bias for load balance.
Mapping table (HF → vLLM):
model.word_embeddings.* → model.embed_tokens.*
*.attention.query_key_value.* → *.self_attn.qkv_proj.*
*.attention.dense.* → *.self_attn.o_proj.*
*.attention.query_layernorm.* → *.self_attn.q_layernorm.*
*.attention.key_layernorm.* → *.self_attn.k_layernorm.*
*.mlp.gate.expert_bias → *.mlp.gate.e_score_correction_bias
(also zero-meant for load balance)
The rename to ``e_score_correction_bias`` is done by the mapper; only the
tensor adjustment lives here, since a WeightsMapper cannot transform data.
"""
for name, w in weights:
# Embedding table
name = name.replace("model.word_embeddings.", "model.embed_tokens.")
# Fused QKV projection (HF: query_key_value → vLLM: qkv_proj)
name = name.replace(".attention.query_key_value.", ".self_attn.qkv_proj.")
# Output projection (HF: dense → vLLM: o_proj)
name = name.replace(".attention.dense.", ".self_attn.o_proj.")
# Per-head query norm
name = name.replace(".attention.query_layernorm.", ".self_attn.q_layernorm.")
# Per-head key norm
name = name.replace(".attention.key_layernorm.", ".self_attn.k_layernorm.")
# Catch any remaining .attention. → .self_attn. prefixes
# (e.g. future bias params on the projection layers)
name = name.replace(".attention.", ".self_attn.")
# Expert-score bias: rename + zero-mean
if name.endswith(".mlp.gate.expert_bias"):
name = name.replace(
".mlp.gate.expert_bias",
".mlp.gate.e_score_correction_bias",
)
if _is_expert_bias_name(name):
w = _zero_mean_tensor(w)
yield name, w
@@ -123,7 +91,8 @@ class Param2MoEAttention(nn.Module):
Notable differences from a vanilla GQA layer:
* The checkpoint fuses Q, K, V into a single ``query_key_value`` weight.
vLLM receives it already renamed to ``qkv_proj`` by the weight-name
translator and splits it during ``load_weights``.
translator and loads it directly; ``QKVParallelLinear`` splits the
fused ``[Q|K|V]`` tensor internally.
* Optional per-head RMS norms on Q and K (``use_qk_norm=True``).
"""
@@ -472,6 +441,28 @@ class Param2MoEDecoderLayer(nn.Module):
class Param2MoEModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"word_embeddings.": "embed_tokens.",
# Fused query_key_value is renamed to qkv_proj and loaded directly;
# QKVParallelLinear splits the fused [Q|K|V] tensor internally.
".attention.query_key_value.": ".self_attn.qkv_proj.",
".attention.dense.": ".self_attn.o_proj.",
".attention.query_layernorm.": ".self_attn.q_layernorm.",
".attention.key_layernorm.": ".self_attn.k_layernorm.",
# Catch-all for any remaining .attention. names (must come last).
".attention.": ".self_attn.",
".mlp.gate.expert_bias": ".mlp.gate.e_score_correction_bias",
},
orig_to_new_stacked={
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
},
)
def __init__(
self,
*,
@@ -558,147 +549,9 @@ class Param2MoEModel(nn.Module):
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
"""
Custom weight loader for the inner Param2MoEModel.
Receives weights that have already been renamed/normalised by the
outer model and whose ``model.`` prefix has been stripped by
``AutoWeightsLoader``. Handles:
1. Fused QKV split (query_key_value → qkv_proj q/k/v shards).
2. gate_proj + up_proj → gate_up_proj stacking (dense + shared-exp).
3. Routed-expert weights via the fused-MoE mapping.
4. All remaining weights via their default loader.
"""
config = self.config
num_heads: int = config.num_attention_heads
num_kv_heads: int = config.num_key_value_heads
head_dim: int = config.head_dim or (config.hidden_size // num_heads)
q_split = num_heads * head_dim
kv_split = num_kv_heads * head_dim
stacked_params_mapping = [
# (vllm_param_name, ckpt_weight_name, shard_id)
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
# ------------------------------------------------------------------
# 1. Fused QKV: split into q / k / v shards for QKVParallelLinear
# ------------------------------------------------------------------
if name.endswith(".self_attn.qkv_proj.weight"):
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
q_w = loaded_weight[:q_split, :]
k_w = loaded_weight[q_split : q_split + kv_split, :]
v_w = loaded_weight[q_split + kv_split :, :]
weight_loader(param, q_w, "q")
weight_loader(param, k_w, "k")
weight_loader(param, v_w, "v")
loaded_params.add(name)
continue
# ------------------------------------------------------------------
# 2. gate_proj / up_proj → gate_up_proj (dense MLP + shared-exp.)
# ------------------------------------------------------------------
matched_stacked = False
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name: # routed experts handled below
continue
new_name = name.replace(weight_name, param_name)
if new_name.endswith(".bias") and new_name not in params_dict:
continue
if new_name not in params_dict:
continue
if is_pp_missing_parameter(new_name, self):
continue
param = params_dict[new_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(new_name)
matched_stacked = True
break
if matched_stacked:
continue
# ------------------------------------------------------------------
# 3. Routed expert weights → fused-MoE kernel layout
# ------------------------------------------------------------------
matched_expert = False
for (
param_name,
weight_name,
expert_id,
shard_id,
) in expert_params_mapping:
if weight_name not in name:
continue
new_name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(new_name, self):
continue
if new_name not in params_dict:
continue
param = params_dict[new_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
loaded_params.add(new_name)
matched_expert = True
break
if matched_expert:
continue
# ------------------------------------------------------------------
# 4. All other weights: direct load (layernorms, embed_tokens, …)
# ------------------------------------------------------------------
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
try:
weight_loader(param, loaded_weight)
except Exception as e:
raise RuntimeError(
f"[param2moe] Failed to load weight '{name}' "
f"with shape {tuple(loaded_weight.shape)} "
f"into param type {type(param).__name__}: {e}"
) from e
loaded_params.add(name)
return loaded_params
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
loader = AutoWeightsLoader(self)
return loader.load_weights(
_normalize_expert_bias(weights), mapper=self.hf_to_vllm_mapper
)
@@ -859,4 +712,4 @@ class Param2MoEForCausalLM(
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(_rename_and_normalize_weights(weights))
return loader.load_weights(weights)
+10 -9
View File
@@ -93,6 +93,7 @@ from .utils import (
extract_layer_index,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_fuse_shared_experts,
maybe_prefix,
)
@@ -261,20 +262,20 @@ class Qwen3_5Model(Qwen3NextModel):
self.aux_hidden_state_layers: tuple[int, ...] = ()
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
mapper = self.hf_to_vllm_mapper
# FSE must match construction (Qwen3NextSparseMoeBlock): reroute the
# shared expert into the extra fused slot only when AITER FSE is both
# requested and compatible with the quant spec.
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
_is_shared_expert_fse_compatible(self.quant_config)
)
if is_fse:
num_routed = self.config.num_experts
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
if "moe" in self.config.model_type:
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(self.quant_config),
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3_5ForCausalLMBase(
+11 -10
View File
@@ -41,9 +41,9 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
_merge_multimodal_embeddings,
make_empty_intermediate_tensors_factory,
maybe_fuse_shared_experts,
maybe_prefix,
)
@@ -174,17 +174,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
mapper = self.hf_to_vllm_mapper
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
_is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
weights = maybe_fuse_shared_experts(
weights,
enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(
get_current_vllm_config().quant_config
),
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
if is_fse:
num_routed = getattr(self.config, "num_experts", 0)
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@support_torch_compile(
+26 -43
View File
@@ -31,10 +31,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.multimodal.inputs import NestedTensors
from vllm.transformers_utils.config import set_default_rope_theta
from vllm.transformers_utils.repo_utils import get_hf_file_bytes
@@ -44,6 +40,7 @@ from .qwen2 import Qwen2MLP as Qwen3MLP
from .qwen3 import Qwen3ForCausalLM
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_draft_quant_config,
maybe_prefix,
process_eagle_weight,
@@ -333,6 +330,17 @@ class DFlashQwen3DecoderLayer(nn.Module):
@support_torch_compile
class DFlashQwen3Model(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={"midlayer.": "layers.0."},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def __init__(
self,
*,
@@ -624,51 +632,26 @@ class DFlashQwen3Model(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
tp_rank = get_tensor_model_parallel_rank()
def _preprocess(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
tp_size = get_tensor_model_parallel_world_size()
tp_rank = get_tensor_model_parallel_rank()
for name, loaded_weight in weights:
if "midlayer." in name:
name = name.replace("midlayer.", "layers.0.")
if "scale" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if "attention_sink_bias" in name:
if name not in params_dict:
continue
# Sink bias is per-head; shard it across TP ranks like the
# attention heads themselves.
param = params_dict[name]
heads_per_rank = loaded_weight.shape[0] // tp_size
head_start = tp_rank * heads_per_rank
narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank)
param.data.copy_(narrow_weight)
loaded_params.add(name)
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loaded_weight = loaded_weight.narrow(
0, tp_rank * heads_per_rank, heads_per_rank
)
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(
self._preprocess(weights), mapper=self.hf_to_vllm_mapper
)
class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
+8 -10
View File
@@ -71,6 +71,7 @@ from .utils import (
extract_layer_index,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_fuse_shared_experts,
maybe_prefix,
)
@@ -713,17 +714,14 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
mapper = self.hf_to_vllm_mapper
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
# AITER fused-shared-experts: route the shared_expert checkpoint
# weights into the extra fused expert slot. Merge (not mutate) so the
# shared class mapper isn't permanently altered.
num_routed = getattr(self.config, "num_experts", 0)
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
weights = maybe_fuse_shared_experts(
weights,
n_routed_experts=getattr(self.config, "num_experts", 0),
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class QwenNextMixtureOfExperts(MixtureOfExperts):
+8 -11
View File
@@ -7,7 +7,6 @@ from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_pp_group
@@ -30,8 +29,8 @@ from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from .utils import (
AutoWeightsLoader,
WeightsMapper,
make_empty_intermediate_tensors_factory,
maybe_fuse_shared_experts,
maybe_prefix,
)
@@ -154,16 +153,14 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
mapper = self.hf_to_vllm_mapper
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
# AITER fused-shared-experts: route the shared_expert checkpoint
# weights into the extra fused expert slot.
num_routed = self.config.num_experts
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
weights = maybe_fuse_shared_experts(
weights,
n_routed_experts=self.config.num_experts,
n_shared_experts=1,
ckpt_prefix="mlp.shared_expert",
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@support_torch_compile
+16 -97
View File
@@ -35,11 +35,7 @@ from vllm.distributed import (
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
@@ -55,7 +51,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .bailing_moe import BailingMoeForCausalLM
@@ -63,7 +58,7 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -448,6 +443,16 @@ class SarvamMLABlock(nn.Module):
class SarvamMLAModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
)
def __init__(
self,
*,
@@ -532,97 +537,14 @@ class SarvamMLAModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
def load_weights(
self,
weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
"""Load weights with stacked gate+up and MoE expert remapping."""
weights = _normalized_weights(weights)
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
new_name = name.replace(weight_name, param_name)
if new_name.endswith(".bias") and new_name not in params_dict:
continue
if new_name not in params_dict:
continue
if is_pp_missing_parameter(new_name, self):
continue
param = params_dict[new_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(new_name)
break
else:
mapped = False
for (
param_name,
weight_name,
expert_id,
shard_id,
) in expert_params_mapping:
if weight_name not in name:
continue
new_name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(new_name, self):
continue
if new_name not in params_dict:
continue
param = params_dict[new_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
loaded_params.add(new_name)
mapped = True
break
if mapped:
continue
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(
_normalized_weights(weights), mapper=self.hf_to_vllm_mapper
)
class SarvamMixtureOfExperts(MixtureOfExperts):
@@ -764,9 +686,6 @@ class SarvamMLAForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SarvamMixtureOfE
)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
class SarvamMoEForCausalLM(BailingMoeForCausalLM):
"""Same as BailingMoeForCausalLM, but normalizes gate expert_bias pre-load."""
+1 -15
View File
@@ -53,6 +53,7 @@ from .utils import (
PPMissingLayer,
WeightsMapper,
extract_layer_index,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
@@ -978,18 +979,3 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_spec_layer_idx_from_weight_name(
config: ModelConfig, weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
layer_idx = config.num_hidden_layers
for i in range(config.num_nextn_predict_layers):
if weight_name.startswith(
f"layers.{layer_idx + i}." # Step3p5Model
) or weight_name.startswith(f"model.layers.{layer_idx + i}."): # Step3p5MTP
return layer_idx + i
return None
+2 -2
View File
@@ -18,8 +18,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .step3p5 import Step3p5DecoderLayer, get_spec_layer_idx_from_weight_name
from .utils import maybe_prefix
from .step3p5 import Step3p5DecoderLayer
from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix
logger = init_logger(__name__)
+34 -58
View File
@@ -26,19 +26,23 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.llama import LlamaForCausalLM, LlamaModel
from .llama import LlamaDecoderLayer
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
is_pp_missing_parameter,
)
from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper
class TeleChat2Model(LlamaModel):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".query": (".qkv_proj", "q"),
".key": (".qkv_proj", "k"),
".value": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
hf_config = vllm_config.model_config.hf_config
@@ -65,62 +69,34 @@ class TeleChat2Model(LlamaModel):
layer.mlp.gate_up_proj.bias = None
layer.mlp.gate_up_proj.skip_bias_add = True
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
def _split_key_value(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
# TeleChat2 stores k/v as a single per-head-interleaved `key_value`
# tensor. De-interleave it into separate k/v so the qkv_proj mapper
# can stack them.
total_num_heads = self.config.n_head
head_dim = self.config.hidden_size // total_num_heads
for name, loaded_weight in weights:
if "self_attn.key_value" in name:
k_weight = []
v_weight = []
for i in range(total_num_heads):
start = i * head_dim * 2
k_weight.append(loaded_weight[start : start + head_dim, :])
v_weight.append(
loaded_weight[start + head_dim : start + 2 * head_dim :]
)
k_weight = torch.cat(k_weight, dim=0)
v_weight = torch.cat(v_weight, dim=0)
name = name.replace("key_value", "qkv_proj")
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, k_weight, "k")
weight_loader(param, v_weight, "v")
elif "query" in name:
name = name.replace("query", "qkv_proj")
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, "q")
starts = [i * head_dim * 2 for i in range(total_num_heads)]
k_weight = torch.cat(
[loaded_weight[s : s + head_dim, :] for s in starts], dim=0
)
v_weight = torch.cat(
[loaded_weight[s + head_dim : s + 2 * head_dim, :] for s in starts],
dim=0,
)
yield name.replace("key_value", "key"), k_weight
yield name.replace("key_value", "value"), v_weight
else:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(
self._split_key_value(weights), mapper=self.hf_to_vllm_mapper
)
class TeleChat2ForCausalLM(LlamaForCausalLM):
+111
View File
@@ -35,6 +35,7 @@ if TYPE_CHECKING:
from transformers import PretrainedConfig
from transformers.conversion_mapping import WeightRenaming
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
@@ -424,6 +425,116 @@ class AutoWeightsLoader:
return autoloaded_weights
def maybe_fuse_shared_experts(
weights: Iterable[tuple[str, torch.Tensor]],
*,
n_routed_experts: int,
n_shared_experts: int,
ckpt_prefix: str = "mlp.shared_experts",
enabled: bool | None = None,
) -> Iterable[tuple[str, torch.Tensor]]:
"""Route AITER fused-shared-expert checkpoint weights into fused slots.
When AITER fused-shared-experts is active, shared experts are packed into
the routed expert tensor. The checkpoint stores them under `ckpt_prefix`
as a single (possibly widened) tensor; this splits it into
`n_shared_experts` chunks named `mlp.experts.{n_routed_experts + j}` so
the `RoutedExperts` loader treats them as extra experts. Yields the input
unchanged when the fusion is inactive, so callers can wrap unconditionally.
Args:
weights: Iterable of `(name, tensor)` checkpoint pairs.
n_routed_experts: Number of routed experts; offsets the fused slots.
n_shared_experts: Number of shared experts packed into the tensor.
ckpt_prefix: Checkpoint module name of the shared experts.
enabled: Whether AITER fused-shared-experts is active. Defaults to
`rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()`; pass an
explicit value only when the model gates on something more (e.g.
quant-spec compatibility) and it must match its construction-time
decision.
Yields:
`(name, tensor)` pairs with shared experts routed to fused slots.
"""
if enabled is None:
from vllm._aiter_ops import rocm_aiter_ops
enabled = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
if not enabled:
yield from weights
return
# Match on the dotted boundary so e.g. "mlp.shared_expert." does not also
# catch a sibling "mlp.shared_expert_gate".
prefix = f"{ckpt_prefix}."
for name, loaded_weight in weights:
if prefix not in name:
yield name, loaded_weight
continue
# gate/up split on the output dim; down_proj on the input dim.
split_dim = 1 if ("down_proj.weight" in name and loaded_weight.ndim > 1) else 0
total = loaded_weight.shape[split_dim]
if total % n_shared_experts != 0:
raise ValueError(
f"FSE shared-expert weight {name!r} has size {total} along axis "
f"{split_dim}, not divisible by n_shared_experts={n_shared_experts}."
)
chunk = total // n_shared_experts
for j in range(n_shared_experts):
sl = slice(j * chunk, (j + 1) * chunk)
if loaded_weight.ndim == 1:
chunk_weight = loaded_weight[sl]
elif split_dim == 0:
chunk_weight = loaded_weight[sl, :]
else:
chunk_weight = loaded_weight[:, sl]
yield (
name.replace(prefix, f"mlp.experts.{n_routed_experts + j}."),
chunk_weight,
)
def get_spec_layer_idx_from_weight_name(
config: "ModelConfig", weight_name: str
) -> int | None:
"""Return the MTP layer index a weight belongs to, or None.
Args:
config: The model config; must expose `num_hidden_layers` and, for MTP
checkpoints, `num_nextn_predict_layers`.
weight_name: Checkpoint weight name to classify.
Returns:
The absolute layer index for an MTP-layer weight, else None.
"""
if not (n := getattr(config, "num_nextn_predict_layers", 0)):
return None
base = config.num_hidden_layers
for i in range(n):
if weight_name.startswith((f"model.layers.{base + i}.", f"layers.{base + i}.")):
return base + i
return None
def skip_spec_layers(
weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig"
) -> Iterable[tuple[str, torch.Tensor]]:
"""Drop MTP spec-layer weights (loaded by the MTP head, not the base model).
Args:
weights: Iterable of `(name, tensor)` checkpoint pairs.
config: The model config, passed to `get_spec_layer_idx_from_weight_name`.
Yields:
`(name, tensor)` pairs whose weight is not an MTP-layer weight.
"""
return (
(name, w)
for name, w in weights
if get_spec_layer_idx_from_weight_name(config, name) is None
)
def init_vllm_registered_model(
vllm_config: VllmConfig,
*,