diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index a465c6b563..7818200dff 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -79,6 +79,7 @@ from .interfaces import MixtureOfExperts, SupportsEagle, SupportsLoRA, SupportsP from .utils import ( AutoWeightsLoader, PPMissingLayer, + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, @@ -1143,16 +1144,3 @@ class AXK1ForCausalLM( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) - - -def get_spec_layer_idx_from_weight_name( - config: AXK1Config, weight_name: str -) -> int | None: - if config.num_nextn_predict_layers and config.num_nextn_predict_layers > 0: - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if weight_name.startswith( - f"model.layers.{layer_idx + i}." - ) or weight_name.startswith(f"layers.{layer_idx + i}."): - return layer_idx + i - return None diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index 6b72388342..5117f541cf 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -11,21 +11,13 @@ from transformers.models.aria.processing_aria import AriaProcessor from vllm.config import VllmConfig from vllm.config.multimodal import BaseDummyOptions -from vllm.distributed import get_tensor_model_parallel_rank from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - RoutedExperts, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, @@ -51,7 +43,6 @@ from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel from .utils import ( AutoWeightsLoader, WeightsMapper, - is_pp_missing_parameter, maybe_prefix, ) @@ -94,34 +85,18 @@ class AriaVisionTransformer(Idefics3VisionTransformer, SupportsQuant): # Identity layer self.post_layernorm = nn.Identity() - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - # NOTE: post_layernorm is not used in Aria - if "post_layernorm" in name: - continue + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # NOTE: post_layernorm is not used in Aria. + loader = AutoWeightsLoader(self, skip_substrs=["post_layernorm"]) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class AriaProjectorMLP(nn.Module): @@ -217,39 +192,6 @@ class AriaProjector(nn.Module): return out -class AriaRoutedExperts(RoutedExperts): - def weight_loader( - self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str - ) -> None: - # Override the weight_loader to handle the expert weights in the Aria - # model, which are already packed with experts, and merge the gate and - # up weights for each expert. - # Note: Loading expert weights with quantization is not supported - tp_rank = get_tensor_model_parallel_rank() - tp_size = self.moe_config.tp_size - if shard_id == "w13": - # the shape of loaded_weight is - # (num_experts, hidden_size, 2 * moe_intermediate_size) - if tp_size > 1: - up, gate = loaded_weight.chunk(2, dim=-1) - up_current_rank = up.chunk(tp_size, dim=-1)[tp_rank] - gate_current_rank = gate.chunk(tp_size, dim=-1)[tp_rank] - up_and_gate = torch.cat( - [up_current_rank, gate_current_rank], dim=-1 - ).transpose(1, 2) - param.data.copy_(up_and_gate) - else: - param.data.copy_(loaded_weight.transpose(1, 2)) - elif shard_id == "w2": - # the shape of loaded_weight is - # (num_experts, moe_intermediate_size, hidden_size) - if tp_size > 1: - down_current_rank = loaded_weight.chunk(tp_size, dim=1)[tp_rank] - param.data.copy_(down_current_rank.transpose(1, 2)) - else: - param.data.copy_(loaded_weight.transpose(1, 2)) - - class AriaTextMoELayer(nn.Module): """ Mixture of Experts (MoE) Layer for the AriaMoE model. @@ -288,7 +230,6 @@ class AriaTextMoELayer(nn.Module): intermediate_size=config.intermediate_size, quant_config=quant_config, prefix=f"{prefix}.experts", - routed_experts_cls=AriaRoutedExperts, ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -335,70 +276,23 @@ class AriaTextModel(LlamaModel, SupportsQuant): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], "gate_up_proj": ["gate_proj", "up_proj"], - "experts.routed_experts.w13_weight": ["experts.fc1.weight"], - "experts.routed_experts.w2_weight": ["experts.fc2.weight"], } + # Aria packs all experts into single (transposed) fc1/fc2 tensors, which is + # exactly the pre-fused checkpoint layout FusedMoE self-loads once fc1/fc2 + # are renamed to the fused gate_up_proj/down_proj names. + hf_to_vllm_mapper = LlamaModel.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_substr={ + "experts.fc1": "experts.gate_up_proj", + "experts.fc2": "experts.down_proj", + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__( vllm_config=vllm_config, prefix=prefix, layer_type=AriaTextDecoderLayer ) - # Adapted from LlamaModel.load_weights with the modification of adding - # the expert weights mapping to `stacked_params_mapping` - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ("experts.routed_experts.w13_weight", "experts.fc1.weight", "w13"), - ("experts.routed_experts.w2_weight", "experts.fc2.weight", "w2"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class AriaProcessingInfo(BaseProcessingInfo): def get_hf_config(self): diff --git a/vllm/model_executor/models/bailing_moe.py b/vllm/model_executor/models/bailing_moe.py index a45d0ca81d..642d07ee65 100644 --- a/vllm/model_executor/models/bailing_moe.py +++ b/vllm/model_executor/models/bailing_moe.py @@ -41,10 +41,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -58,14 +55,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -379,6 +375,16 @@ class BailingMoeBlock(nn.Module): @support_torch_compile class BailingMoeModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + ) + def __init__( self, *, @@ -468,89 +474,9 @@ class BailingMoeModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if ( - hasattr(self.config, "norm_head") - and self.config.norm_head - and "lm_head.weight" in name - ): - loaded_weight = F.normalize(loaded_weight, dim=0, p=2, eps=1e-7) - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "mlp.experts" in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): @@ -622,15 +548,21 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): logits = self.logits_processor(self.lm_head, hidden_states) return logits + def _normalize_lm_head( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + norm_head = getattr(self.config, "norm_head", False) + for name, loaded_weight in weights: + if norm_head and "lm_head.weight" in name: + loaded_weight = F.normalize(loaded_weight, dim=0, p=2, eps=1e-7) + yield name, loaded_weight + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader( self, skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None), ) - return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() + return loader.load_weights(self._normalize_lm_head(weights)) class BailingMoeV2ForCausalLM(BailingMoeForCausalLM): diff --git a/vllm/model_executor/models/bloom.py b/vllm/model_executor/models/bloom.py index 233028a905..cdcf82f385 100644 --- a/vllm/model_executor/models/bloom.py +++ b/vllm/model_executor/models/bloom.py @@ -47,13 +47,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import SupportsPP, SupportsQuant from .utils import ( AutoWeightsLoader, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -297,36 +295,23 @@ class BloomModel(nn.Module): hidden_states = self.ln_f(hidden_states) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() + def _repack_qkv( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + # BLOOM's fused QKV is laid out as (num_heads * 3 * head_size) on its + # output dim (0), while vLLM expects (3 * num_heads * head_size). + num_heads = self.config.num_attention_heads for name, loaded_weight in weights: - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - if "query_key_value" in name: - # NOTE: BLOOM's fused QKV's output_dim has the shape of - # (num_heads * 3 * head_size), while the - # required shape is (3 * num_heads * head_size). - # Thus, we need weight conversion. - output_dim = getattr(param, "output_dim", None) - num_heads = self.config.num_attention_heads - if output_dim is not None: - loaded_weight_shape = loaded_weight.shape - loaded_weight = loaded_weight.view( - loaded_weight_shape[:output_dim] - + (num_heads, 3, -1) - + loaded_weight_shape[output_dim + 1 :] - ) - loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1) - loaded_weight = loaded_weight.reshape(loaded_weight_shape) + shape = loaded_weight.shape + loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:]) + loaded_weight = loaded_weight.transpose(0, 1) + loaded_weight = loaded_weight.reshape(shape) + yield name, loaded_weight - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(self._repack_qkv(weights)) class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant): diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index 80dc680206..b993247ec6 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -16,9 +16,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, QKVParallelLinear, diff --git a/vllm/model_executor/models/deepencoder.py b/vllm/model_executor/models/deepencoder.py index 68c101460d..fffd9382dd 100644 --- a/vllm/model_executor/models/deepencoder.py +++ b/vllm/model_executor/models/deepencoder.py @@ -22,9 +22,9 @@ from vllm.model_executor.custom_op import PluggableLayer from vllm.model_executor.layers.attention import MMEncoderAttention from vllm.model_executor.layers.conv import Conv2dLayer from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from .clip import CLIPEncoder, CLIPVisionEmbeddings +from .utils import AutoWeightsLoader class MLPBlock(nn.Module): @@ -671,12 +671,5 @@ class DeepCLIPVisionTransformer(nn.Module): return encoder_outputs def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index a5dd6ae8f9..3a0c21fe7d 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -33,9 +33,12 @@ from .deepseek_v2 import ( DeepseekV2MixtureOfExperts, DeepseekV2MoE, _try_load_fp8_indexer_wk, - get_spec_layer_idx_from_weight_name, ) -from .utils import get_pp_missing_layer_names, maybe_prefix +from .utils import ( + get_pp_missing_layer_names, + get_spec_layer_idx_from_weight_name, + maybe_prefix, +) def _restore_full_token_layout_if_needed( diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index ac187151a0..daf4ea3de9 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -108,6 +108,7 @@ from .interfaces import ( from .utils import ( PPMissingLayer, get_pp_missing_layer_names, + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, @@ -1918,21 +1919,3 @@ class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM): class GlmMoeDsaForCausalLM(DeepseekV2ForCausalLM): pass - - -# Compatibility with -# https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/blob/main/configuration_deepseek.py -def get_spec_layer_idx_from_weight_name( - config: DeepseekV2Config | DeepseekV3Config, weight_name: str -) -> int | None: - if ( - hasattr(config, "num_nextn_predict_layers") - and config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if weight_name.startswith( - f"model.layers.{layer_idx + i}." - ) or weight_name.startswith(f"layers.{layer_idx + i}."): - return layer_idx + i - return None diff --git a/vllm/model_executor/models/ernie45_moe.py b/vllm/model_executor/models/ernie45_moe.py index c2d9f92a66..fea390ca21 100644 --- a/vllm/model_executor/models/ernie45_moe.py +++ b/vllm/model_executor/models/ernie45_moe.py @@ -23,8 +23,7 @@ # limitations under the License. """Inference-only ErineMoE model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice from typing import Any @@ -42,11 +41,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - MoERunner, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -61,10 +56,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.config import set_default_rope_theta @@ -72,8 +63,8 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -407,6 +398,19 @@ class Ernie4_5_MoeDecoderLayer(nn.Module): @support_torch_compile class Ernie4_5_MoeModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -486,132 +490,26 @@ class Ernie4_5_MoeModel(nn.Module): return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.moe_num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() + def _preprocess( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: for name, loaded_weight in weights: - if self.config.tie_word_embeddings and name.endswith("lm_head.weight"): - continue - # MTP will be supported soon. - if "mtp" in name: - continue - + # moe_statics.e_score_correction_bias is stored with a leading + # singleton dim and under a different module name. if "e_score_correction_bias" in name: name = name.replace("moe_statics", "gate") loaded_weight = loaded_weight.squeeze(0) + yield name, loaded_weight - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - - if ("mlp.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - - if weight_name not in name: - continue - - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name_mapped, self): - continue - - # Skip loading extra bias for GPTQ models. - if ( - name_mapped.endswith(".bias") or name_mapped.endswith("_bias") - ) and name_mapped not in params_dict: - continue - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_substrs=["mtp"], + ignore_unexpected_suffixes=[".bias", "_bias"], + ) + return loader.load_weights( + self._preprocess(weights), mapper=self.hf_to_vllm_mapper + ) class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExperts): @@ -741,6 +639,3 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), ) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/funaudiochat.py b/vllm/model_executor/models/funaudiochat.py index 7e7cdcd822..72b12e26b5 100644 --- a/vllm/model_executor/models/funaudiochat.py +++ b/vllm/model_executor/models/funaudiochat.py @@ -30,7 +30,6 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, @@ -53,7 +52,12 @@ from vllm.sequence import IntermediateTensors from vllm.utils.import_utils import _has_module from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP -from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix +from .utils import ( + AutoWeightsLoader, + WeightsMapper, + init_vllm_registered_model, + maybe_prefix, +) class _SinusoidsPositionEmbedding(nn.Module): @@ -79,6 +83,14 @@ class _SinusoidsPositionEmbedding(nn.Module): class FunAudioChatAudioAttention(nn.Module): """Multi-headed attention used inside the continuous audio tower.""" + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__(self, config: Any): super().__init__() self.embed_dim = int(config.d_model) @@ -123,42 +135,15 @@ class FunAudioChatAudioAttention(nn.Module): bias=True, ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(self.named_parameters()) with torch.no_grad(): if self.qkv_proj.bias is not None: # HF FunAudioChat uses bias=False for k_proj. Ensure the missing # shard starts as zeros, while allowing q/v shards to load. self.qkv_proj.bias.zero_() - loaded_params: set[str] = set() - for name, loaded_weight in weights: - for param_name, shard_name, shard_id in stacked_params_mapping: - if shard_name not in name: - continue - name = name.replace(shard_name, param_name) - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - loaded_params.add(name) - - return loaded_params + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def forward( self, diff --git a/vllm/model_executor/models/gemma3n.py b/vllm/model_executor/models/gemma3n.py index ad8b21d86b..4b06fd418f 100644 --- a/vllm/model_executor/models/gemma3n.py +++ b/vllm/model_executor/models/gemma3n.py @@ -44,18 +44,14 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from vllm.v1.attention.backends.utils import KVSharingFastPrefillMetadata from .interfaces import SupportsQuant from .utils import ( AutoWeightsLoader, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_layers, maybe_prefix, ) @@ -788,6 +784,25 @@ class Gemma3nCrossDecoder(nn.Module): enable_if=lambda vllm_config: not vllm_config.cache_config.kv_sharing_fast_prefill ) class Gemma3nTextModel(nn.Module, SupportsQuant): + # Decoder layers, altup_unembed_projections and norm live on the text + # model; every other submodule lives under self_decoder. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={ + "embed_tokens.": "self_decoder.embed_tokens.", + "embed_tokens_per_layer.": "self_decoder.embed_tokens_per_layer.", + "per_layer_model_projection.": "self_decoder.per_layer_model_projection.", + "per_layer_projection_norm.": "self_decoder.per_layer_projection_norm.", + "altup_projections.": "self_decoder.altup_projections.", + }, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config @@ -1036,58 +1051,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant): return self.norm(hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - # decoder layer weights, altup_unembed_projections and rmsnorm - # are initialized in text model, others are in self decoder - if ( - not name.startswith("layers") - and not name.startswith("altup_unembed_projections") - and not name.startswith("norm") - ): - name = f"self_decoder.{name}" - - for param_name, shard_name, shard_id in stacked_params_mapping: - if shard_name not in name: - continue - # Avoid spurious match with ".up_proj". - if "altup_projections" in name: - continue - name = name.replace(shard_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Gemma3nForCausalLM(nn.Module): diff --git a/vllm/model_executor/models/glm4.py b/vllm/model_executor/models/glm4.py index 3a25f90ad2..a1fb94fb26 100644 --- a/vllm/model_executor/models/glm4.py +++ b/vllm/model_executor/models/glm4.py @@ -303,16 +303,3 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): ] loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) return loader.load_weights(weights) - - -def get_spec_layer_idx_from_weight_name( - config: Glm4Config, weight_name: str -) -> int | None: - if hasattr(config, "num_nextn_predict_layers") and ( - config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if f"layers.{layer_idx + i}." in weight_name: - return layer_idx + i - return None diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index e3f94c673f..abb9970c40 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -24,8 +24,7 @@ """Inference-only GLM-4.5, GLM-4.6, GLM-4.7 model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice import torch @@ -43,10 +42,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -60,20 +56,18 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, + maybe_fuse_shared_experts, maybe_prefix, + skip_spec_layers, ) logger = init_logger(__name__) @@ -410,6 +404,19 @@ class Glm4MoeDecoderLayer(nn.Module): } ) class Glm4MoeModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -480,189 +487,14 @@ class Glm4MoeModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - # FSE widens the mapping by n_shared_experts slots; see deepseek_v2.py. - num_experts = self.config.n_routed_experts - if ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - and self.config.n_shared_experts - ): - num_experts += self.config.n_shared_experts - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=num_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - rocm_aiter_moe_shared_expert_enabled = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + weights = maybe_fuse_shared_experts( + skip_spec_layers(weights, self.config), + n_routed_experts=self.config.n_routed_experts, + n_shared_experts=self.config.n_shared_experts or 1, ) - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) - if spec_layer is not None: - continue - - is_fusion_moe_shared_experts_layer = ( - rocm_aiter_moe_shared_expert_enabled and ("mlp.shared_experts" in name) - ) - - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - if is_fusion_moe_shared_experts_layer: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - if weight_loader == default_weight_loader: - weight_loader(param, loaded_weight) - else: - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - - # FSE: split a widened mlp.shared_experts tensor into - # n_shared_experts chunks; see deepseek_v2.py for details. - num_chunks = 1 - split_dim = 0 - chunk_size = 0 - if is_fusion_moe_shared_experts_layer: - num_chunks = getattr(self.config, "n_shared_experts", 1) or 1 - split_dim = ( - 1 - if ("down_proj.weight" in name and loaded_weight.ndim > 1) - else 0 - ) - total = loaded_weight.shape[split_dim] - if total % num_chunks != 0: - raise ValueError( - f"FSE shared-expert weight {name} has dim " - f"{total} along axis {split_dim} which is not " - f"divisible by n_shared_experts={num_chunks}." - ) - chunk_size = total // num_chunks - - for j in range(num_chunks): - chunk_name = name - weight_to_load = loaded_weight - - if is_fusion_moe_shared_experts_layer: - chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size) - if loaded_weight.ndim == 1: - weight_to_load = loaded_weight[chunk_slice] - elif split_dim == 0: - weight_to_load = loaded_weight[chunk_slice, :] - else: - weight_to_load = loaded_weight[:, chunk_slice] - # Synthesize an expert-style name for expert mapping. - chunk_name = name.replace( - "mlp.shared_experts", - f"mlp.experts.{self.config.n_routed_experts + j}", - ) - - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in chunk_name: - continue - - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = chunk_name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - - param = params_dict[name_mapped] - # We should ask the weight loader to return success - # or not here since otherwise we may skip experts - # with other available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - weight_to_load, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - if not is_fusion_moe_shared_experts_layer: - name = name_mapped - else: - loaded_params.add(name_mapped) - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - if name is not None and not is_fusion_moe_shared_experts_layer: - loaded_params.add(name) - - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Glm4MixtureOfExperts(MixtureOfExperts): @@ -776,19 +608,3 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, Glm4MixtureOfExper def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - - -def get_spec_layer_idx_from_weight_name( - config: Glm4MoeConfig, weight_name: str -) -> int | None: - if hasattr(config, "num_nextn_predict_layers") and ( - config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if f"layers.{layer_idx + i}." in weight_name: - return layer_idx + i - return None diff --git a/vllm/model_executor/models/glm4_moe_lite.py b/vllm/model_executor/models/glm4_moe_lite.py index 432fa5e6fa..7f63130d88 100644 --- a/vllm/model_executor/models/glm4_moe_lite.py +++ b/vllm/model_executor/models/glm4_moe_lite.py @@ -70,6 +70,7 @@ from .interfaces import SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, @@ -628,16 +629,3 @@ class Glm4MoeLiteForCausalLM( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) - - -def get_spec_layer_idx_from_weight_name( - config: "Glm4MoeLiteConfig", weight_name: str -) -> int | None: - if hasattr(config, "num_nextn_predict_layers") and ( - config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if f"layers.{layer_idx + i}." in weight_name: - return layer_idx + i - return None diff --git a/vllm/model_executor/models/glm4_moe_lite_mtp.py b/vllm/model_executor/models/glm4_moe_lite_mtp.py index 222705c14e..bbede18c40 100644 --- a/vllm/model_executor/models/glm4_moe_lite_mtp.py +++ b/vllm/model_executor/models/glm4_moe_lite_mtp.py @@ -54,10 +54,9 @@ from .glm4_moe_lite import ( Glm4MixtureOfExperts, Glm4MoeLite, Glm4MoeLiteDecoderLayer, - get_spec_layer_idx_from_weight_name, ) from .interfaces import SupportsPP -from .utils import maybe_prefix +from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix class SharedHead(nn.Module): diff --git a/vllm/model_executor/models/glm4_moe_mtp.py b/vllm/model_executor/models/glm4_moe_mtp.py index 4d7b291df1..6708ba8139 100644 --- a/vllm/model_executor/models/glm4_moe_mtp.py +++ b/vllm/model_executor/models/glm4_moe_mtp.py @@ -51,9 +51,8 @@ from .glm4_moe import ( Glm4MixtureOfExperts, Glm4MoE, Glm4MoeDecoderLayer, - get_spec_layer_idx_from_weight_name, ) -from .utils import maybe_prefix +from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix class SharedHead(nn.Module): diff --git a/vllm/model_executor/models/glm_ocr_mtp.py b/vllm/model_executor/models/glm_ocr_mtp.py index 9b2369f93d..0904b44eb2 100644 --- a/vllm/model_executor/models/glm_ocr_mtp.py +++ b/vllm/model_executor/models/glm_ocr_mtp.py @@ -41,13 +41,14 @@ from vllm.model_executor.model_loader.weight_utils import ( from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from .glm4 import Glm4DecoderLayer, get_spec_layer_idx_from_weight_name +from .glm4 import Glm4DecoderLayer from .glm4_moe_lite_mtp import ( Glm4MoeLiteMultiTokenPredictor, SharedHead, ) from .interfaces import SupportsPP from .utils import ( + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, maybe_prefix, ) diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index cd168b6b46..1ff7224d7e 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -65,7 +65,12 @@ from .interfaces import ( SupportsPP, SupportsTranscription, ) -from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix +from .utils import ( + AutoWeightsLoader, + WeightsMapper, + init_vllm_registered_model, + maybe_prefix, +) from .whisper import ISO639_1_SUPPORTED_LANGS, _create_fake_bias_for_k_proj @@ -395,6 +400,14 @@ class GlmAsrEncoder(nn.Module): "qkv_proj": ["q_proj", "k_proj", "v_proj"], } + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config, @@ -496,44 +509,9 @@ class GlmAsrEncoder(nn.Module): return _GlmAsrEncoderOutput(last_hidden_state=hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Custom weight loading to handle q_proj/k_proj/v_proj -> qkv_proj mapping.""" - from vllm.model_executor.model_loader.weight_utils import default_weight_loader - weights = _create_fake_bias_for_k_proj(weights, ".k_proj.weight") - - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Default weight loading for non-stacked params - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class GlmAsrFeatureInputs(TensorSchema): diff --git a/vllm/model_executor/models/gpt2.py b/vllm/model_executor/models/gpt2.py index 31fc394653..01dc119f85 100644 --- a/vllm/model_executor/models/gpt2.py +++ b/vllm/model_executor/models/gpt2.py @@ -47,13 +47,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import SupportsCrossEncoding, SupportsPP from .utils import ( AutoWeightsLoader, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -241,32 +239,25 @@ class GPT2Model(nn.Module): hidden_states = self.ln_f(hidden_states) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() + def _transpose_conv1d( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + # HF's GPT-2 uses Conv1D instead of Linear, so its 2D weights are + # stored transposed relative to what vLLM expects. + # Note(zhuohan): the logic below might break quantized models. for name, loaded_weight in weights: - if ".attn.bias" in name or ".attn.masked_bias" in name: - # Skip attention mask. - # NOTE: "c_attn.bias" should not be skipped. - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - # The HF's GPT-2 implementation uses Conv1D instead of Linear. - # Because of this, we need to transpose the weights. - # Note(zhuohan): the logic below might break quantized models. - for conv1d_weight_name in ["c_attn", "c_proj", "c_fc"]: - if conv1d_weight_name not in name: - continue - if not name.endswith(".weight"): - continue + if name.endswith(".weight") and any( + proj in name for proj in ("c_attn", "c_proj", "c_fc") + ): loaded_weight = loaded_weight.t() - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + yield name, loaded_weight + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # Skip attention mask buffers; NOTE: "c_attn.bias" must not be skipped. + loader = AutoWeightsLoader( + self, skip_substrs=[".attn.bias", ".attn.masked_bias"] + ) + return loader.load_weights(self._transpose_conv1d(weights)) class GPT2LMHeadModel(nn.Module, SupportsPP): diff --git a/vllm/model_executor/models/gpt_neox.py b/vllm/model_executor/models/gpt_neox.py index 8d44d12fc2..907ab77760 100644 --- a/vllm/model_executor/models/gpt_neox.py +++ b/vllm/model_executor/models/gpt_neox.py @@ -43,13 +43,11 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import SupportsPP from .utils import ( AutoWeightsLoader, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -249,45 +247,25 @@ class GPTNeoXModel(nn.Module): hidden_states = self.final_layer_norm(hidden_states) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() + def _repack_qkv( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + # GPT-NeoX's fused QKV is laid out as (num_heads * 3 * head_size) on + # its output dim (0), while vLLM expects (3 * num_heads * head_size). + num_heads = self.config.num_attention_heads for name, loaded_weight in weights: - if ( - "attention.bias" in name - or "attention.masked_bias" in name - or "rotary_emb.inv_freq" in name - ): - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using OpenRLHF may include - # these tensors in the checkpoint. Skip them. - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - if "query_key_value" in name: - # NOTE: GPT-NeoX's fused QKV's output_dim has the shape of - # (num_heads * 3 * head_size), while the - # required shape is (3 * num_heads * head_size). - # Thus, we need weight conversion. - output_dim = getattr(param, "output_dim", None) - num_heads = self.config.num_attention_heads - if output_dim is not None: - loaded_weight_shape = loaded_weight.shape - loaded_weight = loaded_weight.view( - loaded_weight_shape[:output_dim] - + (num_heads, 3, -1) - + loaded_weight_shape[output_dim + 1 :] - ) - loaded_weight = loaded_weight.transpose(output_dim, output_dim + 1) - loaded_weight = loaded_weight.reshape(loaded_weight_shape) + shape = loaded_weight.shape + loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:]) + loaded_weight = loaded_weight.transpose(0, 1) + loaded_weight = loaded_weight.reshape(shape) + yield name, loaded_weight - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, skip_substrs=["attention.bias", "attention.masked_bias"] + ) + return loader.load_weights(self._repack_qkv(weights)) class GPTNeoXForCausalLM(nn.Module, SupportsPP): diff --git a/vllm/model_executor/models/hy_v3.py b/vllm/model_executor/models/hy_v3.py index 62bdd028bb..68c6f23827 100644 --- a/vllm/model_executor/models/hy_v3.py +++ b/vllm/model_executor/models/hy_v3.py @@ -73,6 +73,7 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, @@ -650,21 +651,6 @@ class HYV3Model(nn.Module, MixtureOfExperts): return loaded_params -def get_spec_layer_idx_from_weight_name( - config: PretrainedConfig, weight_name: str -) -> int | None: - # HYV3MTP is enabled only when num_nextn_predict_layers is greater than 1 - if ( - hasattr(config, "num_nextn_predict_layers") - and config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if weight_name.startswith(f"model.layers.{layer_idx + i}."): - return layer_idx + i - return None - - class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA): packed_modules_mapping = { "qkv_proj": ["q_proj", "k_proj", "v_proj"], diff --git a/vllm/model_executor/models/hy_v3_mtp.py b/vllm/model_executor/models/hy_v3_mtp.py index 77f323aae4..193280d7c8 100644 --- a/vllm/model_executor/models/hy_v3_mtp.py +++ b/vllm/model_executor/models/hy_v3_mtp.py @@ -51,8 +51,12 @@ from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata from vllm.v1.sample.sampler import Sampler -from .hy_v3 import HYV3DecoderLayer, get_spec_layer_idx_from_weight_name -from .utils import is_pp_missing_parameter, maybe_prefix +from .hy_v3 import HYV3DecoderLayer +from .utils import ( + get_spec_layer_idx_from_weight_name, + is_pp_missing_parameter, + maybe_prefix, +) def _is_moe(config: PretrainedConfig) -> bool: diff --git a/vllm/model_executor/models/intern_vit.py b/vllm/model_executor/models/intern_vit.py index 7da498bf6d..44ed3ec0fe 100644 --- a/vllm/model_executor/models/intern_vit.py +++ b/vllm/model_executor/models/intern_vit.py @@ -36,8 +36,8 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from .utils import AutoWeightsLoader from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model NORM2FN = { @@ -445,11 +445,5 @@ class InternVisionModel(nn.Module): return encoder_outputs def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) diff --git a/vllm/model_executor/models/interns1_vit.py b/vllm/model_executor/models/interns1_vit.py index 533f0681c1..a5bc781561 100644 --- a/vllm/model_executor/models/interns1_vit.py +++ b/vllm/model_executor/models/interns1_vit.py @@ -20,7 +20,8 @@ from vllm.model_executor.layers.conv import Conv2dLayer from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader + +from .utils import AutoWeightsLoader NORM2FN = { "rms_norm": RMSNorm, @@ -433,11 +434,5 @@ class InternS1VisionModel(nn.Module): return encoder_outputs def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights) diff --git a/vllm/model_executor/models/jamba.py b/vllm/model_executor/models/jamba.py index 84e96def6c..33a8c63641 100644 --- a/vllm/model_executor/models/jamba.py +++ b/vllm/model_executor/models/jamba.py @@ -14,10 +14,7 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -38,7 +35,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.llama import LlamaMLP as JambaMLP from vllm.sequence import IntermediateTensors @@ -52,7 +48,6 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, WeightsMapper, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -378,87 +373,6 @@ class JambaModel(nn.Module): hidden_states, _ = self.final_layernorm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "experts" in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for ( - param_name, - weight_name, - expert_id, - shard_id, - ) in expert_params_mapping: - if weight_name not in name: - continue - - if is_pp_missing_parameter(name, self): - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class JambaForCausalLM( nn.Module, @@ -470,6 +384,14 @@ class JambaForCausalLM( ): hf_to_vllm_mapper = WeightsMapper( orig_to_new_substr={".self_attn.": ".", ".A_log": ".A"}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".feed_forward.gate_proj": (".feed_forward.gate_up_proj", 0), + ".feed_forward.up_proj": (".feed_forward.gate_up_proj", 1), + }, ) packed_modules_mapping = { "qkv_proj": [ @@ -577,9 +499,6 @@ class JambaForCausalLM( loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - class JambaForSequenceClassification(JambaForCausalLM): is_pooling_model = True diff --git a/vllm/model_executor/models/kimi_linear.py b/vllm/model_executor/models/kimi_linear.py index 307b24ac11..057d4d01cb 100644 --- a/vllm/model_executor/models/kimi_linear.py +++ b/vllm/model_executor/models/kimi_linear.py @@ -52,6 +52,7 @@ from .interfaces import HasInnerState, IsHybrid, MixtureOfExperts, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, make_layers, maybe_prefix, @@ -643,16 +644,3 @@ class KimiLinearForCausalLM( skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), ) return loader.load_weights(weights) - - -def get_spec_layer_idx_from_weight_name( - config: KimiLinearConfig, weight_name: str -) -> int | None: - if hasattr(config, "num_nextn_predict_layers") and ( - config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if weight_name.startswith(f"model.layers.{layer_idx + i}."): - return layer_idx + i - return None diff --git a/vllm/model_executor/models/laguna.py b/vllm/model_executor/models/laguna.py index c79e8d48cb..e71054f4da 100644 --- a/vllm/model_executor/models/laguna.py +++ b/vllm/model_executor/models/laguna.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Inference-only Laguna model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice import torch @@ -20,10 +19,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -38,10 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.interfaces import ( EagleModelMixin, SupportsEagle3, @@ -51,8 +43,8 @@ from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -575,6 +567,14 @@ class LagunaDecoderLayer(nn.Module): @support_torch_compile class LagunaModel(nn.Module, EagleModelMixin): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -675,167 +675,41 @@ class LagunaModel(nn.Module, EagleModelMixin): return hidden_states, aux_hidden_states return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - """Get expert parameter mapping for weight loading. - - Returns mapping tuples of (param_name, weight_name, expert_id, shard_id) - that handle both weights and quantization scales. - """ - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - # gate_proj and up_proj are loaded as separate Linears (see - # LagunaMLP) so no merge entry is needed here. - ] - - # Suffixes to skip for GPTQ/modelopt models if param doesn't exist - ignore_suffixes = ( - ".bias", - "_bias", - ".k_scale", - "_k_scale", - ".v_scale", - "_v_scale", - ".weight_scale", - "_weight_scale", - ".input_scale", - "_input_scale", - ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - + def _slice_sink( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + tp_size = get_tensor_model_parallel_world_size() tp_rank = get_tensor_model_parallel_rank() - for name, loaded_weight in weights: # Handle attention sinks (distributed across ranks). Derive the # per-rank slice from the parameter's own shape so per-layer # variations in head count are handled correctly. if "sink" in name: - param = params_dict.get(name) - if param is not None: - layer_heads_per_rank = param.shape[0] - layer_head_start = tp_rank * layer_heads_per_rank - narrow_weight = loaded_weight.narrow( - 0, layer_head_start, layer_heads_per_rank - ) - param.data.copy_(narrow_weight) - loaded_params.add(name) - continue + heads_per_rank = loaded_weight.shape[0] // tp_size + loaded_weight = loaded_weight.narrow( + 0, tp_rank * heads_per_rank, heads_per_rank + ) + yield name, loaded_weight - # Handle stacked params (QKV, gate_up for - # non-expert layers and shared_expert) - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - # Skip expert weights - handled below via expert_params_mapping - if "mlp.experts" in name and "shared_expert" not in name: - continue - name = name.replace(weight_name, param_name) - - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - # Remap FP8 kv_scale names for backwards compatibility - if name.endswith("scale"): - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - if weight_loader == default_weight_loader: - weight_loader(param, loaded_weight) - else: - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(name) - break - else: - # Try expert params mapping (handles weights + quantization scales) - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - - # Mark as expert weight so we skip regular loading below - is_expert_weight = True - - # Create mapped name without modifying original - name_mapped = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - if ( - name_mapped.endswith(ignore_suffixes) - and name_mapped not in params_dict - ): - continue - if name_mapped not in params_dict: - continue - - param = params_dict[name_mapped] - # Use return_success to handle expert parallelism correctly - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - loaded_params.add(name_mapped) - break - else: - # Expert weight not mapped to this rank - skip - if is_expert_weight: - continue - - # Remap kv_scale names before the ignore_suffixes filter: - # the suffix list includes .k_scale/.v_scale, so filtering - # first drops the checkpoint key before remap can rewrite - # it to the .attn.* name that exists in params_dict. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=[ + ".bias", + "_bias", + ".k_scale", + "_k_scale", + ".v_scale", + "_v_scale", + ".weight_scale", + "_weight_scale", + ".input_scale", + "_input_scale", + ], + ) + return loader.load_weights( + self._slice_sink(weights), mapper=self.hf_to_vllm_mapper + ) class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3): @@ -892,9 +766,6 @@ class LagunaForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SupportsEagle3): logits = self.logits_processor(self.lm_head, hidden_states) return logits - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader( self, diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 94f7f4e289..698f1fb72e 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -15,10 +15,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -40,10 +37,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_moe_expert_param_name, -) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.lfm2_moe import Lfm2MoeConfig @@ -60,7 +53,6 @@ from .utils import ( PPMissingLayer, WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -167,6 +159,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module): scoring_func="sigmoid", e_score_correction_bias=self.gate.e_score_correction_bias, routed_scaling_factor=self.routed_scaling_factor, + ckpt_names=("w1", "w2", "w3"), ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -402,6 +395,22 @@ class Lfm2MoeShortConvDecoderLayer(nn.Module): @support_torch_compile class Lfm2MoeModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + ".conv.": ".short_conv.", + "expert_bias": "gate.e_score_correction_bias", + }, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # Scoped (with trailing dots) to the dense MLP so routed experts.*.w1/w3 + # (loaded by FusedMoE) are left untouched and .w1 does not match inside .w13 + ".feed_forward.w1.": (".feed_forward.w13.", 0), + ".feed_forward.w3.": (".feed_forward.w13.", 1), + }, + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -487,102 +496,9 @@ class Lfm2MoeModel(nn.Module): hidden_states, _ = self.embedding_norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="w1", - ckpt_down_proj_name="w2", - ckpt_up_proj_name="w3", - num_experts=self.config.num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".w13", ".w1", 0), - (".w13", ".w3", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if "expert_bias" in name: - name = name.replace("expert_bias", "gate.e_score_correction_bias") - - if ".conv." in name: - name = name.replace(".conv.", ".short_conv.", 1) - - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - # Use segment-boundary matching (trailing dot) to prevent - # e.g. ".w1" from matching inside ".w13" in pre-fused keys. - if weight_name + "." not in name: - continue - - if ("feed_forward.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name + ".", param_name + ".") - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - - if weight_name not in name: - continue - - name = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - param = params_dict[name] - - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - name = maybe_remap_moe_expert_param_name(name, params_dict) - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self, ignore_unexpected_suffixes=[".bias", "_bias"]) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Lfm2MoeForCausalLM( @@ -763,6 +679,3 @@ class Lfm2MoeForCausalLM( skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), ) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index b3279e7dbd..cf4550a107 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -46,7 +46,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -76,7 +75,6 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, WeightsMapper, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -888,20 +886,7 @@ class MolmoModel(nn.Module, SupportsQuant): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return AutoWeightsLoader(self).load_weights(weights) def _lowest_multiple(x: int, k: int) -> int: diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 22a2b0cf32..d8de4f5ed1 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -51,7 +51,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -89,7 +88,6 @@ from .utils import ( WeightsMapper, _merge_multimodal_embeddings, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -1221,20 +1219,7 @@ class Molmo2TextModel(nn.Module, SupportsQuant): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return AutoWeightsLoader(self).load_weights(weights) def get_patches_grid_size( diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index 016bff18ec..84f24094a3 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -18,8 +18,7 @@ # limitations under the License. """Inference-only NemotronH model.""" -import typing -from collections.abc import Callable, Iterable, Mapping +from collections.abc import Iterable, Mapping from itertools import islice import torch @@ -59,10 +58,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.interfaces import ( EagleModelMixin, HasInnerState, @@ -78,7 +73,6 @@ from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.utils import ( AutoWeightsLoader, WeightsMapper, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -222,6 +216,7 @@ class NemotronHMoE(nn.Module): intermediate_size=config.moe_intermediate_size, renormalize=config.norm_topk_prob, quant_config=quant_config, + ckpt_names=("up_proj", "down_proj", ""), use_grouped_topk=True, num_expert_group=config.n_group, topk_group=config.topk_group, @@ -680,9 +675,12 @@ class NemotronHModel(nn.Module, EagleModelMixin): return max_experts def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + # Consumed by `get_moe_expert_mapping` (bitsandbytes / LoRA); the main + # weight load is self-served by each `RoutedExperts` layer. Sized to the + # MAX expert count so heterogeneous puzzle models load every expert. if self.has_moe: # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = fused_moe_make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( # - FusedMoe.w1 (aka gate_proj) should be up_proj since that's # what the activation is applied to # - FusedMoe.w3 (aka up_proj) should be ignored since we're @@ -694,102 +692,9 @@ class NemotronHModel(nn.Module, EagleModelMixin): num_experts=self._get_max_n_routed_experts(), num_redundant_experts=getattr(self, "num_redundant_experts", 0), ) - return expert_params_mapping return [] - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - expert_params_mapping = self.get_expert_mapping() - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "scale" in name or "zero_point" in name: - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - # Skip MTP/spec decode layers early (before stacked params mapping) - if name.startswith("mtp."): - continue - - # load stacked params - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - - # load other params - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - - loaded_params.add(name) - return loaded_params - class NemotronHForCausalLM( nn.Module, @@ -809,6 +714,11 @@ class NemotronHForCausalLM( hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={"backbone": "model"}, orig_to_new_substr={"A_log": "A", "embeddings": "embed_tokens"}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + }, ) packed_modules_mapping = { diff --git a/vllm/model_executor/models/olmo_hybrid.py b/vllm/model_executor/models/olmo_hybrid.py index 49969cdc58..51bc410363 100644 --- a/vllm/model_executor/models/olmo_hybrid.py +++ b/vllm/model_executor/models/olmo_hybrid.py @@ -63,16 +63,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, -) from vllm.sequence import IntermediateTensors from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -302,6 +299,23 @@ class OlmoHybridDecoderLayer(nn.Module): @support_torch_compile class OlmoHybridModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".self_attn.q_proj": (".self_attn.qkv_proj", "q"), + ".self_attn.k_proj": (".self_attn.qkv_proj", "k"), + ".self_attn.v_proj": (".self_attn.qkv_proj", "v"), + ".linear_attn.q_proj": (".linear_attn.in_proj_qkvg", 0), + ".linear_attn.k_proj": (".linear_attn.in_proj_qkvg", 1), + ".linear_attn.v_proj": (".linear_attn.in_proj_qkvg", 2), + ".linear_attn.g_proj": (".linear_attn.in_proj_qkvg", 3), + ".q_conv1d": (".conv1d", 0), + ".k_conv1d": (".conv1d", 1), + ".v_conv1d": (".conv1d", 2), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() self.config = vllm_config.model_config.hf_config @@ -359,79 +373,8 @@ class OlmoHybridModel(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - linear_attn_stacked_params_mapping = [ - ("in_proj_qkvg", "q_proj", 0), - ("in_proj_qkvg", "k_proj", 1), - ("in_proj_qkvg", "v_proj", 2), - ("in_proj_qkvg", "g_proj", 3), - ("conv1d", "q_conv1d", 0), - ("conv1d", "k_conv1d", 1), - ("conv1d", "v_conv1d", 2), - ] - - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - if is_pp_missing_parameter(name, self): - continue - - handled = False - - if "linear_attn" in name: - for ( - param_name, - weight_name, - shard_id, - ) in linear_attn_stacked_params_mapping: - if weight_name not in name: - continue - mapped_name = name.replace(weight_name, param_name) - if mapped_name.endswith(".bias") and ( - mapped_name not in params_dict - ): - continue - if mapped_name not in params_dict: - continue - param = params_dict[mapped_name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - name = mapped_name - handled = True - break - else: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - handled = True - break - - if not handled: - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class OlmoHybridForCausalLM( diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index 91120840bd..fd40033e71 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -21,8 +21,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from typing import Any import torch @@ -44,10 +43,7 @@ from vllm.model_executor.layers.attention import ( Attention, StaticSinkAttention, ) -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -64,10 +60,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.interfaces import ( MixtureOfExperts, SupportsLoRA, @@ -76,8 +68,8 @@ from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -1056,153 +1048,48 @@ class OpenPanguModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def load_attn_mlp_weight( - self, - attn_mlp_replace_mapping: list[tuple[str, str, int]], - params_dict: dict[str, Any], - weight_name: str, - loaded_weight: torch.Tensor, - loaded_params: set[str], - ) -> bool: - for param_name, origin_name, shard_id in attn_mlp_replace_mapping: - if origin_name not in weight_name or ( - ("mlp.experts." in weight_name) and weight_name not in params_dict - ): - continue - weight_name_mapped = weight_name.replace(origin_name, param_name) - if ( - param_name == "fused_qkv_a_proj" - and weight_name_mapped not in params_dict - ): - continue - else: - weight_name = weight_name_mapped - if weight_name.endswith(".bias") and weight_name not in params_dict: - continue - if is_pp_missing_parameter(weight_name, self): - continue - - param = params_dict[weight_name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(weight_name) - return True - return False - - def load_expert_weight( - self, - expert_merge_mapping: list[tuple[str, str, int, str]], - params_dict: dict[str, Any], - weight_name: str, - loaded_weight: torch.Tensor, - loaded_params: set[str], - flag_dict: dict[str, bool], - ) -> bool: - for mapping in expert_merge_mapping: - param_name, origin_name, expert_id, shard_id = mapping - if origin_name not in weight_name: - continue - flag_dict["is_expert_weight"] = True - weight_name_mapped = weight_name.replace(origin_name, param_name) - if is_pp_missing_parameter(weight_name_mapped, self): - continue - param = params_dict[weight_name_mapped] - weight_loader = typing.cast(Callable[..., bool], param.weight_loader) - success = weight_loader( - param, - loaded_weight, - weight_name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - weight_name = weight_name_mapped - loaded_params.add(weight_name_mapped) - return True - return False - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - attn_mlp_replace_mapping = [ - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".fused_qkv_a_proj", ".q_a_proj", 0), - (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - has_experts = hasattr(self.config, "n_routed_experts") - if has_experts: - expert_merge_mapping = fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.n_routed_experts, - num_redundant_experts=self.num_redundant_experts, - ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() + def _filter_spec_layers( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + # Skip the MTP (spec-decode) layers that follow the main model's layers. + num_mtp = getattr(self.config, "num_nextn_predict_layers", 0) for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if self.config.tie_word_embeddings and "lm_head.weight" in name: - continue - - if ( - "layers" in name - and hasattr(self.config, "num_nextn_predict_layers") - and (self.config.num_nextn_predict_layers > 0) - ): + if "layers" in name and num_mtp > 0: layer_idx = int(name.split("layers.")[-1].split(".")[0]) mtp_idx = layer_idx - self.config.num_hidden_layers - if mtp_idx >= 0 and mtp_idx < self.config.num_nextn_predict_layers: - continue # skip spec decode layers for main model - - flag_dict = {"is_expert_weight": False} - if ( - self.load_attn_mlp_weight( - attn_mlp_replace_mapping, - params_dict, - name, - loaded_weight, - loaded_params, - ) - or has_experts - and self.load_expert_weight( - expert_merge_mapping, - params_dict, - name, - loaded_weight, - loaded_params, - flag_dict, - ) - ): - continue - else: - if flag_dict["is_expert_weight"]: + if 0 <= mtp_idx < num_mtp: continue - if name.endswith(".bias") and name not in params_dict: - continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name.endswith("e_score_correction_bias"): - name = name.replace( - "e_score_correction_bias", "gate.e_score_correction_bias" - ) - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) + yield name, loaded_weight + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + stacked: dict[str, tuple[str, int | str]] = { + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + # The attention layout is config-dependent: MLA with a fused low-rank + # projection, MLA without it, or standard qkv. Only add the stacked + # entries whose fused target actually exists as a parameter. + param_names = list(dict(self.named_parameters())) + if any(".fused_qkv_a_proj." in n for n in param_names): + stacked[".q_a_proj"] = (".fused_qkv_a_proj", 0) + stacked[".kv_a_proj_with_mqa"] = (".fused_qkv_a_proj", 1) + if any(".qkv_proj." in n for n in param_names): + stacked[".q_proj"] = (".qkv_proj", "q") + stacked[".k_proj"] = (".qkv_proj", "k") + stacked[".v_proj"] = (".qkv_proj", "v") + mapper = WeightsMapper( + orig_to_new_substr={ + "e_score_correction_bias": "gate.e_score_correction_bias", + }, + orig_to_new_stacked=stacked, + ) + loader = AutoWeightsLoader(self) + loaded = loader.load_weights(self._filter_spec_layers(weights), mapper=mapper) self.post_weight_load() - return loaded_params + return loaded def post_weight_load(self) -> None: for name, module in self.named_modules(): diff --git a/vllm/model_executor/models/param2moe.py b/vllm/model_executor/models/param2moe.py index ff56cf505f..48c0c714ed 100644 --- a/vllm/model_executor/models/param2moe.py +++ b/vllm/model_executor/models/param2moe.py @@ -32,10 +32,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -49,14 +46,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -74,45 +70,17 @@ def _zero_mean_tensor(t: torch.Tensor) -> torch.Tensor: return t - t.mean() -def _rename_and_normalize_weights( +def _normalize_expert_bias( weights: Iterable[tuple[str, torch.Tensor]], ) -> Iterator[tuple[str, torch.Tensor]]: - """ - Translate HuggingFace Param2MoE weight names to vLLM internal names - and zero-mean the expert-bias tensor so the router stays balanced. + """Zero-mean the MoE router's per-expert score bias for load balance. - Mapping table (HF → vLLM): - model.word_embeddings.* → model.embed_tokens.* - *.attention.query_key_value.* → *.self_attn.qkv_proj.* - *.attention.dense.* → *.self_attn.o_proj.* - *.attention.query_layernorm.* → *.self_attn.q_layernorm.* - *.attention.key_layernorm.* → *.self_attn.k_layernorm.* - *.mlp.gate.expert_bias → *.mlp.gate.e_score_correction_bias - (also zero-meant for load balance) + The rename to ``e_score_correction_bias`` is done by the mapper; only the + tensor adjustment lives here, since a WeightsMapper cannot transform data. """ for name, w in weights: - # Embedding table - name = name.replace("model.word_embeddings.", "model.embed_tokens.") - # Fused QKV projection (HF: query_key_value → vLLM: qkv_proj) - name = name.replace(".attention.query_key_value.", ".self_attn.qkv_proj.") - # Output projection (HF: dense → vLLM: o_proj) - name = name.replace(".attention.dense.", ".self_attn.o_proj.") - # Per-head query norm - name = name.replace(".attention.query_layernorm.", ".self_attn.q_layernorm.") - # Per-head key norm - name = name.replace(".attention.key_layernorm.", ".self_attn.k_layernorm.") - # Catch any remaining .attention. → .self_attn. prefixes - # (e.g. future bias params on the projection layers) - name = name.replace(".attention.", ".self_attn.") - - # Expert-score bias: rename + zero-mean - if name.endswith(".mlp.gate.expert_bias"): - name = name.replace( - ".mlp.gate.expert_bias", - ".mlp.gate.e_score_correction_bias", - ) + if _is_expert_bias_name(name): w = _zero_mean_tensor(w) - yield name, w @@ -123,7 +91,8 @@ class Param2MoEAttention(nn.Module): Notable differences from a vanilla GQA layer: * The checkpoint fuses Q, K, V into a single ``query_key_value`` weight. vLLM receives it already renamed to ``qkv_proj`` by the weight-name - translator and splits it during ``load_weights``. + translator and loads it directly; ``QKVParallelLinear`` splits the + fused ``[Q|K|V]`` tensor internally. * Optional per-head RMS norms on Q and K (``use_qk_norm=True``). """ @@ -472,6 +441,28 @@ class Param2MoEDecoderLayer(nn.Module): class Param2MoEModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + "word_embeddings.": "embed_tokens.", + # Fused query_key_value is renamed to qkv_proj and loaded directly; + # QKVParallelLinear splits the fused [Q|K|V] tensor internally. + ".attention.query_key_value.": ".self_attn.qkv_proj.", + ".attention.dense.": ".self_attn.o_proj.", + ".attention.query_layernorm.": ".self_attn.q_layernorm.", + ".attention.key_layernorm.": ".self_attn.k_layernorm.", + # Catch-all for any remaining .attention. names (must come last). + ".attention.": ".self_attn.", + ".mlp.gate.expert_bias": ".mlp.gate.e_score_correction_bias", + }, + orig_to_new_stacked={ + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + }, + ) + def __init__( self, *, @@ -558,147 +549,9 @@ class Param2MoEModel(nn.Module): self, weights: Iterable[tuple[str, torch.Tensor]], ) -> set[str]: - """ - Custom weight loader for the inner Param2MoEModel. - - Receives weights that have already been renamed/normalised by the - outer model and whose ``model.`` prefix has been stripped by - ``AutoWeightsLoader``. Handles: - 1. Fused QKV split (query_key_value → qkv_proj q/k/v shards). - 2. gate_proj + up_proj → gate_up_proj stacking (dense + shared-exp). - 3. Routed-expert weights via the fused-MoE mapping. - 4. All remaining weights via their default loader. - """ - config = self.config - num_heads: int = config.num_attention_heads - num_kv_heads: int = config.num_key_value_heads - head_dim: int = config.head_dim or (config.hidden_size // num_heads) - q_split = num_heads * head_dim - kv_split = num_kv_heads * head_dim - - stacked_params_mapping = [ - # (vllm_param_name, ckpt_weight_name, shard_id) - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - - for name, loaded_weight in weights: - # ------------------------------------------------------------------ - # 1. Fused QKV: split into q / k / v shards for QKVParallelLinear - # ------------------------------------------------------------------ - if name.endswith(".self_attn.qkv_proj.weight"): - if name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - q_w = loaded_weight[:q_split, :] - k_w = loaded_weight[q_split : q_split + kv_split, :] - v_w = loaded_weight[q_split + kv_split :, :] - weight_loader(param, q_w, "q") - weight_loader(param, k_w, "k") - weight_loader(param, v_w, "v") - loaded_params.add(name) - continue - - # ------------------------------------------------------------------ - # 2. gate_proj / up_proj → gate_up_proj (dense MLP + shared-exp.) - # ------------------------------------------------------------------ - matched_stacked = False - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "mlp.experts" in name: # routed experts handled below - continue - new_name = name.replace(weight_name, param_name) - if new_name.endswith(".bias") and new_name not in params_dict: - continue - if new_name not in params_dict: - continue - if is_pp_missing_parameter(new_name, self): - continue - - param = params_dict[new_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(new_name) - matched_stacked = True - break - - if matched_stacked: - continue - - # ------------------------------------------------------------------ - # 3. Routed expert weights → fused-MoE kernel layout - # ------------------------------------------------------------------ - matched_expert = False - for ( - param_name, - weight_name, - expert_id, - shard_id, - ) in expert_params_mapping: - if weight_name not in name: - continue - new_name = name.replace(weight_name, param_name) - if is_pp_missing_parameter(new_name, self): - continue - if new_name not in params_dict: - continue - - param = params_dict[new_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - loaded_params.add(new_name) - matched_expert = True - break - - if matched_expert: - continue - - # ------------------------------------------------------------------ - # 4. All other weights: direct load (layernorms, embed_tokens, …) - # ------------------------------------------------------------------ - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - try: - weight_loader(param, loaded_weight) - except Exception as e: - raise RuntimeError( - f"[param2moe] Failed to load weight '{name}' " - f"with shape {tuple(loaded_weight.shape)} " - f"into param type {type(param).__name__}: {e}" - ) from e - loaded_params.add(name) - - return loaded_params - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, + loader = AutoWeightsLoader(self) + return loader.load_weights( + _normalize_expert_bias(weights), mapper=self.hf_to_vllm_mapper ) @@ -859,4 +712,4 @@ class Param2MoEForCausalLM( weights: Iterable[tuple[str, torch.Tensor]], ) -> set[str]: loader = AutoWeightsLoader(self) - return loader.load_weights(_rename_and_normalize_weights(weights)) + return loader.load_weights(weights) diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index a58c3c4dd7..bcd2576b74 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -93,6 +93,7 @@ from .utils import ( extract_layer_index, make_empty_intermediate_tensors_factory, make_layers, + maybe_fuse_shared_experts, maybe_prefix, ) @@ -261,20 +262,20 @@ class Qwen3_5Model(Qwen3NextModel): self.aux_hidden_state_layers: tuple[int, ...] = () def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - mapper = self.hf_to_vllm_mapper # FSE must match construction (Qwen3NextSparseMoeBlock): reroute the # shared expert into the extra fused slot only when AITER FSE is both # requested and compatible with the quant spec. - is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and ( - _is_shared_expert_fse_compatible(self.quant_config) - ) - if is_fse: - num_routed = self.config.num_experts - mapper = mapper | WeightsMapper( - orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} + if "moe" in self.config.model_type: + weights = maybe_fuse_shared_experts( + weights, + enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + and _is_shared_expert_fse_compatible(self.quant_config), + n_routed_experts=self.config.num_experts, + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", ) loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Qwen3_5ForCausalLMBase( diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index b1580bfee7..0620509f2d 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -41,9 +41,9 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, - WeightsMapper, _merge_multimodal_embeddings, make_empty_intermediate_tensors_factory, + maybe_fuse_shared_experts, maybe_prefix, ) @@ -174,17 +174,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - mapper = self.hf_to_vllm_mapper - is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and ( - _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config) + weights = maybe_fuse_shared_experts( + weights, + enabled=rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + and _is_shared_expert_fse_compatible( + get_current_vllm_config().quant_config + ), + n_routed_experts=getattr(self.config, "num_experts", 0), + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", ) - if is_fse: - num_routed = getattr(self.config, "num_experts", 0) - mapper = mapper | WeightsMapper( - orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} - ) loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @support_torch_compile( diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 7843ce72bd..bf5ea501cc 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -31,10 +31,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.multimodal.inputs import NestedTensors from vllm.transformers_utils.config import set_default_rope_theta from vllm.transformers_utils.repo_utils import get_hf_file_bytes @@ -44,6 +40,7 @@ from .qwen2 import Qwen2MLP as Qwen3MLP from .qwen3 import Qwen3ForCausalLM from .utils import ( AutoWeightsLoader, + WeightsMapper, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -333,6 +330,17 @@ class DFlashQwen3DecoderLayer(nn.Module): @support_torch_compile class DFlashQwen3Model(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={"midlayer.": "layers.0."}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + def __init__( self, *, @@ -624,51 +632,26 @@ class DFlashQwen3Model(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - tp_rank = get_tensor_model_parallel_rank() + def _preprocess( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: tp_size = get_tensor_model_parallel_world_size() + tp_rank = get_tensor_model_parallel_rank() for name, loaded_weight in weights: - if "midlayer." in name: - name = name.replace("midlayer.", "layers.0.") - if "scale" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue if "attention_sink_bias" in name: - if name not in params_dict: - continue # Sink bias is per-head; shard it across TP ranks like the # attention heads themselves. - param = params_dict[name] heads_per_rank = loaded_weight.shape[0] // tp_size - head_start = tp_rank * heads_per_rank - narrow_weight = loaded_weight.narrow(0, head_start, heads_per_rank) - param.data.copy_(narrow_weight) - loaded_params.add(name) - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loaded_weight = loaded_weight.narrow( + 0, tp_rank * heads_per_rank, heads_per_rank + ) + yield name, loaded_weight + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights( + self._preprocess(weights), mapper=self.hf_to_vllm_mapper + ) class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index d87d19f02a..50bc4cce18 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -71,6 +71,7 @@ from .utils import ( extract_layer_index, make_empty_intermediate_tensors_factory, make_layers, + maybe_fuse_shared_experts, maybe_prefix, ) @@ -713,17 +714,14 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - mapper = self.hf_to_vllm_mapper - if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): - # AITER fused-shared-experts: route the shared_expert checkpoint - # weights into the extra fused expert slot. Merge (not mutate) so the - # shared class mapper isn't permanently altered. - num_routed = getattr(self.config, "num_experts", 0) - mapper = mapper | WeightsMapper( - orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} - ) + weights = maybe_fuse_shared_experts( + weights, + n_routed_experts=getattr(self.config, "num_experts", 0), + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", + ) loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class QwenNextMixtureOfExperts(MixtureOfExperts): diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 1704f30a84..c832e955b2 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -7,7 +7,6 @@ from collections.abc import Iterable import torch from torch import nn -from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_pp_group @@ -30,8 +29,8 @@ from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig from .utils import ( AutoWeightsLoader, - WeightsMapper, make_empty_intermediate_tensors_factory, + maybe_fuse_shared_experts, maybe_prefix, ) @@ -154,16 +153,14 @@ class Qwen3NextMultiTokenPredictor(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - mapper = self.hf_to_vllm_mapper - if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): - # AITER fused-shared-experts: route the shared_expert checkpoint - # weights into the extra fused expert slot. - num_routed = self.config.num_experts - mapper = mapper | WeightsMapper( - orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} - ) + weights = maybe_fuse_shared_experts( + weights, + n_routed_experts=self.config.num_experts, + n_shared_experts=1, + ckpt_prefix="mlp.shared_expert", + ) loader = AutoWeightsLoader(self) - return loader.load_weights(weights, mapper=mapper) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @support_torch_compile diff --git a/vllm/model_executor/models/sarvam.py b/vllm/model_executor/models/sarvam.py index f59579b1bc..b9f9532ae1 100644 --- a/vllm/model_executor/models/sarvam.py +++ b/vllm/model_executor/models/sarvam.py @@ -35,11 +35,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - MoERunner, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -55,7 +51,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .bailing_moe import BailingMoeForCausalLM @@ -63,7 +58,7 @@ from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -448,6 +443,16 @@ class SarvamMLABlock(nn.Module): class SarvamMLAModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + ) + def __init__( self, *, @@ -532,97 +537,14 @@ class SarvamMLAModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - def load_weights( self, weights: Iterable[tuple[str, torch.Tensor]], ) -> set[str]: - """Load weights with stacked gate+up and MoE expert remapping.""" - weights = _normalized_weights(weights) - stacked_params_mapping = [ - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "mlp.experts" in name: - continue - new_name = name.replace(weight_name, param_name) - if new_name.endswith(".bias") and new_name not in params_dict: - continue - if new_name not in params_dict: - continue - if is_pp_missing_parameter(new_name, self): - continue - - param = params_dict[new_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight, shard_id) - loaded_params.add(new_name) - break - else: - mapped = False - for ( - param_name, - weight_name, - expert_id, - shard_id, - ) in expert_params_mapping: - if weight_name not in name: - continue - - new_name = name.replace(weight_name, param_name) - if is_pp_missing_parameter(new_name, self): - continue - if new_name not in params_dict: - continue - - param = params_dict[new_name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - loaded_params.add(new_name) - mapped = True - break - - if mapped: - continue - - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights( + _normalized_weights(weights), mapper=self.hf_to_vllm_mapper + ) class SarvamMixtureOfExperts(MixtureOfExperts): @@ -764,9 +686,6 @@ class SarvamMLAForCausalLM(nn.Module, SupportsPP, SupportsLoRA, SarvamMixtureOfE ) return loader.load_weights(weights) - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - class SarvamMoEForCausalLM(BailingMoeForCausalLM): """Same as BailingMoeForCausalLM, but normalizes gate expert_bias pre-load.""" diff --git a/vllm/model_executor/models/step3p5.py b/vllm/model_executor/models/step3p5.py index f8bd529e27..07a25d23c8 100644 --- a/vllm/model_executor/models/step3p5.py +++ b/vllm/model_executor/models/step3p5.py @@ -53,6 +53,7 @@ from .utils import ( PPMissingLayer, WeightsMapper, extract_layer_index, + get_spec_layer_idx_from_weight_name, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, @@ -978,18 +979,3 @@ class Step3p5ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - -def get_spec_layer_idx_from_weight_name( - config: ModelConfig, weight_name: str -) -> int | None: - if hasattr(config, "num_nextn_predict_layers") and ( - config.num_nextn_predict_layers > 0 - ): - layer_idx = config.num_hidden_layers - for i in range(config.num_nextn_predict_layers): - if weight_name.startswith( - f"layers.{layer_idx + i}." # Step3p5Model - ) or weight_name.startswith(f"model.layers.{layer_idx + i}."): # Step3p5MTP - return layer_idx + i - return None diff --git a/vllm/model_executor/models/step3p5_mtp.py b/vllm/model_executor/models/step3p5_mtp.py index 092f7a31aa..b533a9111d 100644 --- a/vllm/model_executor/models/step3p5_mtp.py +++ b/vllm/model_executor/models/step3p5_mtp.py @@ -18,8 +18,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors -from .step3p5 import Step3p5DecoderLayer, get_spec_layer_idx_from_weight_name -from .utils import maybe_prefix +from .step3p5 import Step3p5DecoderLayer +from .utils import get_spec_layer_idx_from_weight_name, maybe_prefix logger = init_logger(__name__) diff --git a/vllm/model_executor/models/telechat2.py b/vllm/model_executor/models/telechat2.py index 113581d55f..42fa6d6871 100644 --- a/vllm/model_executor/models/telechat2.py +++ b/vllm/model_executor/models/telechat2.py @@ -26,19 +26,23 @@ import torch import torch.nn as nn from vllm.config import VllmConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.llama import LlamaForCausalLM, LlamaModel from .llama import LlamaDecoderLayer -from .utils import ( - AutoWeightsLoader, - PPMissingLayer, - WeightsMapper, - is_pp_missing_parameter, -) +from .utils import AutoWeightsLoader, PPMissingLayer, WeightsMapper class TeleChat2Model(LlamaModel): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".query": (".qkv_proj", "q"), + ".key": (".qkv_proj", "k"), + ".value": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): hf_config = vllm_config.model_config.hf_config @@ -65,62 +69,34 @@ class TeleChat2Model(LlamaModel): layer.mlp.gate_up_proj.bias = None layer.mlp.gate_up_proj.skip_bias_add = True - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() + def _split_key_value( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[tuple[str, torch.Tensor]]: + # TeleChat2 stores k/v as a single per-head-interleaved `key_value` + # tensor. De-interleave it into separate k/v so the qkv_proj mapper + # can stack them. total_num_heads = self.config.n_head head_dim = self.config.hidden_size // total_num_heads for name, loaded_weight in weights: if "self_attn.key_value" in name: - k_weight = [] - v_weight = [] - for i in range(total_num_heads): - start = i * head_dim * 2 - k_weight.append(loaded_weight[start : start + head_dim, :]) - v_weight.append( - loaded_weight[start + head_dim : start + 2 * head_dim :] - ) - k_weight = torch.cat(k_weight, dim=0) - v_weight = torch.cat(v_weight, dim=0) - name = name.replace("key_value", "qkv_proj") - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, k_weight, "k") - weight_loader(param, v_weight, "v") - elif "query" in name: - name = name.replace("query", "qkv_proj") - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, "q") + starts = [i * head_dim * 2 for i in range(total_num_heads)] + k_weight = torch.cat( + [loaded_weight[s : s + head_dim, :] for s in starts], dim=0 + ) + v_weight = torch.cat( + [loaded_weight[s + head_dim : s + 2 * head_dim, :] for s in starts], + dim=0, + ) + yield name.replace("key_value", "key"), k_weight + yield name.replace("key_value", "value"), v_weight else: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + yield name, loaded_weight + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader(self) + return loader.load_weights( + self._split_key_value(weights), mapper=self.hf_to_vllm_mapper + ) class TeleChat2ForCausalLM(LlamaForCausalLM): diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 86dc72de76..a6f39f6ebf 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -35,6 +35,7 @@ if TYPE_CHECKING: from transformers import PretrainedConfig from transformers.conversion_mapping import WeightRenaming + from vllm.config.model import ModelConfig from vllm.model_executor.layers.quantization import QuantizationConfig logger = init_logger(__name__) @@ -424,6 +425,116 @@ class AutoWeightsLoader: return autoloaded_weights +def maybe_fuse_shared_experts( + weights: Iterable[tuple[str, torch.Tensor]], + *, + n_routed_experts: int, + n_shared_experts: int, + ckpt_prefix: str = "mlp.shared_experts", + enabled: bool | None = None, +) -> Iterable[tuple[str, torch.Tensor]]: + """Route AITER fused-shared-expert checkpoint weights into fused slots. + + When AITER fused-shared-experts is active, shared experts are packed into + the routed expert tensor. The checkpoint stores them under `ckpt_prefix` + as a single (possibly widened) tensor; this splits it into + `n_shared_experts` chunks named `mlp.experts.{n_routed_experts + j}` so + the `RoutedExperts` loader treats them as extra experts. Yields the input + unchanged when the fusion is inactive, so callers can wrap unconditionally. + + Args: + weights: Iterable of `(name, tensor)` checkpoint pairs. + n_routed_experts: Number of routed experts; offsets the fused slots. + n_shared_experts: Number of shared experts packed into the tensor. + ckpt_prefix: Checkpoint module name of the shared experts. + enabled: Whether AITER fused-shared-experts is active. Defaults to + `rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()`; pass an + explicit value only when the model gates on something more (e.g. + quant-spec compatibility) and it must match its construction-time + decision. + + Yields: + `(name, tensor)` pairs with shared experts routed to fused slots. + """ + if enabled is None: + from vllm._aiter_ops import rocm_aiter_ops + + enabled = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() + if not enabled: + yield from weights + return + + # Match on the dotted boundary so e.g. "mlp.shared_expert." does not also + # catch a sibling "mlp.shared_expert_gate". + prefix = f"{ckpt_prefix}." + for name, loaded_weight in weights: + if prefix not in name: + yield name, loaded_weight + continue + # gate/up split on the output dim; down_proj on the input dim. + split_dim = 1 if ("down_proj.weight" in name and loaded_weight.ndim > 1) else 0 + total = loaded_weight.shape[split_dim] + if total % n_shared_experts != 0: + raise ValueError( + f"FSE shared-expert weight {name!r} has size {total} along axis " + f"{split_dim}, not divisible by n_shared_experts={n_shared_experts}." + ) + chunk = total // n_shared_experts + for j in range(n_shared_experts): + sl = slice(j * chunk, (j + 1) * chunk) + if loaded_weight.ndim == 1: + chunk_weight = loaded_weight[sl] + elif split_dim == 0: + chunk_weight = loaded_weight[sl, :] + else: + chunk_weight = loaded_weight[:, sl] + yield ( + name.replace(prefix, f"mlp.experts.{n_routed_experts + j}."), + chunk_weight, + ) + + +def get_spec_layer_idx_from_weight_name( + config: "ModelConfig", weight_name: str +) -> int | None: + """Return the MTP layer index a weight belongs to, or None. + + Args: + config: The model config; must expose `num_hidden_layers` and, for MTP + checkpoints, `num_nextn_predict_layers`. + weight_name: Checkpoint weight name to classify. + + Returns: + The absolute layer index for an MTP-layer weight, else None. + """ + if not (n := getattr(config, "num_nextn_predict_layers", 0)): + return None + base = config.num_hidden_layers + for i in range(n): + if weight_name.startswith((f"model.layers.{base + i}.", f"layers.{base + i}.")): + return base + i + return None + + +def skip_spec_layers( + weights: Iterable[tuple[str, torch.Tensor]], config: "ModelConfig" +) -> Iterable[tuple[str, torch.Tensor]]: + """Drop MTP spec-layer weights (loaded by the MTP head, not the base model). + + Args: + weights: Iterable of `(name, tensor)` checkpoint pairs. + config: The model config, passed to `get_spec_layer_idx_from_weight_name`. + + Yields: + `(name, tensor)` pairs whose weight is not an MTP-layer weight. + """ + return ( + (name, w) + for name, w in weights + if get_spec_layer_idx_from_weight_name(config, name) is None + ) + + def init_vllm_registered_model( vllm_config: VllmConfig, *,