diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py index d5c573c4ff..6a18f63642 100644 --- a/tests/models/multimodal/processing/test_moss_audio.py +++ b/tests/models/multimodal/processing/test_moss_audio.py @@ -17,7 +17,6 @@ from vllm.model_executor.models.moss_audio import ( MOSS_AUDIO_PLACEHOLDER, MOSS_AUDIO_TOKEN, MOSS_AUDIO_TOKEN_ID, - GatedMLP, MossAudioConfig, MossAudioDummyInputsBuilder, MossAudioEncoder, @@ -594,53 +593,6 @@ def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch): assert set(forward_tensors.tensors) == set(tensors.tensors) -@pytest.mark.parametrize("tp_size", [1, 2]) -def test_moss_audio_gated_mlp_tp_shapes_and_loading(monkeypatch, tp_size): - from vllm.config import VllmConfig, set_current_vllm_config - from vllm.config.device import DeviceConfig - - _patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=tp_size) - with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))): - mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6) - - params = dict(mlp.named_parameters()) - assert params["gate_up_proj.weight"].shape == torch.Size([16 // tp_size, 4]) - assert params["down_proj.weight"].shape == torch.Size([6, 8 // tp_size]) - - gate_weight = torch.arange(32, dtype=torch.float32).reshape(8, 4) - up_weight = torch.arange(100, 132, dtype=torch.float32).reshape(8, 4) - down_weight = torch.arange(48, dtype=torch.float32).reshape(6, 8) - loaded = mlp.load_weights( - [ - ("gate_proj.weight", gate_weight), - ("up_proj.weight", up_weight), - ("down_proj.weight", down_weight), - ] - ) - - assert loaded == {"gate_up_proj.weight", "down_proj.weight"} - shard = 8 // tp_size - assert torch.equal(params["gate_up_proj.weight"][:shard], gate_weight[:shard]) - assert torch.equal(params["gate_up_proj.weight"][shard:], up_weight[:shard]) - assert torch.equal(params["down_proj.weight"], down_weight[:, : 8 // tp_size]) - - with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))): - packed_mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6) - packed_params = dict(packed_mlp.named_parameters()) - loaded = packed_mlp.load_weights( - [("gate_up_proj.weight", torch.cat([gate_weight, up_weight], dim=0))] - ) - assert loaded == {"gate_up_proj.weight"} - assert torch.equal( - packed_params["gate_up_proj.weight"][:shard], - gate_weight[:shard], - ) - assert torch.equal( - packed_params["gate_up_proj.weight"][shard:], - up_weight[:shard], - ) - - def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch): from vllm.config import VllmConfig, set_current_vllm_config from vllm.config.device import DeviceConfig diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index c3763f5448..63a4ea9a82 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -39,6 +39,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): self._ep_check() routed_experts = self.base_layer.routed_experts + routed_experts.lora_base_layer_prefix = "base_layer." assert not routed_experts.quant_method.is_monolithic, ( "Monolithic kernels are not supported for Fused MoE LoRA." ) diff --git a/vllm/lora/utils.py b/vllm/lora/utils.py index a628d70cba..6b9c66b980 100644 --- a/vllm/lora/utils.py +++ b/vllm/lora/utils.py @@ -372,32 +372,26 @@ def process_packed_modules_mapping( model: nn.Module, force_2d_moe: bool = False ) -> dict[str, list[str]]: if is_moe_model(model): - if moe_packed_mapping := get_moe_expert_mapping(model): - # This method generates and returns a dictionary mapping packed module - # names to lists of their corresponding submodule names. It includes - # both static mappings and dynamic mappings for expert layers, where - # the expert indices are expanded based on the configured number - # of routed experts. - packed_modules_mapping = get_packed_modules_mapping(model) - # The 2D mapping is needed when the model itself is 2D, or when - # the engine forces the universal 2D wrapper via - # enable_mixed_moe_lora_format (so 3D models can also load 2D - # adapters through FusedMoEWithLoRA). - if (not model.is_3d_moe_weight) or force_2d_moe: - # Filter out malformed entries: non-gated MoE has empty - # ckpt_up_proj_name which results in weight_name containing ".." - # (e.g., "experts.0.." instead of "experts.0.layer_name.") - packed_modules_mapping["experts"] = [ - weight_name.rstrip(".") - for _, weight_name, _, _ in moe_packed_mapping - if ".." not in weight_name - ] + # This method generates and returns a dictionary mapping packed module + # names to lists of their corresponding submodule names. It includes + # both static mappings and dynamic mappings for expert layers, where + # the expert indices are expanded based on the configured number + # of routed experts. + packed_modules_mapping = get_packed_modules_mapping(model) + # The 2D mapping is needed when the model itself is 2D, or when + # the engine forces the universal 2D wrapper via + # enable_mixed_moe_lora_format (so 3D models can also load 2D + # adapters through FusedMoEWithLoRA). + if (not model.is_3d_moe_weight) or force_2d_moe: + # Filter out malformed entries: non-gated MoE has empty + # ckpt_up_proj_name which results in weight_name containing ".." + # (e.g., "experts.0.." instead of "experts.0.layer_name.") + packed_modules_mapping["experts"] = [ + weight_name.rstrip(".") + for _, weight_name, _, _ in get_moe_expert_mapping(model) + if ".." not in weight_name + ] - return packed_modules_mapping - else: - raise AttributeError( - "To support LoRA for MoE model, " - "'get_expert_mapping' must be implemented" - ) + return packed_modules_mapping else: return get_packed_modules_mapping(model) diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 871f905bad..a522a39153 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -127,7 +127,7 @@ def FusedMoE( num_redundant_experts: int = 0, has_bias: bool = False, is_sequence_parallel: bool = False, - expert_mapping: list[tuple[str, str, int, str]] | None = None, + ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"), n_shared_experts: int | None = None, router_logits_dtype: torch.dtype | None = None, gate: torch.nn.Module | None = None, @@ -356,7 +356,9 @@ def FusedMoE( moe_config, quant_config, expert_map_manager=expert_map_manager, - expert_mapping=expert_mapping, + ckpt_gate_proj_name=ckpt_names[0], + ckpt_down_proj_name=ckpt_names[1], + ckpt_up_proj_name=ckpt_names[2], # Extra params that are needed by quant_methods, pass along for now # Prefer getting these from other sources, e.g. moe_config or # router object diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py index 99a481cf67..1b1deb250c 100644 --- a/vllm/model_executor/layers/fused_moe/routed_experts.py +++ b/vllm/model_executor/layers/fused_moe/routed_experts.py @@ -58,7 +58,9 @@ class RoutedExperts(PluggableLayer): moe_config: FusedMoEConfig, quant_config: QuantizationConfig | None, expert_map_manager: ExpertMapManager, - expert_mapping: list[tuple[str, str, int, str]] | None = None, + ckpt_gate_proj_name: str = "gate_proj", + ckpt_down_proj_name: str = "down_proj", + ckpt_up_proj_name: str = "up_proj", # # Extra params that are needed by quant_methods, pass along for now # Prefer getting these from other sources, e.g. moe_config or @@ -81,7 +83,9 @@ class RoutedExperts(PluggableLayer): self.layer_name = layer_name self.moe_config = moe_config self.quant_config = quant_config - self.expert_mapping = expert_mapping + self.ckpt_gate_proj_name = ckpt_gate_proj_name + self.ckpt_down_proj_name = ckpt_down_proj_name + self.ckpt_up_proj_name = ckpt_up_proj_name self.expert_map_manager = expert_map_manager self.hidden_size = moe_config.hidden_dim self.global_num_experts = moe_config.num_experts @@ -167,6 +171,8 @@ class RoutedExperts(PluggableLayer): self.quant_method.create_weights(layer=self, **moe_quant_params) + self.lora_base_layer_prefix = "" + # TODO(bnell): Temporary hack. Get rid of this. def _replace_quant_method(self, quant_method: FusedMoEMethodBase): self.quant_method = quant_method @@ -863,26 +869,33 @@ class RoutedExperts(PluggableLayer): def load_weights( self, weights: Iterable[tuple[str, torch.Tensor]] ) -> Iterable[str]: - if (expert_mapping := self.expert_mapping) is None: - raise ValueError( - "`self.expert_mapping` must be provided to " - "load weights using `self.load_weights`." - ) + expert_mapping = self.get_expert_mapping(include_fused=True) + unpadded_hidden = self.moe_config.hidden_dim_unpadded for expert_name, loaded_weight in weights: qual_name = f"{self.layer_name}.{expert_name}" + # Fused expert weights can be identified by their 3D tensors + is_fused = loaded_weight.dim() == 3 + matched = False for param_name, weight_name, expert_id, shard_id in expert_mapping: if weight_name not in qual_name: + if matched and is_fused: + break continue + matched = True weight_name = qual_name.replace(weight_name, param_name) param_name = weight_name.removeprefix(f"{self.layer_name}.") param = getattr(self, param_name) - # Fused expert weights can be identified by their 3D tensors - if loaded_weight.dim() == 3: - # Repurpose expert_id as shard_idx for deconcatenating w1 and w3 + if is_fused: if shard_id in {"w1", "w3"}: - shard_idx = expert_id - experts_shard = loaded_weight.chunk(2, dim=1)[shard_idx] + if loaded_weight.shape[-1] != unpadded_hidden: + # [..., hidden, intermediate] -> [..., intermediate, hidden] + loaded_weight = loaded_weight.transpose(-1, -2) + # Repurpose expert_id for deconcatenating w1 and w3 + experts_shard = loaded_weight.chunk(2, dim=1)[expert_id] else: + if loaded_weight.shape[-2] != unpadded_hidden: + # [..., intermediate, hidden] -> [..., hidden, intermediate] + loaded_weight = loaded_weight.transpose(-1, -2) experts_shard = loaded_weight start = 0 else: @@ -912,6 +925,27 @@ class RoutedExperts(PluggableLayer): ) yield param_name + def get_expert_mapping( + self, + ckpt_gate_proj_name: str | None = None, + ckpt_down_proj_name: str | None = None, + ckpt_up_proj_name: str | None = None, + include_fused: bool = False, + ) -> list[tuple[str, str, int, str]]: + moe_config = self.moe_config + num_fused_shared_experts = self.expert_map_manager.num_fused_shared_experts + num_redundant_experts = moe_config.num_experts - moe_config.num_logical_experts + return self.build_expert_params_mapping( + ckpt_gate_proj_name or self.ckpt_gate_proj_name, + ckpt_down_proj_name or self.ckpt_down_proj_name, + ckpt_up_proj_name or self.ckpt_up_proj_name, + num_experts=moe_config.num_logical_experts + num_fused_shared_experts, + num_redundant_experts=num_redundant_experts, + routed_experts_prefix="", + lora_base_layer_prefix=self.lora_base_layer_prefix, + include_fused=include_fused, + ) + @staticmethod def make_expert_params_mapping( model: torch.nn.Module, @@ -921,6 +955,36 @@ class RoutedExperts(PluggableLayer): num_experts: int, num_redundant_experts: int = 0, routed_experts_prefix: str = "routed_experts", + ) -> list[tuple[str, str, int, str]]: + """Build the expert mapping, detecting the LoRA `base_layer.` prefix by + scanning `model`'s parameters. + + Legacy entry point for models that still hand-roll `load_weights`; the + `RoutedExperts` weight loader uses `get_expert_mapping` / + `build_expert_params_mapping` instead (which take the prefix directly). + See `build_expert_params_mapping` for the returned tuple format. + """ + has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters()) + return RoutedExperts.build_expert_params_mapping( + ckpt_gate_proj_name, + ckpt_down_proj_name, + ckpt_up_proj_name, + num_experts, + num_redundant_experts, + routed_experts_prefix, + "base_layer." if has_base_layer else "", + ) + + @staticmethod + def build_expert_params_mapping( + ckpt_gate_proj_name: str, + ckpt_down_proj_name: str, + ckpt_up_proj_name: str, + num_experts: int, + num_redundant_experts: int = 0, + routed_experts_prefix: str = "routed_experts", + lora_base_layer_prefix: str = "", + include_fused: bool = False, ) -> list[tuple[str, str, int, str]]: """ Create expert parameter mapping for weight loading with redundant experts. @@ -929,12 +993,13 @@ class RoutedExperts(PluggableLayer): when loading weights with EPLB redundant experts. Args: - model: The model containing the MoE layer ckpt_gate_proj_name: Name of gate projection in checkpoint ckpt_down_proj_name: Name of down projection in checkpoint ckpt_up_proj_name: Name of up projection in checkpoint num_experts: Number of logical (non-redundant) experts num_redundant_experts: Number of redundant experts + lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer + include_fused: Prepend the fused pre-fused-checkpoint entries Returns: List of tuples (param_name, weight_name, expert_id, shard_id) @@ -956,22 +1021,39 @@ class RoutedExperts(PluggableLayer): ) ) - base_layer = ( - "base_layer." - if any(".base_layer." in name for name, _ in model.named_parameters()) - else "" - ) - if routed_experts_prefix != "": routed_experts_prefix = f"{routed_experts_prefix}." - return [ + w13 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w13_" + w2 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w2_" + + fused_mapping = [] + if include_fused: + gate_up = None + if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj": + gate_up = "gate_up_proj" + elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3": + gate_up = "w13" + else: + logger.warning( + "Unexpected gate/up projection names: %s, %s. " + "Fused gate/up mapping will be skipped.", + ckpt_gate_proj_name, + ckpt_up_proj_name, + ) + if gate_up is not None: + fused_mapping = [ + # (param_name, weight_name, expert_id, shard_id) + (f"{w13}weight", f"experts.{gate_up}", 0, "w1"), + (f"{w13}weight", f"experts.{gate_up}", 1, "w3"), + (f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"), + ] + + per_expert_mapping = [ # (param_name, weight_name, expert_id, shard_id) ( - f"experts.{routed_experts_prefix}{base_layer}w13_" - if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] - else f"experts.{routed_experts_prefix}{base_layer}w2_", - f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{base_layer}", + w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2, + f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}", expert_id, shard_id, ) @@ -983,6 +1065,8 @@ class RoutedExperts(PluggableLayer): ] ] + return fused_mapping + per_expert_mapping + def get_expert_weights(self) -> Iterable[torch.Tensor]: def _maybe_make_contiguous( name: str, p: torch.nn.Parameter diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 140466c7f4..d8deb73e9b 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -292,6 +292,11 @@ class MoERunner(MoERunnerInterface): # For smuggling this layer into the fused moe custom op register_layer_for_moe_forward_op(get_current_vllm_config(), self) + def load_weights( + self, weights: Iterable[tuple[str, torch.Tensor]] + ) -> Iterable[str]: + return self.routed_experts.load_weights(weights) + def _select_forward(self) -> Callable: if current_platform.is_tpu() or current_platform.is_cpu(): # TODO: Once the OOM issue for the TPU backend is resolved, we diff --git a/vllm/model_executor/model_loader/bitsandbytes_loader.py b/vllm/model_executor/model_loader/bitsandbytes_loader.py index 55b5d617a7..cc9af05af7 100644 --- a/vllm/model_executor/model_loader/bitsandbytes_loader.py +++ b/vllm/model_executor/model_loader/bitsandbytes_loader.py @@ -567,12 +567,6 @@ class BitsAndBytesModelLoader(BaseModelLoader): if is_moe_model(model): self.expert_params_mapping = get_moe_expert_mapping(model) - if not self.expert_params_mapping: - raise AttributeError( - f"MoE Model {type(model).__name__} does not support " - "BitsAndBytes quantization yet. Ensure this model has " - "'get_expert_mapping' method." - ) # For some models like Molmo, we need to use hf_to_vllm_mapper # to ensure correct loading of weights. if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None): diff --git a/vllm/model_executor/models/afmoe.py b/vllm/model_executor/models/afmoe.py index 369b7c3b3a..0122d01958 100644 --- a/vllm/model_executor/models/afmoe.py +++ b/vllm/model_executor/models/afmoe.py @@ -2,8 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Inference-only AfMoE model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice import torch @@ -21,7 +20,6 @@ from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, MoERunner, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -36,10 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.interfaces import ( EagleModelMixin, MixtureOfExperts, @@ -53,7 +47,6 @@ from vllm.model_executor.models.utils import ( PPMissingLayer, WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -481,120 +474,6 @@ class AfmoeModel(nn.Module, EagleModelMixin): } ) - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if (weight_name not in name) or ("self_attn.gate_proj" in name): - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params - class AfmoeForCausalLM( nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA, MixtureOfExperts @@ -615,6 +494,17 @@ class AfmoeForCausalLM( orig_to_new_suffix={ ".router.gate.weight": ".gate.weight", }, + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # `.self_attn.gate_proj` is a gated-attention projection (not fused). + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + }, ) fall_back_to_pt_during_load = False @@ -705,6 +595,3 @@ class AfmoeForCausalLM( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/aimv2.py b/vllm/model_executor/models/aimv2.py index 63cb9c96e2..bc4b3e8cd0 100644 --- a/vllm/model_executor/models/aimv2.py +++ b/vllm/model_executor/models/aimv2.py @@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper from vllm.transformers_utils.configs.ovis import AIMv2Config @@ -194,6 +194,13 @@ class AIMv2Transformer(nn.Module): class AIMv2Model(torch.nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".fc1": (".fc13", 0), + ".fc3": (".fc13", 1), + } + ) + def __init__( self, config: AIMv2Config, @@ -218,34 +225,13 @@ class AIMv2Model(torch.nn.Module): return x def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".fc13", ".fc1", 0), - (".fc13", ".fc3", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - # post_layernorm is optional in SiglipVisionModel - if ( - name.startswith("trunk.post_trunk_norm") - and self.trunk.post_trunk_norm is None - ): - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader( + self, + # post_trunk_norm is optional (absent for clip-skip backbones). + skip_prefixes=( + ["trunk.post_trunk_norm."] + if self.trunk.post_trunk_norm is None + else None + ), + ) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/apertus.py b/vllm/model_executor/models/apertus.py index a3ea9ba434..b997e153a9 100644 --- a/vllm/model_executor/models/apertus.py +++ b/vllm/model_executor/models/apertus.py @@ -53,10 +53,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from vllm.v1.attention.backend import AttentionType @@ -70,8 +66,8 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -404,66 +400,17 @@ class ApertusModel(nn.Module, EagleModelMixin): return hidden_states, aux_hidden_states return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - - # we need to load the buffers for beta and eps (XIELU) - for name, buffer in self.named_buffers(): - if name.endswith(".beta") or name.endswith(".eps"): - params_dict[name] = buffer - - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - if "scale" in name or "zero_point" in name: - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class ApertusForCausalLM( nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3 ): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} # LoRA specific attributes @@ -548,4 +495,4 @@ class ApertusForCausalLM( self, skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), ) - return loader.load_weights(weights) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/bert.py b/vllm/model_executor/models/bert.py index c9ea736329..cf968902f6 100644 --- a/vllm/model_executor/models/bert.py +++ b/vllm/model_executor/models/bert.py @@ -369,6 +369,14 @@ class BertModel(nn.Module, SupportsQuant): packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]} + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".self.query": (".self.qkv_proj", "q"), + ".self.key": (".self.qkv_proj", "k"), + ".self.value": (".self.qkv_proj", "v"), + } + ) + def __init__( self, *, @@ -400,43 +408,9 @@ class BertModel(nn.Module, SupportsQuant): return self.encoder(hidden_states) - def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "query", "q"), - ("qkv_proj", "key", "k"), - ("qkv_proj", "value", "v"), - ] - - loaded_stacked_params = [] - other_weights = [] - params_dict = dict(self.named_parameters()) - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - - name = name.replace(weight_name, param_name) - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - loaded_stacked_params.append(name) - break - else: - if name in params_dict: - other_weights.append((name, loaded_weight)) - - return other_weights, loaded_stacked_params - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - other_weights, loaded_stacked_params = self._load_weights(weights) - loader = AutoWeightsLoader(self, skip_prefixes=["pooler."]) - loaded_params = loader.load_weights(other_weights) - loaded_params.update(loaded_stacked_params) - return loaded_params + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class BertPoolingModel(BertModel): @@ -458,12 +432,8 @@ class BertPoolingModel(BertModel): self.pooler = BertPooler(vllm_config.model_config) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - other_weights, loaded_stacked_params = self._load_weights(weights) - loader = AutoWeightsLoader(self) - loaded_params = loader.load_weights(other_weights) - loaded_params.update(loaded_stacked_params) - return loaded_params + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @default_pooling_type(seq_pooling_type="CLS") diff --git a/vllm/model_executor/models/blip.py b/vllm/model_executor/models/blip.py index 73b0b8af93..aecb997161 100644 --- a/vllm/model_executor/models/blip.py +++ b/vllm/model_executor/models/blip.py @@ -19,9 +19,9 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from .interfaces import SupportsQuant +from .utils import AutoWeightsLoader, WeightsMapper def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int: @@ -268,6 +268,14 @@ class BlipVisionModel(nn.Module, SupportsQuant): main_input_name = "pixel_values" packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]} + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv", "q"), + ".k_proj": (".qkv", "k"), + ".v_proj": (".qkv", "v"), + } + ) + def __init__( self, config: BlipVisionConfig, @@ -316,38 +324,18 @@ class BlipVisionModel(nn.Module, SupportsQuant): return self.post_layernorm(hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - layer_count = len(self.encoder.layers) + skip_prefixes: list[str] = [] + if self.post_layernorm is None: + skip_prefixes.append("post_layernorm.") + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) - for name, loaded_weight in weights: - # post_layernorm is not needed in BlipVisionModel - if name.startswith("post_layernorm") and self.post_layernorm is None: - continue - - # omit layers when num_hidden_layers_override is set - if name.startswith("encoder.layers"): - layer_idx = int(name.split(".")[2]) - if layer_idx >= layer_count: + # omit layers when num_hidden_layers_override is set + def _filter(ws): + for name, weight in ws: + if name.startswith("encoder.layers.") and int( + name.split(".")[2] + ) >= len(self.encoder.layers): continue + yield name, weight - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/chameleon.py b/vllm/model_executor/models/chameleon.py index a150428baf..fec2fa6409 100644 --- a/vllm/model_executor/models/chameleon.py +++ b/vllm/model_executor/models/chameleon.py @@ -37,10 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - row_parallel_weight_loader, -) +from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader from vllm.model_executor.utils import set_weight_attrs from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -66,7 +63,8 @@ from .interfaces import ( SupportsQuant, ) from .utils import ( - is_pp_missing_parameter, + AutoWeightsLoader, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -922,6 +920,16 @@ class ChameleonForConditionalGeneration( "gate_up_proj": ["gate_proj", "up_proj"], } + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("image"): @@ -1023,81 +1031,6 @@ class ChameleonForConditionalGeneration( return logits def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - - # With tie_word_embeddings, we can skip lm_head.weight - # The weight might appear unnecessarily in the files if the model is - # processed with quantization, LoRA, fine-tuning, etc. - if self.config.tie_word_embeddings and "lm_head.weight" in name: - continue - - use_default_weight_loading = False - if "vqmodel" in name: - if self.model.vqmodel is not None: - # We only do sharding for language model and - # not vqvae for now. - use_default_weight_loading = True - else: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Remapping the name of FP8 kv-scale. - if name.endswith("kv_scale"): - remapped_kv_scale_name = name.replace( - ".kv_scale", ".attn.kv_scale" - ) - if remapped_kv_scale_name not in params_dict: - logger.warning_once( - "Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501 - name, - remapped_kv_scale_name, - ) - continue - else: - name = remapped_kv_scale_name - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - if use_default_weight_loading and name in params_dict: - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index d16bc1d32d..8919d65de3 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -29,7 +29,6 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import SupportsQuant from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -56,7 +55,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal from .interfaces_base import default_pooling_type -from .utils import AutoWeightsLoader, maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix from .vision import ( VisionEncoderInfo, VisionFeatureSelectStrategy, @@ -555,6 +554,14 @@ class CLIPEncoder(nn.Module): class CLIPTextTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: CLIPTextConfig, @@ -605,34 +612,19 @@ class CLIPTextTransformer(nn.Module): return last_hidden_state def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class CLIPVisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: CLIPVisionConfig, @@ -714,42 +706,21 @@ class CLIPVisionTransformer(nn.Module): return encoder_outputs def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - layer_count = len(self.encoder.layers) + skip_prefixes: list[str] = [] + if self.post_layernorm is None: + skip_prefixes.append("post_layernorm.") + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) - for name, loaded_weight in weights: - # post_layernorm is not needed in CLIPVisionModel - if name.startswith("post_layernorm") and self.post_layernorm is None: - continue - - # omit layers when num_hidden_layers_override is set - if name.startswith("encoder.layers"): - layer_idx = int(name.split(".")[2]) - if layer_idx >= layer_count: + # Drop layers beyond num_hidden_layers_override. + def _filter(ws): + for name, w in ws: + if name.startswith("encoder.layers.") and int( + name.split(".")[2] + ) >= len(self.encoder.layers): continue + yield name, w - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper) class CLIPVisionModel(nn.Module): diff --git a/vllm/model_executor/models/cohere2_moe.py b/vllm/model_executor/models/cohere2_moe.py index 3869a06569..80dc680206 100644 --- a/vllm/model_executor/models/cohere2_moe.py +++ b/vllm/model_executor/models/cohere2_moe.py @@ -18,7 +18,6 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -31,8 +30,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, row_parallel_weight_loader, ) from vllm.model_executor.utils import set_weight_attrs @@ -43,8 +40,8 @@ from .commandr import LayerNorm from .interfaces import SupportsPP, SupportsQuant from .utils import ( AutoWeightsLoader, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -462,88 +459,23 @@ class Cohere2MoeModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - expert_params_mapping = fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - for param_name, shard_name, shard_id in stacked_params_mapping: - if shard_name not in name: - continue - if "mlp.experts" in name: - continue - name = name.replace(shard_name, param_name) - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name, self): - continue - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params - class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant): is_text_generation_model = True + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + ) packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -599,4 +531,4 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self, skip_prefixes=["lm_head."]) - return loader.load_weights(weights) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/deepseek_eagle3.py b/vllm/model_executor/models/deepseek_eagle3.py index 492081fd66..127734808d 100644 --- a/vllm/model_executor/models/deepseek_eagle3.py +++ b/vllm/model_executor/models/deepseek_eagle3.py @@ -20,10 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.deepseek_v2 import ( DeepseekV2ForCausalLM, DeepseekV2MLAAttention, @@ -34,6 +30,7 @@ from vllm.multimodal.inputs import NestedTensors from .interfaces import LocalArgmaxMixin from .utils import ( AutoWeightsLoader, + WeightsMapper, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -270,44 +267,20 @@ class DeepseekV2Eagle3Model(nn.Module): return hidden_states, aux_output + # midlayer rename + gate_up / MLA fused_qkv_a merges + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={"midlayer.": "layers.0."}, + orig_to_new_stacked={ + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + ".q_a_proj": (".fused_qkv_a_proj", 0), + ".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1), + }, + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - (".fused_qkv_a_proj", ".q_a_proj", 0), - (".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - if "midlayer." in name: - name = name.replace("midlayer.", "layers.0.") - - # Remapping the name FP8 kv-scale - if "scale" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM): diff --git a/vllm/model_executor/models/dots_ocr.py b/vllm/model_executor/models/dots_ocr.py index b5c1616acf..65298a5fd3 100644 --- a/vllm/model_executor/models/dots_ocr.py +++ b/vllm/model_executor/models/dots_ocr.py @@ -32,7 +32,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, SupportsLoRA, @@ -355,36 +354,6 @@ class DotsSwiGLUFFN(nn.Module): x, _ = self.fc2(x) return x - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("fc13", "fc1", 0), - ("fc13", "fc3", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class DotsPatchEmbed(nn.Module): def __init__(self, config): @@ -622,6 +591,10 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA ".attn.qkv_proj.": ".attn.qkv.", ".attn.out_proj.": ".attn.proj.", }, + orig_to_new_stacked={ + ".fc1.": (".fc13.", 0), + ".fc3.": (".fc13.", 1), + }, orig_to_new_prefix={ "lm_head.": "language_model.lm_head.", "model.": "language_model.model.", diff --git a/vllm/model_executor/models/ernie_mtp.py b/vllm/model_executor/models/ernie_mtp.py index ef37fd3555..b57da3698a 100644 --- a/vllm/model_executor/models/ernie_mtp.py +++ b/vllm/model_executor/models/ernie_mtp.py @@ -27,7 +27,6 @@ from collections.abc import Iterable import torch import torch.nn as nn -from transformers import PretrainedConfig from vllm.config import VllmConfig from vllm.model_executor.layers.layernorm import RMSNorm @@ -36,11 +35,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .llama import LlamaDecoderLayer -from .utils import is_pp_missing_parameter, maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix class ErnieMultiTokenPredictorLayer(nn.Module): @@ -147,6 +145,28 @@ class ErnieMTP(nn.Module): super().__init__() self.config = vllm_config.model_config.hf_config + # MTP weights are stored under a flat `mtp_*.0.` block in the + # checkpoint; rewrite them into `model.layers.{spec_layer}.*`. + spec_layer = self.config.num_hidden_layers + self.hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + "model.mtp_emb_norm.0.": f"model.layers.{spec_layer}.mtp_emb_norm.", + "model.mtp_hidden_norm.0.": ( + f"model.layers.{spec_layer}.mtp_hidden_norm." + ), + "model.mtp_linear_proj.0.": ( + f"model.layers.{spec_layer}.mtp_linear_proj." + ), + "model.mtp_block.0.": f"model.layers.{spec_layer}.mtp_block.", + }, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) self.model = ErnieMultiTokenPredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") ) @@ -185,94 +205,15 @@ class ErnieMTP(nn.Module): return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] + # Checkpoint bundles the full base model; only MTP, embed_tokens and + # lm_head weights belong to this module. + def _filter( + weights: Iterable[tuple[str, torch.Tensor]], + ) -> Iterable[tuple[str, torch.Tensor]]: + for name, weight in weights: + if any(k in name for k in ("mtp", "embed_tokens", "lm_head")): + yield name, weight - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if self.config.tie_word_embeddings and name.endswith("lm_head.weight"): - continue - if "rotary_emb.inv_freq" in name: - continue - if "mtp" in name: - name = self._rewrite_spec_layer_name(self.config, name) - - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - if "mtp" not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - - # According to DeepSeek-V3 Technical Report, MTP modules - # shares embedding layer. We only load the first weights. - if "mtp_" not in name and ( - "embed_tokens" not in name and "lm_head" not in name - ): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - def _rewrite_spec_layer_name(self, config: PretrainedConfig, name: str) -> str: - """ - Rewrite the weight name to match the format of the original model. - """ - spec_layer_weight_names = [ - "embed_tokens", - "mtp_emb_norm", - "mtp_hidden_norm", - "mtp_linear_proj", - ] - layer_idx = config.num_hidden_layers - for weight_name in spec_layer_weight_names: - if weight_name in name: - name = name.replace( - f"model.{weight_name}.0.", - f"model.layers.{layer_idx}.{weight_name}.", - ) - return name - name = name.replace( - "model.mtp_block.0.", f"model.layers.{layer_idx}.mtp_block." - ) - return name + skip_prefixes = ["lm_head"] if self.config.tie_word_embeddings else [] + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) + return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index 18900557f6..086040e2ea 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -15,8 +15,7 @@ # limitations under the License. """Inference-only K-EXAONE-236B-A22B model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice import torch @@ -32,7 +31,6 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear @@ -43,10 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from .exaone4 import Exaone4Attention as ExaoneMoeAttention @@ -55,8 +49,8 @@ from .interfaces import SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -325,142 +319,21 @@ class ExaoneMoeModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - - # Skip loading extra parameters for GPTQ/modelopt models. - ignore_suffixes = ( - ".bias", - "_bias", - ".k_scale", - "_k_scale", - ".v_scale", - "_v_scale", - ".weight_scale", - "_weight_scale", - ".input_scale", - "_input_scale", - ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if name.startswith("mtp."): - continue - if "rotary_emb.inv_freq" in name: - continue - if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: - # Models trained using ColossalAI may include these tensors in - # the checkpoint. Skip them. - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - if "mlp.experts" in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - - # Skip loading extra parameters for GPTQ/modelopt models. - if ( - name_mapped.endswith(ignore_suffixes) - and name_mapped not in params_dict - ): - continue - - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip loading extra parameters for GPTQ/modelopt models. - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0), + ".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1), + } + ) packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -554,5 +427,18 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP): skip_prefixes=( ["lm_head.", "mtp."] if self.config.tie_word_embeddings else ["mtp."] ), + # Skip loading extra parameters for GPTQ/modelopt models. + ignore_unexpected_suffixes=[ + ".bias", + "_bias", + ".k_scale", + "_k_scale", + ".v_scale", + "_v_scale", + ".weight_scale", + "_weight_scale", + ".input_scale", + "_input_scale", + ], ) - return loader.load_weights(weights) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/exaone_moe_mtp.py b/vllm/model_executor/models/exaone_moe_mtp.py index b3f8552aac..a37da487db 100644 --- a/vllm/model_executor/models/exaone_moe_mtp.py +++ b/vllm/model_executor/models/exaone_moe_mtp.py @@ -18,15 +18,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.exaone_moe import ExaoneMoeDecoderLayer from vllm.sequence import IntermediateTensors -from .utils import ( - AutoWeightsLoader, - is_pp_missing_parameter, - maybe_prefix, -) +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix logger = init_logger(__name__) @@ -35,6 +30,17 @@ KVCache = tuple[torch.Tensor, torch.Tensor] @support_torch_compile class ExaoneMoeMultiTokenPredictor(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # Scope to dense mlp; experts are handled separately. + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -128,55 +134,8 @@ class ExaoneMoeMultiTokenPredictor(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - - if "mlp.experts" in name: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @support_torch_compile diff --git a/vllm/model_executor/models/falcon_h1.py b/vllm/model_executor/models/falcon_h1.py index b837dc010d..3c96d00c28 100644 --- a/vllm/model_executor/models/falcon_h1.py +++ b/vllm/model_executor/models/falcon_h1.py @@ -35,10 +35,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.config import set_default_rope_theta @@ -52,7 +48,7 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -496,63 +492,6 @@ class FalconH1Model(nn.Module): hidden_states = self.final_layernorm(hidden_states) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - if "A_log" in name: - name = name.replace("A_log", "A") - - if "mamba" in name: - name = name.replace("mamba", "mamba.mamba") - - if "scale" in name: - # Remapping the name of kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params - class FalconH1ForCausalLM( nn.Module, @@ -567,6 +506,20 @@ class FalconH1ForCausalLM( "gate_up_proj": ["gate_proj", "up_proj"], } + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + "A_log": "A", + "mamba": "mamba.mamba", + }, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + embedding_modules = { "embed_tokens": "input_embeddings", "lm_head": "output_embeddings", @@ -694,4 +647,4 @@ class FalconH1ForCausalLM( self, skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None), ) - return loader.load_weights(weights) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index f4df0da8cc..e128b22e8e 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -31,7 +31,6 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.whisper_utils import ( ISO639_1_SUPPORTED_LANGS, ) @@ -618,6 +617,14 @@ class FunASRAudioInputs(TensorSchema): class FunASREncoder(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".self_attn.q_proj": (".self_attn.qkv", "q"), + ".self_attn.k_proj": (".self_attn.qkv", "k"), + ".self_attn.v_proj": (".self_attn.qkv", "v"), + } + ) + def __init__( self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False ): @@ -637,35 +644,8 @@ class FunASREncoder(nn.Module): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load weights with mapping from HuggingFace format.""" - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("self_attn.qkv.", "self_attn.q_proj.", "q"), - ("self_attn.qkv.", "self_attn.k_proj.", "k"), - ("self_attn.qkv.", "self_attn.v_proj.", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict.get(name) - if param is not None: - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class FunASRModel(nn.Module): diff --git a/vllm/model_executor/models/gemma4_mtp.py b/vllm/model_executor/models/gemma4_mtp.py index 03961cac19..b30ab4b7ae 100644 --- a/vllm/model_executor/models/gemma4_mtp.py +++ b/vllm/model_executor/models/gemma4_mtp.py @@ -44,7 +44,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .gemma4 import Gemma4MLP, _get_text_config @@ -412,44 +411,6 @@ class Gemma4MultiTokenPredictor(nn.Module): def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) * self.normalizer - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - params_dict.update(dict(self.named_buffers())) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - - return loaded_params - def forward( self, input_ids: torch.Tensor, @@ -502,6 +463,10 @@ class Gemma4MTP(nn.Module): "pre_projection.": "model.pre_projection.", "post_projection.": "model.post_projection.", }, + orig_to_new_stacked={ + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, ) def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): diff --git a/vllm/model_executor/models/glm4_1v.py b/vllm/model_executor/models/glm4_1v.py index d4ed8a9c85..d122427542 100644 --- a/vllm/model_executor/models/glm4_1v.py +++ b/vllm/model_executor/models/glm4_1v.py @@ -76,7 +76,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -618,6 +617,16 @@ class Glm4vVisionEmbeddings(nn.Module): class Glm4vVisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".attn.q.": (".attn.qkv.", "q"), + ".attn.k.": (".attn.qkv.", "k"), + ".attn.v.": (".attn.qkv.", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + } + ) + def __init__( self, text_config: Glm4vTextConfig, @@ -958,33 +967,8 @@ class Glm4vVisionTransformer(nn.Module): return x def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("attn.qkv.", "attn.q.", "q"), - ("attn.qkv.", "attn.k.", "k"), - ("attn.qkv.", "attn.v.", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Glm4vProcessingInfo(BaseProcessingInfo): diff --git a/vllm/model_executor/models/hunyuan_vision.py b/vllm/model_executor/models/hunyuan_vision.py index 5125406ab3..313e13c915 100644 --- a/vllm/model_executor/models/hunyuan_vision.py +++ b/vllm/model_executor/models/hunyuan_vision.py @@ -48,7 +48,6 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -431,6 +430,14 @@ class HunYuanVisionPatchMerger(nn.Module): class HunYuanVisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv", "q"), + ".k_proj": (".qkv", "k"), + ".v_proj": (".qkv", "v"), + } + ) + def __init__( self, vision_config: HunYuanVLVisionConfig, @@ -529,31 +536,8 @@ class HunYuanVisionTransformer(nn.Module): return image_embeds_list def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv", ".q_proj", "q"), - (".qkv", ".k_proj", "k"), - (".qkv", ".v_proj", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]): diff --git a/vllm/model_executor/models/idefics2_vision_model.py b/vllm/model_executor/models/idefics2_vision_model.py index 7db2e823fb..81e7e51eb9 100644 --- a/vllm/model_executor/models/idefics2_vision_model.py +++ b/vllm/model_executor/models/idefics2_vision_model.py @@ -38,8 +38,8 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from .utils import AutoWeightsLoader, WeightsMapper from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model @@ -352,6 +352,14 @@ class Idefics2Encoder(nn.Module): class Idefics2VisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: Idefics2VisionConfig, @@ -453,42 +461,22 @@ class Idefics2VisionTransformer(nn.Module): return last_hidden_state def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() + # head is a pooling header absent from this model. + skip_prefixes = ["head."] + if not self.require_post_norm: + skip_prefixes.append("post_layernorm.") + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) + layer_count = len(self.encoder.layers) - for name, loaded_weight in weights: - # skip pooling header - if name.startswith("head."): - continue - - # post_layernorm is optional - if name.startswith("post_layernorm.") and not self.require_post_norm: - continue - - # omit layers when num_hidden_layers_override is set - if name.startswith("encoder.layers."): - layer_idx = int(name.split(".")[2]) - if layer_idx >= layer_count: + def _filter(ws: Iterable[tuple[str, torch.Tensor]]): + # Drop layers beyond num_hidden_layers_override. + for name, w in ws: + if ( + name.startswith("encoder.layers.") + and int(name.split(".")[2]) >= layer_count + ): continue + yield name, w - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name or self.use_data_parallel: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/isaac.py b/vllm/model_executor/models/isaac.py index 412948c482..87932b5032 100644 --- a/vllm/model_executor/models/isaac.py +++ b/vllm/model_executor/models/isaac.py @@ -25,9 +25,6 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, -) from vllm.model_executor.models.interfaces import ( MultiModalEmbeddings, SupportsLoRA, @@ -652,6 +649,14 @@ class Siglip2Encoder(nn.Module): class Siglip2VisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: PixelShuffleSiglip2VisionConfig, @@ -715,31 +720,8 @@ class Siglip2VisionTransformer(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def _resolve_vision_token_id(model_config: ModelConfig, vision_token: str) -> int: diff --git a/vllm/model_executor/models/keye.py b/vllm/model_executor/models/keye.py index 8a0b7ceea5..dd1fb892ad 100644 --- a/vllm/model_executor/models/keye.py +++ b/vllm/model_executor/models/keye.py @@ -34,10 +34,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -76,7 +72,6 @@ from .utils import ( AutoWeightsLoader, WeightsMapper, init_vllm_registered_model, - is_pp_missing_parameter, maybe_prefix, ) from .vision import is_vit_use_data_parallel @@ -718,6 +713,14 @@ class KeyeSiglipVisionModel(nn.Module): config_class = PretrainedConfig main_input_name = "pixel_values" + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: PretrainedConfig, @@ -776,53 +779,8 @@ class KeyeSiglipVisionModel(nn.Module): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "head.attention" in name or "head.layernorm" in name: - continue - if "head.mlp" in name or "head.probe" in name: - continue - for ( - param_name, - weight_name, - shard_id, - ) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr( - param, - "weight_loader", - default_weight_loader, - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self, skip_prefixes=["vision_model.head."]) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Projector(nn.Module): diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index 6232c39a44..cb3c83e708 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -17,7 +17,6 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import PromptType, TokensPrompt from vllm.model_executor.model_loader import DefaultModelLoader -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import ( SupportsMultiModal, SupportsPP, @@ -81,6 +80,14 @@ class KimiAudioWhisperEncoder(WhisperEncoder): "qkv_proj": ["q_proj", "k_proj", "v_proj"], } + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".self_attn.q_proj": (".self_attn.qkv_proj", "q"), + ".self_attn.k_proj": (".self_attn.qkv_proj", "k"), + ".self_attn.v_proj": (".self_attn.qkv_proj", "v"), + }, + ) + def __init__( self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False ): @@ -102,37 +109,8 @@ class KimiAudioWhisperEncoder(WhisperEncoder): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) # ----------------------------------------------------------------------------- diff --git a/vllm/model_executor/models/lfm2.py b/vllm/model_executor/models/lfm2.py index 95a8cdb871..601bd22b2f 100644 --- a/vllm/model_executor/models/lfm2.py +++ b/vllm/model_executor/models/lfm2.py @@ -32,7 +32,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP, SupportsQuant @@ -41,7 +40,6 @@ from .utils import ( PPMissingLayer, WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -297,6 +295,20 @@ class Lfm2ShortConvDecoderLayer(nn.Module): @support_torch_compile class Lfm2Model(nn.Module): + # HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision + # with the inner .conv.conv child (ShortConv has a child self.conv, so + # naming the container .conv too makes _match_target_modules match both). + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={".conv.": ".short_conv."}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".w1": (".w13", 0), + ".w3": (".w13", 1), + }, + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -375,40 +387,8 @@ class Lfm2Model(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".w13", ".w1", 0), - (".w13", ".w3", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if ".conv." in name: - name = name.replace(".conv.", ".short_conv.", 1) - - for param_name, weight_name, shard_id in stacked_params_mapping: - # Use segment-boundary matching (trailing dot) to prevent - # e.g. ".w1" from matching inside ".w13" in pre-fused keys. - if weight_name + "." not in name: - continue - name = name.replace(weight_name + ".", param_name + ".") - - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Lfm2ForCausalLM( @@ -427,12 +407,8 @@ class Lfm2ForCausalLM( "in_proj": ["in_proj"], } - # HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision - # with the inner .conv.conv child (ShortConv has a child self.conv, so - # naming the container .conv too makes _match_target_modules match both) - hf_to_vllm_mapper = WeightsMapper( - orig_to_new_substr={".conv.": ".short_conv."}, - ) + # Reuse the backbone mapper so LoRA/quantization see the same name mapping. + hf_to_vllm_mapper = Lfm2Model.hf_to_vllm_mapper # LoRA specific attributes embedding_modules = { diff --git a/vllm/model_executor/models/lfm2_siglip2.py b/vllm/model_executor/models/lfm2_siglip2.py index cb51c6bd8c..f1679af813 100644 --- a/vllm/model_executor/models/lfm2_siglip2.py +++ b/vllm/model_executor/models/lfm2_siglip2.py @@ -23,9 +23,8 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader -from .utils import maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix from .vision import ( is_vit_use_data_parallel, resolve_visual_encoder_outputs, @@ -458,6 +457,14 @@ class Siglip2VisionTransformer(nn.Module): class Siglip2Model(torch.nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: Siglip2VisionConfig, @@ -501,42 +508,21 @@ class Siglip2Model(torch.nn.Module): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() + skip_prefixes = [] + if self.vision_model.post_layernorm is None: + skip_prefixes.append("vision_model.post_layernorm.") + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) + + # Drop layers omitted by num_hidden_layers_override. layer_count = len(self.vision_model.encoder.layers) - for name, loaded_weight in weights: - # post_layernorm is optional in Siglip2Model - if ( - name.startswith("vision_model.post_layernorm") - and self.vision_model.post_layernorm is None - ): - continue - - # omit layers when num_hidden_layers_override is set - if name.startswith("vision_model.encoder.layers"): - layer_idx = int(name.split(".")[3]) - if layer_idx >= layer_count: + def _filter(ws): + for n, w in ws: + if ( + n.startswith("vision_model.encoder.layers.") + and int(n.split(".")[3]) >= layer_count + ): continue + yield n, w - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/llama4_eagle.py b/vllm/model_executor/models/llama4_eagle.py index 068a15b625..d94dbb0496 100644 --- a/vllm/model_executor/models/llama4_eagle.py +++ b/vllm/model_executor/models/llama4_eagle.py @@ -32,18 +32,33 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM from vllm.model_executor.models.utils import extract_layer_index from .interfaces import SupportsMultiModal -from .utils import AutoWeightsLoader, maybe_prefix, process_eagle_weight +from .utils import ( + AutoWeightsLoader, + WeightsMapper, + maybe_prefix, + process_eagle_weight, +) logger = init_logger(__name__) @support_torch_compile class LlamaModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={"model.": ""}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + def __init__( self, *, @@ -108,34 +123,8 @@ class LlamaModel(nn.Module): return hidden_states, hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - name = name.removeprefix("model.") - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - for name in params_dict: - assert name in loaded_params, f"{name} is not loaded!" - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def validate_and_update_config( self, start_layer_id: int, quant_config: QuantizationConfig | None = None diff --git a/vllm/model_executor/models/llama_eagle.py b/vllm/model_executor/models/llama_eagle.py index 14842a75fe..5d13b29c0c 100644 --- a/vllm/model_executor/models/llama_eagle.py +++ b/vllm/model_executor/models/llama_eagle.py @@ -14,14 +14,11 @@ from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization.base_config import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from .utils import ( AutoWeightsLoader, + WeightsMapper, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -53,6 +50,17 @@ class LlamaDecoderLayer(LlamaDecoderLayer): @support_torch_compile class LlamaModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + } + ) + def __init__( self, *, @@ -116,36 +124,8 @@ class LlamaModel(nn.Module): return hidden_states, hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - # Remapping the name FP8 kv-scale or zero point. - if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class EagleLlamaForCausalLM(LlamaForCausalLM): diff --git a/vllm/model_executor/models/llama_eagle3.py b/vllm/model_executor/models/llama_eagle3.py index bb1bbb8553..8ce86c7780 100644 --- a/vllm/model_executor/models/llama_eagle3.py +++ b/vllm/model_executor/models/llama_eagle3.py @@ -18,15 +18,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM from vllm.multimodal.inputs import NestedTensors from .utils import ( AutoWeightsLoader, + WeightsMapper, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -253,39 +250,20 @@ class LlamaModel(nn.Module): return hidden_states, aux_output + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={"midlayer.": "layers.0."}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "midlayer." in name: - name = name.replace("midlayer.", "layers.0.") - # Remapping the name FP8 kv-scale or zero point. - if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Eagle3LlamaForCausalLM(LlamaForCausalLM): diff --git a/vllm/model_executor/models/mimo_mtp.py b/vllm/model_executor/models/mimo_mtp.py index 3558ddf39b..32d3b1f662 100644 --- a/vllm/model_executor/models/mimo_mtp.py +++ b/vllm/model_executor/models/mimo_mtp.py @@ -34,11 +34,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen2 import Qwen2DecoderLayer from vllm.sequence import IntermediateTensors -from .utils import maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix class MiMoMultiTokenPredictorLayer(nn.Module): @@ -163,6 +162,28 @@ class MiMoMTP(nn.Module): self.config.hidden_size, prefix=maybe_prefix(prefix, "lm_head"), ) + # Checkpoint stores MTP layers 0-indexed and without the `mtp_block` + # wrapper around the transformer block; remap onto the offset index. + start = self.config.num_hidden_layers + self.hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={ + ".self_attn.": ".mtp_block.self_attn.", + ".mlp.": ".mtp_block.mlp.", + ".input_layernorm.": ".mtp_block.input_layernorm.", + ".post_attention_layernorm.": ".mtp_block.post_attention_layernorm.", + }, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + orig_to_new_prefix={ + f"model.mtp_layers.{i}.": f"model.mtp_layers.{i + start}." + for i in range(self.config.num_nextn_predict_layers) + }, + ) def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) @@ -190,105 +211,12 @@ class MiMoMTP(nn.Module): return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] + # The checkpoint carries the full model; keep only the MTP layers and + # the shared embedding/head. + def mtp_weights(): + for name, weight in weights: + if "mtp_layers" in name or "embed_tokens" in name or "lm_head" in name: + yield name, weight - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - name = self.map_model_name_to_mtp_param_name(name) - - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - if "mtp_layers" not in name: - break - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if "mtp_layers" not in name and ( - "embed_tokens" not in name and "lm_head" not in name - ): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - - def map_model_name_to_mtp_param_name(self, name: str) -> str: - import regex as re - - # append mtp_start_layer_idx - pattern = r"(model\.mtp_layers\.)(\d+)(\.)" - match = re.match(pattern, name) - if match: - original_num = int(match.group(2)) - new_num = original_num + self.config.num_hidden_layers - name = name.replace(match.group(), f"{match.group(1)}{new_num}.") - # check for early turn - name_without_prefix = [ - "token_layernorm", - "hidden_layernorm", - "input_proj", - "final_layernorm", - ] - for sub_name in name_without_prefix: - if sub_name in name: - return name - # add mtp_block - pattern = r"(model\.mtp_layers\.\d+\.)" - match = re.match(pattern, name) - if match: - name = name.replace(match.group(), match.group() + "mtp_block.") - return name - - def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: - """ - Rewrite the weight name to match the format of the original model. - Add .mtp_block for modules in transformer layer block for spec layer - """ - spec_layer_weight_names = [ - "embed_tokens", - "enorm", - "hnorm", - "eh_proj", - "shared_head", - ] - spec_layer_weight = False - for weight_name in spec_layer_weight_names: - if weight_name in name: - spec_layer_weight = True - break - if not spec_layer_weight: - # treat rest weights as weights for transformer layer block - name = name.replace( - f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." - ) - return name + loader = AutoWeightsLoader(self) + return loader.load_weights(mtp_weights(), mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/mimo_v2_omni.py b/vllm/model_executor/models/mimo_v2_omni.py index 1cd2c6919a..d0d9589ae1 100644 --- a/vllm/model_executor/models/mimo_v2_omni.py +++ b/vllm/model_executor/models/mimo_v2_omni.py @@ -29,7 +29,6 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.vision import is_vit_use_data_parallel from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems @@ -378,6 +377,13 @@ class MiMoVisionBlock(nn.Module): class MiMoVisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + "mlp.gate_proj": ("mlp.gate_up_proj", 0), + "mlp.up_proj": ("mlp.gate_up_proj", 1), + } + ) + def __init__( self, vision_cfg: PretrainedConfig, @@ -627,28 +633,8 @@ class MiMoVisionTransformer(nn.Module): return x def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("mlp.gate_up_proj", "mlp.gate_proj", 0), - ("mlp.gate_up_proj", "mlp.up_proj", 1), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class MiMoV2OmniProcessingInfo(BaseProcessingInfo): diff --git a/vllm/model_executor/models/minicpmv4_6.py b/vllm/model_executor/models/minicpmv4_6.py index e1cb7c1d2d..79ac79d709 100644 --- a/vllm/model_executor/models/minicpmv4_6.py +++ b/vllm/model_executor/models/minicpmv4_6.py @@ -25,7 +25,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateShapeCalculator, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, @@ -619,6 +618,14 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo): class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + "q_proj": ("qkv_proj", "q"), + "k_proj": ("qkv_proj", "k"), + "v_proj": ("qkv_proj", "v"), + } + ) + def __init__( self, config, @@ -667,31 +674,8 @@ class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module): return out def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - mapped_name = name.replace(weight_name, param_name, 1) - if mapped_name not in params_dict: - continue - param = params_dict[mapped_name] - param.weight_loader(param, loaded_weight, shard_id) - break - else: - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module): diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index e5da915415..79bd8f439e 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -41,7 +41,6 @@ from vllm.distributed import ( from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear from vllm.model_executor.layers.layernorm import RMSNorm @@ -57,17 +56,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -111,6 +106,7 @@ class MiniMaxM2MoE(nn.Module): quant_config=quant_config, prefix=f"{prefix}.experts", router_logits_dtype=torch.float32, + ckpt_names=("w1", "w2", "w3"), ) self.gate = GateLinear( @@ -327,6 +323,15 @@ class MiniMaxM2DecoderLayer(nn.Module): class MiniMaxM2Model(nn.Module, EagleModelMixin): fall_back_to_pt_during_load = False + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -409,115 +414,23 @@ class MiniMaxM2Model(nn.Module, EagleModelMixin): return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="w1", - ckpt_down_proj_name="w2", - ckpt_up_proj_name="w3", - num_experts=self.config.num_local_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = self.get_expert_mapping() - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) - if spec_layer is not None: - continue # skip spec decode layers for main model - - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if ("mlp.experts." in name) and name not in params_dict: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - if is_pp_missing_parameter(name, self): - continue - - # Remap qkv_proj.[kv]_scale to attn.[kv]_scale - if name.endswith((".k_scale", ".v_scale")): - remapped_name = maybe_remap_kv_scale_name(name, params_dict) - if remapped_name is not None and remapped_name in params_dict: - param = params_dict[remapped_name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - break - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + # Skip spec-decode (MTP) layers; they are appended after the main + # decoder layers and have no destination in the main model. + skip_prefixes = None + num_mtp = getattr(self.config, "num_mtp_modules", 0) + if num_mtp: + base = self.config.num_hidden_layers + skip_prefixes = [f"layers.{base + i}." for i in range(num_mtp)] + loader = AutoWeightsLoader( + self, + skip_prefixes=skip_prefixes, + ) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3): + hf_to_vllm_mapper = MiniMaxM2Model.hf_to_vllm_mapper packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -577,17 +490,3 @@ class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - - -def get_spec_layer_idx_from_weight_name( - config: PretrainedConfig, weight_name: str -) -> int | None: - if hasattr(config, "num_mtp_modules") and (config.num_mtp_modules > 0): - layer_idx = config.num_hidden_layers - for i in range(config.num_mtp_modules): - if weight_name.startswith(f"model.layers.{layer_idx + i}."): - return layer_idx + i - return None diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py index 57eb820ad9..8305ecc330 100644 --- a/vllm/model_executor/models/mixtral.py +++ b/vllm/model_executor/models/mixtral.py @@ -24,8 +24,7 @@ # limitations under the License. """Inference-only Mixtral model.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice import torch @@ -42,7 +41,6 @@ from vllm.distributed import ( from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -57,17 +55,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, PPMissingLayer, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -142,6 +136,7 @@ class MixtralMoE(nn.Module): prefix=f"{prefix}.experts", enable_eplb=self.enable_eplb, num_redundant_experts=self.n_redundant_experts, + ckpt_names=("w1", "w2", "w3"), ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -295,6 +290,15 @@ class MixtralDecoderLayer(nn.Module): @support_torch_compile class MixtralModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -364,114 +368,15 @@ class MixtralModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="w1", - ckpt_down_proj_name="w2", - ckpt_up_proj_name="w3", - num_experts=self.config.num_local_experts, - num_redundant_experts=self.num_redundant_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name.endswith("scale"): - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - - if weight_name not in name: - continue - - is_expert_weight = True - name_mapped = name.replace(weight_name, param_name) - - # Skip layers on other devices. - if is_pp_missing_parameter(name_mapped, self): - continue - - if ( - name_mapped.endswith(".bias") or name_mapped.endswith("_bias") - ) and name_mapped not in params_dict: - continue - - param = params_dict[name_mapped] - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - continue - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts): fall_back_to_pt_during_load = False + hf_to_vllm_mapper = MixtralModel.hf_to_vllm_mapper packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -582,6 +487,3 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/molmo.py b/vllm/model_executor/models/molmo.py index 1d756a2add..b3279e7dbd 100644 --- a/vllm/model_executor/models/molmo.py +++ b/vllm/model_executor/models/molmo.py @@ -672,6 +672,14 @@ class MolmoDecoderNormAfterLayer(MolmoDecoderLayer): class MolmoVisionBackbone(nn.Module, SupportsQuant): packed_modules_mapping = {"merged_linear": ["gate_proj", "up_proj"]} + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # image_projector gate_up merge + "gate_proj": ("merged_linear", 0), + "up_proj": ("merged_linear", 1), + } + ) + def __init__( self, config: PretrainedConfig, @@ -800,38 +808,8 @@ class MolmoVisionBackbone(nn.Module, SupportsQuant): return image_features def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("merged_linear", "gate_proj", 0), - ("merged_linear", "up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @support_torch_compile diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 9ad3810e41..22a2b0cf32 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -709,6 +709,20 @@ class Molmo2VisionBackbone(nn.Module, SupportsQuant): "merged_linear": ["gate_proj", "up_proj"], } + # Runs after the top-level mapper, so image_pooling_2d/image_projector + # source names are already renamed to q/k/v_proj and gate/up_proj. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + "wq": ("merged_qkv", "q"), + "wk": ("merged_qkv", "k"), + "wv": ("merged_qkv", "v"), + "k_proj": ("merged_kv", 0), + "v_proj": ("merged_kv", 1), + "gate_proj": ("merged_linear", 0), + "up_proj": ("merged_linear", 1), + }, + ) + def __init__( self, vit_config: VitConfig, @@ -839,43 +853,8 @@ class Molmo2VisionBackbone(nn.Module, SupportsQuant): ] def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("merged_qkv", "wq", "q"), - ("merged_qkv", "wk", "k"), - ("merged_qkv", "wv", "v"), - ("merged_kv", "k_proj", 0), - ("merged_kv", "v_proj", 1), - ("merged_linear", "gate_proj", 0), - ("merged_linear", "up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Molmo2Attention(nn.Module): diff --git a/vllm/model_executor/models/moss_audio.py b/vllm/model_executor/models/moss_audio.py index 00a7047c6c..fdfe982fca 100644 --- a/vllm/model_executor/models/moss_audio.py +++ b/vllm/model_executor/models/moss_audio.py @@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( AudioItem, @@ -742,37 +741,6 @@ class GatedMLP(nn.Module): x, _ = self.down_proj(x) return x - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - target_name = name - for param_name, weight_name, shard_id in stacked_params_mapping: - components = target_name.split(".") - if weight_name not in components: - continue - - target_name = ".".join( - param_name if component == weight_name else component - for component in components - ) - param = params_dict[target_name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[target_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - - loaded_params.add(target_name) - return loaded_params - @support_torch_compile( dynamic_arg_dims={ @@ -1478,7 +1446,11 @@ class MossAudioModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA): "language_model.embed_tokens.": "language_model.model.embed_tokens.", "language_model.layers.": "language_model.model.layers.", "language_model.norm.": "language_model.model.norm.", - } + }, + orig_to_new_stacked={ + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, ) def get_mm_mapping(self) -> MultiModelKeys: diff --git a/vllm/model_executor/models/olmoe.py b/vllm/model_executor/models/olmoe.py index 5b661aa4e4..a57ef9cf43 100644 --- a/vllm/model_executor/models/olmoe.py +++ b/vllm/model_executor/models/olmoe.py @@ -34,7 +34,6 @@ from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -49,13 +48,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -338,106 +336,16 @@ class OlmoeModel(nn.Module): hidden_states = self.norm(hidden_states) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if "mlp.experts" in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Remapping the name of FP8 kv-scale. - if name.endswith("kv_scale"): - remapped_kv_scale_name = name.replace( - ".kv_scale", ".attn.kv_scale" - ) - if remapped_kv_scale_name not in params_dict: - logger.warning_once( - "Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501 - name, - remapped_kv_scale_name, - ) - continue - else: - name = remapped_kv_scale_name - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class OlmoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -496,7 +404,4 @@ class OlmoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) - return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/paddleocr_vl.py b/vllm/model_executor/models/paddleocr_vl.py index 0bf10e3ce7..0dae22115d 100644 --- a/vllm/model_executor/models/paddleocr_vl.py +++ b/vllm/model_executor/models/paddleocr_vl.py @@ -48,10 +48,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFeatureSpec, @@ -81,7 +77,6 @@ from .utils import ( AutoWeightsLoader, PPMissingLayer, WeightsMapper, - is_pp_missing_parameter, maybe_prefix, ) from .vision import get_vit_attn_backend @@ -884,6 +879,14 @@ class SiglipVisionTransformer(nn.Module): class SiglipVisionModel(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config, @@ -928,55 +931,19 @@ class SiglipVisionModel(nn.Module): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - if "head.attention" in name or "head.layernorm" in name: - continue - if "head.mlp" in name or "head.probe" in name: - continue - if "packing_position_embedding" in name: - continue - for ( - param_name, - weight_name, - shard_id, - ) in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if name.endswith(".bias") and name not in params_dict: - continue - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = getattr( - param, - "weight_loader", - default_weight_loader, - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + # Skip the SigLIP attention pooling head and packing pos embedding + # present in the checkpoint but absent from this vision tower. + loader = AutoWeightsLoader( + self, + skip_substrs=[ + "head.attention", + "head.layernorm", + "head.mlp", + "head.probe", + "packing_position_embedding", + ], + ) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @MULTIMODAL_REGISTRY.register_processor( diff --git a/vllm/model_executor/models/phimoe.py b/vllm/model_executor/models/phimoe.py index a49e8ce2e8..dc2af1e8d6 100644 --- a/vllm/model_executor/models/phimoe.py +++ b/vllm/model_executor/models/phimoe.py @@ -37,7 +37,6 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -51,16 +50,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -289,6 +284,7 @@ class PhiMoE(nn.Module): tp_size=tp_size, custom_routing_function=phimoe_routing_function, prefix=f"{prefix}.experts", + ckpt_names=("w1", "w2", "w3"), ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -516,84 +512,18 @@ class PhiMoEModel(nn.Module): hidden_states = self.norm(hidden_states) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="w1", - ckpt_down_proj_name="w2", - ckpt_up_proj_name="w3", - num_experts=self.config.num_local_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP): fall_back_to_pt_during_load = False + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -654,7 +584,4 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP): def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) - return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index ebc51e9683..c987e07b43 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -65,7 +65,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.evs import ( @@ -613,6 +612,16 @@ class Qwen2_5_VisionPatchMerger(nn.Module): class Qwen2_5_VisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".attn.q.": (".attn.qkv.", "q"), + ".attn.k.": (".attn.qkv.", "k"), + ".attn.v.": (".attn.qkv.", "v"), + ".mlp.gate_proj.": (".mlp.gate_up_proj.", 0), + ".mlp.up_proj.": (".mlp.gate_up_proj.", 1), + } + ) + def __init__( self, vision_config: Qwen2_5_VLVisionConfig, @@ -1116,32 +1125,8 @@ class Qwen2_5_VisionTransformer(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("attn.qkv.", "attn.q.", "q"), - ("attn.qkv.", "attn.k.", "k"), - ("attn.qkv.", "attn.v.", "v"), - ("mlp.gate_up_proj.", "mlp.gate_proj.", 0), - ("mlp.gate_up_proj.", "mlp.up_proj.", 1), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo): diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index 77eea390ed..a946159b8f 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -42,7 +42,6 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -58,14 +57,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import SupportsLoRA, SupportsPP from .utils import ( AutoWeightsLoader, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -418,124 +416,22 @@ class Qwen2MoeModel(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - ) - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if "mlp.experts" in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Remapping the name of FP8 kv-scale. - if name.endswith("kv_scale"): - remapped_kv_scale_name = name.replace( - ".kv_scale", ".attn.kv_scale" - ) - if remapped_kv_scale_name not in params_dict: - logger.warning_once( - "Found kv_scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv_scale is not loaded.", # noqa: E501 - name, - remapped_kv_scale_name, - ) - continue - else: - name = remapped_kv_scale_name - # GGUF: make sure that shared_expert_gate is a 2D tensor. - if ( - "mlp.shared_expert_gate" in name - and len(loaded_weight.shape) == 1 - ): - loaded_weight = loaded_weight[None, :] - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): fall_back_to_pt_during_load = False + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0), + ".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1), + } + ) packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -596,8 +492,16 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): return logits def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader(self) - return loader.load_weights(weights) + def _maybe_reshape( + weights: Iterable[tuple[str, torch.Tensor]], + ) -> Iterable[tuple[str, torch.Tensor]]: + for name, loaded_weight in weights: + # GGUF: make sure that shared_expert_gate is a 2D tensor. + if "mlp.shared_expert_gate" in name and loaded_weight.dim() == 1: + loaded_weight = loaded_weight[None, :] + yield name, loaded_weight - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() + loader = AutoWeightsLoader(self) + return loader.load_weights( + _maybe_reshape(weights), mapper=self.hf_to_vllm_mapper + ) diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 8625aec7ec..539f141cba 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -60,7 +60,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( @@ -522,6 +521,15 @@ class Qwen2VisionPatchMerger(nn.Module): class Qwen2VisionTransformer(nn.Module): + # Vision checkpoints store qkv pre-fused; merge separate q/k/v into qkv. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv", "q"), + ".k_proj": (".qkv", "k"), + ".v_proj": (".qkv", "v"), + } + ) + def __init__( self, vision_config: Qwen2VLVisionConfig, @@ -742,31 +750,8 @@ class Qwen2VisionTransformer(nn.Module): return x def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) def _create_qwen2vl_field_factory( diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 480ef3678c..2a7dc3e362 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -24,8 +24,7 @@ # limitations under the License. """Inference-only Qwen3.5 Series compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable import torch from torch import nn @@ -37,12 +36,7 @@ from vllm.distributed import ( get_pp_group, ) from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import ( - fused_moe_make_expert_params_mapping, -) -from vllm.model_executor.layers.layernorm import ( - GemmaRMSNorm as Qwen3_5RMSNorm, -) +from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( QwenGatedDeltaNetAttention, @@ -57,16 +51,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs.qwen3_5 import ( - Qwen3_5Config, - Qwen3_5TextConfig, -) +from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig from vllm.transformers_utils.configs.qwen3_5_moe import ( Qwen3_5MoeConfig, Qwen3_5MoeTextConfig, @@ -89,6 +76,7 @@ from .qwen3_next import ( Qwen3NextModel, Qwen3NextSparseMoeBlock, QwenNextMixtureOfExperts, + _is_shared_expert_fse_compatible, ) from .qwen3_vl import ( Qwen3_VisionTransformer, @@ -100,9 +88,9 @@ from .qwen3_vl import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, _merge_multimodal_embeddings, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -210,6 +198,17 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer): } ) class Qwen3_5Model(Qwen3NextModel): + # Qwen3.5 ships the GDN in_proj checkpoints separately (qwen3-next + # pre-fuses them); fuse them on top of the qwen3-next QKV/gate_up mapping. + hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper | WeightsMapper( + orig_to_new_stacked={ + ".in_proj_qkv": (".in_proj_qkvz", (0, 1, 2)), + ".in_proj_z": (".in_proj_qkvz", 3), + ".in_proj_b": (".in_proj_ba", 0), + ".in_proj_a": (".in_proj_ba", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super(Qwen3NextModel, self).__init__() @@ -222,6 +221,7 @@ class Qwen3_5Model(Qwen3NextModel): self.num_redundant_experts = eplb_config.num_redundant_experts self.config = config + self.quant_config = vllm_config.quant_config self.vocab_size = config.vocab_size @@ -251,212 +251,21 @@ class Qwen3_5Model(Qwen3NextModel): self.aux_hidden_state_layers: tuple[int, ...] = () - def load_fused_expert_weights( - self, - name: str, - params_dict: dict, - loaded_weight: torch.Tensor, - shard_id: str, - num_experts: int, - ) -> bool: - param = params_dict[name] - weight_loader = typing.cast(Callable[..., bool], param.weight_loader) - loaded_local_expert = False - for expert_id in range(num_experts): - curr_expert_weight = loaded_weight[expert_id] - success = weight_loader( - param, - curr_expert_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - loaded_local_expert = True - - return loaded_local_expert - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - # GDN - ("in_proj_qkvz", "in_proj_qkv", (0, 1, 2)), - ("in_proj_qkvz", "in_proj_z", 3), - # self attention - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - # mlp - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ("in_proj_ba", "in_proj_b", 0), - ("in_proj_ba", "in_proj_a", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - is_fused_expert = False - fused_expert_params_mapping: list[tuple[str, str, int, str]] = [] - for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_up_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="gate_up_proj", - num_experts=1, - ): - if shard_id == "w3": - continue - parts = ckpt_name.split(".") - fused_expert_params_mapping.append( - (f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id) + mapper = self.hf_to_vllm_mapper + # FSE must match construction (Qwen3NextSparseMoeBlock): reroute the + # shared expert into the extra fused slot only when AITER FSE is both + # requested and compatible with the quant spec. + is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and ( + _is_shared_expert_fse_compatible(self.quant_config) + ) + if is_fse: + num_routed = self.config.num_experts + mapper = mapper | WeightsMapper( + orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} ) - num_experts = ( - self.config.num_experts if hasattr(self.config, "num_experts") else 0 - ) - from vllm.config import get_current_vllm_config - - from .qwen3_next import _is_shared_expert_fse_compatible - - is_fse = ( - rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - and _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config) - ) - - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - if name.startswith("mtp."): - continue - - # Remapping the name of FP8 kv-scale. - if name.endswith("scale"): - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - # FSE: remap shared_expert weights to fused expert slot - if is_fse and "mlp.shared_expert." in name: - name = name.replace( - "mlp.shared_expert.", - f"mlp.experts.{num_experts}.", - ) - is_fused_expert = False - expert_params_mapping = self.get_expert_mapping() - - for param_name, weight_name, shard_id in stacked_params_mapping: - if "experts.gate_up_proj" in name or "experts.down_proj" in name: - is_fused_expert = True - expert_params_mapping = fused_expert_params_mapping - - if weight_name not in name: - continue - - if "mlp.experts" in name: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # name = apply_attn_prefix(name, params_dict) - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - is_expert_weight = True - name_mapped = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name_mapped, self): - continue - if is_fused_expert: - # qwen3.5 no need to transpose - # loaded_weight = loaded_weight.transpose(-1, -2) - if "experts.gate_up_proj" in name: - loaded_weight = loaded_weight.chunk(2, dim=-2) - success_w1 = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight[0], - "w1", - num_experts, - ) - success_w3 = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight[1], - "w3", - num_experts, - ) - success = success_w1 and success_w3 - else: - # down_proj - success = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight, - shard_id, - num_experts, - ) - if success: - name = name_mapped - break - else: - # Skip loading extra bias for GPTQ models. - if ( - name_mapped.endswith(".bias") - or name_mapped.endswith("_bias") - ) and name_mapped not in params_dict: - continue - param = params_dict[name_mapped] - weight_loader = param.weight_loader - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - logger.warning_once( - f"Parameter {name} not found in params_dict, skip loading" - ) - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=mapper) class Qwen3_5ForCausalLMBase( @@ -566,9 +375,6 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLMBase, QwenNextMixtureOfExperts): # set MoE hyperparameters self.set_moe_parameters() - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() - ######################################################## # Qwen3_5-Dense diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 021462f3ee..acc8366fb1 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -2,29 +2,32 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Inference-only Qwen3_5 MTP model.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable import torch from torch import nn +from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile -from vllm.config import VllmConfig +from vllm.config import VllmConfig, get_current_vllm_config from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import ( - fused_moe_make_expert_params_mapping, -) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.interfaces import LocalArgmaxMixin -from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm -from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts +from vllm.model_executor.models.qwen3_5 import ( + Qwen3_5DecoderLayer, + Qwen3_5Model, + Qwen3_5RMSNorm, +) +from vllm.model_executor.models.qwen3_next import ( + QwenNextMixtureOfExperts, + _is_shared_expert_fse_compatible, +) from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig @@ -37,8 +40,8 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, _merge_multimodal_embeddings, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, maybe_prefix, ) @@ -58,6 +61,8 @@ logger = init_logger(__name__) } ) class Qwen3_5MultiTokenPredictor(nn.Module): + hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -159,188 +164,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module): hidden_states, _ = self.norm(hidden_states, residual) return hidden_states - def load_fused_expert_weights( - self, - name: str, - params_dict: dict, - loaded_weight: torch.Tensor, - shard_id: str, - num_experts: int, - ) -> bool: - param = params_dict[name] - weight_loader = typing.cast(Callable[..., bool], param.weight_loader) - loaded_local_expert = False - for expert_id in range(num_experts): - curr_expert_weight = loaded_weight[expert_id] - success = weight_loader( - param, - curr_expert_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - loaded_local_expert = True - - return loaded_local_expert - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts - if hasattr(self.config, "num_experts") - else 0, + mapper = self.hf_to_vllm_mapper + is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and ( + _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config) ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - is_fused_expert = False - fused_expert_params_mapping: list[tuple[str, str, int, str]] = [] - for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_up_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="gate_up_proj", - num_experts=1, - ): - if shard_id == "w3": - continue - parts = ckpt_name.split(".") - fused_expert_params_mapping.append( - (f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id) + if is_fse: + num_routed = getattr(self.config, "num_experts", 0) + mapper = mapper | WeightsMapper( + orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} ) - num_experts = ( - self.config.num_experts if hasattr(self.config, "num_experts") else 0 - ) - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if "experts.gate_up_proj" in name or "experts.down_proj" in name: - is_fused_expert = True - expert_params_mapping = fused_expert_params_mapping - - if weight_name not in name: - continue - - if "mlp.experts" in name: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - is_expert_weight = True - name_mapped = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name_mapped, self): - continue - if is_fused_expert: - # qwen3.5 no need to transpose - # loaded_weight = loaded_weight.transpose(-1, -2) - if "experts.gate_up_proj" in name: - loaded_weight = loaded_weight.chunk(2, dim=-2) - success_w1 = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight[0], - "w1", - num_experts, - ) - success_w3 = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight[1], - "w3", - num_experts, - ) - success = success_w1 and success_w3 - else: - # down_proj - success = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight, - shard_id, - num_experts, - ) - if success: - name = name_mapped - break - else: - # Skip loading extra bias for GPTQ models. - if ( - name_mapped.endswith(".bias") - or name_mapped.endswith("_bias") - ) and name_mapped not in params_dict: - continue - param = params_dict[name_mapped] - weight_loader = param.weight_loader - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - logger.warning_once( - f"Parameter {name} not found in params_dict, skip loading" - ) - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=mapper) @support_torch_compile( diff --git a/vllm/model_executor/models/qwen3_eagle3.py b/vllm/model_executor/models/qwen3_eagle3.py index 6b03dfcdbd..a54d56cc04 100644 --- a/vllm/model_executor/models/qwen3_eagle3.py +++ b/vllm/model_executor/models/qwen3_eagle3.py @@ -17,15 +17,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM from vllm.multimodal.inputs import NestedTensors from .utils import ( AutoWeightsLoader, + WeightsMapper, get_draft_quant_config, maybe_prefix, process_eagle_weight, @@ -132,6 +129,17 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer): } ) class Qwen3Eagle3Model(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={"midlayer.": "layers.0."}, + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".gate_proj": (".gate_up_proj", 0), + ".up_proj": (".gate_up_proj", 1), + }, + ) + def __init__( self, *, @@ -256,38 +264,8 @@ class Qwen3Eagle3Model(nn.Module): return hidden_states, aux_output def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - (".qkv_proj", ".q_proj", "q"), - (".qkv_proj", ".k_proj", "k"), - (".qkv_proj", ".v_proj", "v"), - (".gate_up_proj", ".gate_proj", 0), - (".gate_up_proj", ".up_proj", 1), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "midlayer." in name: - name = name.replace("midlayer.", "layers.0.") - # Remapping the name FP8 kv-scale or zero point. - if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index b7a78acc0e..16a2275cec 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -23,8 +23,7 @@ # limitations under the License. """Inference-only Qwen3MoE model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable +from collections.abc import Iterable from itertools import islice from typing import Any @@ -43,10 +42,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import ( - FusedMoE, - fused_moe_make_expert_params_mapping, -) +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -61,10 +57,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.sequence import IntermediateTensors @@ -75,12 +67,13 @@ from .interfaces import ( SupportsEagle3, SupportsLoRA, SupportsPP, + SupportsQuant, ) from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -438,6 +431,20 @@ class Qwen3MoeDecoderLayer(nn.Module): @support_torch_compile class Qwen3MoeModel(nn.Module, EagleModelMixin): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + # .experts.gate_up_proj must be handled by MoERunner.load_weights for EP + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0), + ".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1), + } + ) + def __init__( self, *, @@ -516,152 +523,31 @@ class Qwen3MoeModel(nn.Module, EagleModelMixin): return hidden_states, aux_hidden_states return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=self.config.num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # Skip loading extra parameters for GPTQ/modelopt models. - ignore_suffixes = ( - ".bias", - "_bias", - ".weight_scale", - "_weight_scale", - ".input_scale", - "_input_scale", + loader = AutoWeightsLoader( + self, + ignore_unexpected_suffixes=[ + ".bias", + "_bias", + ".weight_scale", + "_weight_scale", + ".input_scale", + "_input_scale", + ], ) - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - for name, loaded_weight in weights: - if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - for param_name, weight_name, shard_id in stacked_params_mapping: - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if "mlp.experts" in name: - continue - name = name.replace(weight_name, param_name) - - # Skip loading extra parameters for GPTQ/modelopt models. - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name.endswith("scale"): - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if name not in params_dict: - continue - - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - if weight_loader == default_weight_loader: - weight_loader(param, loaded_weight) - else: - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - - # Do not modify `name` since the loop may continue here - # Instead, create a new variable - name_mapped = name.replace(weight_name, param_name) - - if is_pp_missing_parameter(name_mapped, self): - continue - - # Skip loading extra parameters for GPTQ/modelopt models. - if ( - name_mapped.endswith(ignore_suffixes) - and name_mapped not in params_dict - ): - continue - - param = params_dict[name_mapped] - # We should ask the weight loader to return success or not - # here since otherwise we may skip experts with other - # available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - - # Skip loading extra parameters for GPTQ/modelopt models. - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Qwen3MoeForCausalLM( - nn.Module, SupportsPP, SupportsLoRA, SupportsEagle, SupportsEagle3, MixtureOfExperts + nn.Module, + SupportsPP, + SupportsLoRA, + SupportsEagle, + SupportsEagle3, + MixtureOfExperts, + SupportsQuant, ): + hf_to_vllm_mapper = Qwen3MoeModel.hf_to_vllm_mapper packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -769,6 +655,3 @@ class Qwen3MoeForCausalLM( def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: loader = AutoWeightsLoader(self) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 74c2b1e44a..ef320dd526 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -10,12 +10,7 @@ from torch import nn from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile -from vllm.config import ( - CacheConfig, - ModelConfig, - VllmConfig, - get_current_vllm_config, -) +from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import ( get_ep_group, get_pp_group, @@ -26,7 +21,6 @@ from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( FusedMoE, - fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate from vllm.model_executor.layers.layernorm import ( @@ -53,10 +47,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.platforms import current_platform @@ -76,8 +66,8 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, - is_pp_missing_parameter, make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, @@ -132,7 +122,6 @@ class Qwen3NextSparseMoeBlock(nn.Module): ) # Load balancing settings. - vllm_config = get_current_vllm_config() eplb_config = vllm_config.parallel_config.eplb_config self.enable_eplb = parallel_config.enable_eplb @@ -544,6 +533,19 @@ class Qwen3NextDecoderLayer(nn.Module): @support_torch_compile class Qwen3NextModel(nn.Module, EagleModelMixin): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + # weight_name: (param_name, shard_id) + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + ".mlp.gate_proj": (".mlp.gate_up_proj", 0), + ".mlp.up_proj": (".mlp.gate_up_proj", 1), + ".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0), + ".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1), + } + ) + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -625,123 +627,18 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): return hidden_states, aux_hidden_states return hidden_states - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - num_experts = getattr(self.config, "num_experts", 0) - if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): - num_experts += 1 - return fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=num_experts, - num_redundant_experts=self.num_redundant_experts, - ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - - is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - num_routed = getattr(self.config, "num_experts", 0) - - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - if name.startswith("mtp."): - continue - - # Remapping the name of FP8 kv-scale. - if name.endswith("scale"): - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - # FSE: remap shared_expert weights to the fused expert slot - if is_fse and "mlp.shared_expert." in name: - name = name.replace( - "mlp.shared_expert.", - f"mlp.experts.{num_routed}.", - ) - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - - if "mlp.experts" in name: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # name = apply_attn_prefix(name, params_dict) - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - logger.warning_once( - f"Parameter {name} not found in params_dict, skip loading" - ) - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + mapper = self.hf_to_vllm_mapper + if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): + # AITER fused-shared-experts: route the shared_expert checkpoint + # weights into the extra fused expert slot. Merge (not mutate) so the + # shared class mapper isn't permanently altered. + num_routed = getattr(self.config, "num_experts", 0) + mapper = mapper | WeightsMapper( + orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} + ) + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=mapper) class QwenNextMixtureOfExperts(MixtureOfExperts): @@ -901,11 +798,5 @@ class Qwen3NextForCausalLM( return self.logits_processor(self.lm_head, hidden_states) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - loader = AutoWeightsLoader( - self, - skip_prefixes=["mtp."], - ) + loader = AutoWeightsLoader(self, skip_prefixes=["mtp."]) return loader.load_weights(weights) - - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 5ec0b82dab..48944009cd 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -12,18 +12,15 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import ( - fused_moe_make_expert_params_mapping, -) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.qwen3_next import ( Qwen3NextDecoderLayer, + Qwen3NextModel, Qwen3NextRMSNorm, QwenNextMixtureOfExperts, ) @@ -32,7 +29,7 @@ from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig from .utils import ( AutoWeightsLoader, - is_pp_missing_parameter, + WeightsMapper, make_empty_intermediate_tensors_factory, maybe_prefix, ) @@ -44,6 +41,8 @@ KVCache = tuple[torch.Tensor, torch.Tensor] @support_torch_compile class Qwen3NextMultiTokenPredictor(nn.Module): + hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() @@ -146,101 +145,16 @@ class Qwen3NextMultiTokenPredictor(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - - # Params for weights, fp8 weight scales, fp8 activation scales - # (param_name, weight_name, expert_id, shard_id) - is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() - num_experts = self.config.num_experts - if is_fse: - num_experts += 1 - expert_params_mapping = fused_moe_make_expert_params_mapping( - self, - ckpt_gate_proj_name="gate_proj", - ckpt_down_proj_name="down_proj", - ckpt_up_proj_name="up_proj", - num_experts=num_experts, - ) - num_routed = self.config.num_experts - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for name, loaded_weight in weights: - if "rotary_emb.inv_freq" in name: - continue - - # FSE: remap shared_expert weights to the fused expert slot - if is_fse and "mlp.shared_expert." in name: - name = name.replace( - "mlp.shared_expert.", - f"mlp.experts.{num_routed}.", - ) - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - - if "mlp.experts" in name: - continue - - name = name.replace(weight_name, param_name) - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - # Skip loading extra bias for GPTQ models. - if ( - name.endswith(".bias") or name.endswith("_bias") - ) and name not in params_dict: - continue - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader( - param, - loaded_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - ) - break - else: - # Skip loading extra bias for GPTQ models. - if name.endswith(".bias") and name not in params_dict: - continue - if is_pp_missing_parameter(name, self): - continue - - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + mapper = self.hf_to_vllm_mapper + if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled(): + # AITER fused-shared-experts: route the shared_expert checkpoint + # weights into the extra fused expert slot. + num_routed = self.config.num_experts + mapper = mapper | WeightsMapper( + orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."} + ) + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=mapper) @support_torch_compile diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index f37ecc0ed2..726329c780 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -63,7 +63,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo from vllm.multimodal import MULTIMODAL_REGISTRY @@ -319,6 +318,14 @@ class Qwen3OmniMoeAudioEncoderLayer(nn.Module): class Qwen3OmniMoeAudioEncoder(nn.Module): """vLLM-native Qwen3-Omni Audio Encoder.""" + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".self_attn.q_proj.": (".self_attn.qkv.", "q"), + ".self_attn.k_proj.": (".self_attn.qkv.", "k"), + ".self_attn.v_proj.": (".self_attn.qkv.", "v"), + } + ) + def __init__( self, config: Qwen3OmniMoeAudioEncoderConfig, @@ -531,35 +538,8 @@ class Qwen3OmniMoeAudioEncoder(nn.Module): return lengths def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - """Load weights with mapping from HuggingFace format.""" - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("self_attn.qkv.", "self_attn.q_proj.", "q"), - ("self_attn.qkv.", "self_attn.k_proj.", "k"), - ("self_attn.qkv.", "self_attn.v_proj.", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict.get(name) - if param is not None: - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Qwen3_VisionPatchEmbed(nn.Module): @@ -737,6 +717,14 @@ class Qwen3_VisionPatchMerger(nn.Module): class Qwen3Omni_VisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".attn.q.": (".attn.qkv.", "q"), + ".attn.k.": (".attn.qkv.", "k"), + ".attn.v.": (".attn.qkv.", "v"), + } + ) + def __init__( self, vision_config, @@ -1044,31 +1032,8 @@ class Qwen3Omni_VisionTransformer(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("attn.qkv.", "attn.q.", "q"), - ("attn.qkv.", "attn.k.", "k"), - ("attn.qkv.", "attn.v.", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) @support_torch_compile( diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index a52f725ccb..a85286164e 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -67,7 +67,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.evs import ( @@ -518,6 +517,14 @@ class Qwen3_VisionPatchMerger(nn.Module): class Qwen3_VisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + "attn.q.": ("attn.qkv.", "q"), + "attn.k.": ("attn.qkv.", "k"), + "attn.v.": ("attn.qkv.", "v"), + } + ) + def __init__( self, vision_config: Qwen3VLVisionConfig, @@ -834,31 +841,8 @@ class Qwen3_VisionTransformer(nn.Module): return hidden_states def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("attn.qkv.", "attn.q.", "q"), - ("attn.qkv.", "attn.k.", "k"), - ("attn.qkv.", "attn.v.", "v"), - ] - params_dict = dict(self.named_parameters(remove_duplicate=False)) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): diff --git a/vllm/model_executor/models/qwen3_vl_moe.py b/vllm/model_executor/models/qwen3_vl_moe.py index 5291874dd5..17164d1735 100644 --- a/vllm/model_executor/models/qwen3_vl_moe.py +++ b/vllm/model_executor/models/qwen3_vl_moe.py @@ -24,8 +24,6 @@ # limitations under the License. """Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights.""" -import typing -from collections.abc import Callable, Iterable from itertools import islice import torch @@ -39,10 +37,6 @@ from vllm.distributed import get_pp_group from vllm.logger import init_logger from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -60,7 +54,7 @@ from .qwen3_vl import ( Qwen3VLMultiModalProcessor, Qwen3VLProcessingInfo, ) -from .utils import is_pp_missing_parameter, maybe_prefix +from .utils import maybe_prefix logger = init_logger(__name__) @@ -135,202 +129,6 @@ class Qwen3MoeLLMModel(Qwen3MoeModel): return hidden_states, aux_hidden_states return hidden_states - def load_fused_expert_weights( - self, - name: str, - params_dict: dict, - loaded_weight: torch.Tensor, - shard_id: str, - num_experts: int, - ) -> bool: - param = params_dict[name] - weight_loader = typing.cast(Callable[..., bool], param.weight_loader) - loaded_local_expert = False - for expert_id in range(num_experts): - curr_expert_weight = loaded_weight[expert_id] - success = weight_loader( - param, - curr_expert_weight, - name, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - loaded_local_expert = True - - return loaded_local_expert - - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ("gate_up_proj", "gate_proj", 0), - ("gate_up_proj", "up_proj", 1), - ] - # Skip loading extra parameters for GPTQ/modelopt models. - ignore_suffixes = ( - ".bias", - "_bias", - ".weight_scale", - "_weight_scale", - ".input_scale", - "_input_scale", - ) - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - expert_params_mapping = self.get_expert_mapping() - is_fused_expert = False - base_layer = ( - "base_layer." if any(".base_layer." in name for name in params_dict) else "" - ) - fused_expert_params_mapping = [ - ( - f"experts.routed_experts.{base_layer}w13_weight", - "experts.gate_up_proj", - 0, - "w1", - ), - ( - f"experts.routed_experts.{base_layer}w2_weight", - "experts.down_proj", - 0, - "w2", - ), - ] - num_experts = self.config.num_experts - for name, loaded_weight in weights: - if "scale" in name or "zero_point" in name: - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - - for param_name, weight_name, shard_id in stacked_params_mapping: - if "experts.gate_up_proj" in name or "experts.down_proj" in name: - is_fused_expert = True - expert_params_mapping = fused_expert_params_mapping - - # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: - continue - # We have mlp.experts[0].gate_proj in the checkpoint. - # Since we handle the experts below in expert_params_mapping, - # we need to skip here BEFORE we update the name, otherwise - # name will be updated to mlp.experts[0].gate_up_proj, which - # will then be updated below in expert_params_mapping - # for mlp.experts[0].gate_gate_up_proj, which breaks load. - if "mlp.experts" in name: - continue - name = name.replace(weight_name, param_name) - # Skip loading extra parameters for GPTQ/modelopt models. - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name.endswith("scale"): - # Remapping the name of FP8 kv-scale. - name = maybe_remap_kv_scale_name(name, params_dict) - if name is None: - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - if weight_loader == default_weight_loader: - weight_loader(param, loaded_weight) - else: - weight_loader(param, loaded_weight, shard_id) - break - else: - is_expert_weight = False - for mapping in expert_params_mapping: - param_name, weight_name, expert_id, shard_id = mapping - if weight_name not in name: - continue - # Anyway, this is an expert weight and should not be - # attempted to load as other weights later - is_expert_weight = True - name_mapped = name.replace(weight_name, param_name) - if is_pp_missing_parameter(name_mapped, self): - continue - if is_fused_expert: - loaded_weight = loaded_weight.transpose(-1, -2) # no bias - if "experts.gate_up_proj" in name: - loaded_weight = loaded_weight.chunk(2, dim=-2) - success_w1 = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight[0], - "w1", - num_experts, - ) - success_w3 = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight[1], - "w3", - num_experts, - ) - success = success_w1 and success_w3 - else: - # down_proj - success = self.load_fused_expert_weights( - name_mapped, - params_dict, - loaded_weight, - shard_id, - num_experts, - ) - else: - # Skip loading extra parameters for GPTQ/modelopt models - if ( - name_mapped.endswith(ignore_suffixes) - and name_mapped not in params_dict - ): - continue - param = params_dict[name_mapped] - # We should ask the weight loader to return success or - # not here since otherwise we may skip experts with - # other available replicas. - weight_loader = typing.cast( - Callable[..., bool], param.weight_loader - ) - success = weight_loader( - param, - loaded_weight, - name_mapped, - shard_id=shard_id, - expert_id=expert_id, - return_success=True, - ) - if success: - name = name_mapped - break - else: - if is_expert_weight: - # We've checked that this is an expert weight - # However it's not mapped locally to this rank - # So we simply skip it - continue - # Skip loading extra parameters for GPTQ/modelopt models. - if name.endswith(ignore_suffixes) and name not in params_dict: - continue - # Skip layers on other devices. - if is_pp_missing_parameter(name, self): - continue - if name not in params_dict: - continue - param = params_dict[name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 1970298e76..5808c9539b 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -33,10 +33,6 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.pooler import DispatchPooler from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.model_executor.model_loader.weight_utils import ( - default_weight_loader, - maybe_remap_kv_scale_name, -) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( MultiModalFieldConfig, @@ -62,7 +58,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant from .interfaces_base import default_pooling_type -from .utils import AutoWeightsLoader, maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix from .vision import ( VisionEncoderInfo, VisionFeatureSelectStrategy, @@ -571,6 +567,14 @@ class SiglipEncoder(nn.Module): class SiglipTextTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: SiglipTextConfig, @@ -615,30 +619,8 @@ class SiglipTextTransformer(nn.Module): return last_hidden_state def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) class SiglipMultiheadAttentionPoolingHead(nn.Module): @@ -682,6 +664,14 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module): class SiglipVisionTransformer(nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: SiglipVisionConfig, @@ -806,46 +796,25 @@ class SiglipVisionTransformer(nn.Module): return encoder_outputs def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() + skip_prefixes = [] + if self.post_layernorm is None: + skip_prefixes.append("post_layernorm.") + if self.head is None: + skip_prefixes.append("head.") + loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes) + layer_count = len(self.encoder.layers) - for name, loaded_weight in weights: - # post_layernorm is not needed in SiglipVisionTransformer - if name.startswith("post_layernorm") and self.post_layernorm is None: - continue - - # if the model configuration is not going to use - # the pooling head for inference, don't load its weights - if self.head is None and name.startswith("head"): - continue - - # omit layers when num_hidden_layers_override is set - if name.startswith("encoder.layers"): - layer_idx = int(name.split(".")[2]) - if layer_idx >= layer_count: + def _filter(ws): + for name, w in ws: + # omit layers when num_hidden_layers_override is set + if name.startswith("encoder.layers.") and ( + int(name.split(".")[2]) >= layer_count + ): continue + yield name, w - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper) class SiglipVisionModel(nn.Module): @@ -896,67 +865,6 @@ class SiglipVisionModel(nn.Module): feature_select_strategy=feature_select_strategy, ) - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - layer_count = len(self.vision_model.encoder.layers) - - for name, loaded_weight in weights: - # post_layernorm is optional in SiglipVisionModel - if ( - name.startswith("vision_model.post_layernorm") - and self.vision_model.post_layernorm is None - ): - continue - - # if the model configuration is not going to use - # the pooling head for inference, don't load its weights - if self.vision_model.head is None and name.startswith("vision_model.head"): - continue - - # omit layers when num_hidden_layers_override is set - if name.startswith("vision_model.encoder.layers"): - layer_idx = int(name.split(".")[3]) - if layer_idx >= layer_count: - continue - - # Check if this is a scale parameter that needs remapping first - if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")): - # Try to remap the scale name first - remapped_name = maybe_remap_kv_scale_name(name, params_dict) - if remapped_name is not None and remapped_name in params_dict: - # Successfully remapped, use the remapped name - param = params_dict[remapped_name] - weight_loader = getattr( - param, "weight_loader", default_weight_loader - ) - weight_loader(param, loaded_weight) - loaded_params.add(remapped_name) - continue - # If remapping failed, continue with normal processing - - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params - # Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200 class SiglipTextEmbeddings(nn.Module): diff --git a/vllm/model_executor/models/siglip2navit.py b/vllm/model_executor/models/siglip2navit.py index 906a51bd7b..5ee1c2c910 100644 --- a/vllm/model_executor/models/siglip2navit.py +++ b/vllm/model_executor/models/siglip2navit.py @@ -26,10 +26,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding.common import ( ApplyRotaryEmb, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.platforms import current_platform -from .utils import maybe_prefix +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix from .vision import is_vit_use_data_parallel @@ -588,6 +587,14 @@ class Siglip2VisionTransformer(nn.Module): class Siglip2NavitModel(torch.nn.Module): + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_stacked={ + ".q_proj": (".qkv_proj", "q"), + ".k_proj": (".qkv_proj", "k"), + ".v_proj": (".qkv_proj", "v"), + } + ) + def __init__( self, config: Siglip2VisionConfig, @@ -613,28 +620,5 @@ class Siglip2NavitModel(torch.nn.Module): ) def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - - for name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: - continue - name = name.replace(weight_name, param_name) - - param = params_dict[name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - param = params_dict[name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(name) - return loaded_params + loader = AutoWeightsLoader(self) + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper) diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index 3e04aaa074..d5267a2617 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -16,7 +16,6 @@ # limitations under the License. """Transformers modeling backend mixin for Mixture of Experts (MoE) models.""" -from collections.abc import Iterable from dataclasses import dataclass from functools import partial from typing import TYPE_CHECKING, Any @@ -82,11 +81,6 @@ class TransformersMoERunner(MoERunner): ) -> torch.Tensor: return super().forward(hidden_states, topk_weights) - def load_weights( - self, weights: Iterable[tuple[str, torch.Tensor]] - ) -> Iterable[str]: - return self.routed_experts.load_weights(weights) - def _transformers_moe_forward( hidden_states: torch.Tensor, @@ -119,6 +113,20 @@ direct_register_custom_op( ) +class TransformersRoutedExperts(RoutedExperts): + def get_expert_mapping( + self, include_fused: bool = False + ) -> list[tuple[str, str, int, str]]: + common_names = ("gate_proj", "down_proj", "up_proj") + common_map = super().get_expert_mapping(*common_names, include_fused) + mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused) + if not include_fused: + return common_map + mixtral_map + common_fused, common_unfused = common_map[:3], common_map[3:] + mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:] + return common_fused + mixtral_fused + common_unfused + mixtral_unfused + + class MoEMixin(MixtureOfExperts): def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""): self.check_version("5.0.0", "MoE models support") @@ -140,43 +148,6 @@ class MoEMixin(MixtureOfExperts): mlp.n_redundant_experts = self.num_redundant_experts mlp.experts.update_expert_map() - def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - """ - Params for weights, fp8 weight scales, fp8 activation scales - (param_name, weight_name, expert_id, shard_id) - """ - # Models saved with fused experts. These are checkpoints released: - # - After Transformers v5 - # - Before Transformers v5, but re-saved with save_original_format=False - # In the fused experts case, we repurpose the expert_id as shard_idx for - # deconcatenating w1 and w3 in FusedMoE.load_weights. - expert_mapping = [ - ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"), - ("experts.w13_weight", "experts.gate_up_proj", 1, "w3"), - ("experts.w2_weight", "experts.down_proj", 0, "w2"), - ] - # Models saved with ModuleList experts - ckpt_names = [ - # (ckpt_gate_proj_name, ckpt_down_proj_name, ckpt_up_proj_name) - ("gate_proj", "down_proj", "up_proj"), # Most common MoE style - ("w1", "w2", "w3"), # Granite, Mixtral, Phi MoE style - ] - num_experts = self.model_config.get_num_experts() - num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts - for gate_proj, down_proj, up_proj in ckpt_names: - expert_mapping.extend( - RoutedExperts.make_expert_params_mapping( - self, - ckpt_gate_proj_name=gate_proj, - ckpt_down_proj_name=down_proj, - ckpt_up_proj_name=up_proj, - num_experts=num_experts, - num_redundant_experts=num_redundant_experts, - routed_experts_prefix="", - ) - ) - return expert_mapping - def recursive_replace(self): """Initialize the MoE layers.""" text_config = self.text_config @@ -218,9 +189,6 @@ class MoEMixin(MixtureOfExperts): if "gptoss" in wrapped_arch: activation = "swigluoai" - # Expert mapping for `AutoWeightsLoader` - expert_mapping = self.get_expert_mapping() - # Expert parallel load balancing kwargs enable_eplb = self.parallel_config.enable_eplb num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts @@ -308,12 +276,12 @@ class MoEMixin(MixtureOfExperts): enable_eplb=enable_eplb, num_redundant_experts=num_redundant_experts, has_bias=has_bias, - expert_mapping=expert_mapping, custom_routing_function=partial( custom_routing_function, moe_state=moe_state, ), runner_cls=TransformersMoERunner, + routed_experts_cls=TransformersRoutedExperts, runner_args={"moe_state": moe_state}, ) mlp.experts = fused_experts diff --git a/vllm/model_executor/models/zamba2.py b/vllm/model_executor/models/zamba2.py index b4d844ba6d..f80c8a3b07 100644 --- a/vllm/model_executor/models/zamba2.py +++ b/vllm/model_executor/models/zamba2.py @@ -43,7 +43,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, VocabParallelEmbedding, ) -from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.sequence import IntermediateTensors from .interfaces import HasInnerState, IsHybrid, SupportsMambaPrefixCaching @@ -804,34 +803,6 @@ class Zamba2Model(nn.Module): hidden_states = self.final_layernorm(hidden_states) return hidden_states - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: - stacked_params_mapping = [ - # (param_name, shard_name, shard_id) - ("qkv_proj", "q_proj", "q"), - ("qkv_proj", "k_proj", "k"), - ("qkv_proj", "v_proj", "v"), - ] - - params_dict = dict(self.named_parameters()) - loaded_params: set[str] = set() - for chkpt_weight_name, loaded_weight in weights: - for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in chkpt_weight_name: - continue - chkpt_weight_name = chkpt_weight_name.replace(weight_name, param_name) - param = params_dict[chkpt_weight_name] - weight_loader = param.weight_loader - weight_loader(param, loaded_weight, shard_id) - break - else: - if chkpt_weight_name not in params_dict: - continue - param = params_dict[chkpt_weight_name] - weight_loader = getattr(param, "weight_loader", default_weight_loader) - weight_loader(param, loaded_weight) - loaded_params.add(chkpt_weight_name) - return loaded_params - class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixCaching): """Zamba2 model with causal language modeling head. @@ -849,7 +820,12 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC "A_log": "A", "0.weight": "A.weight", "1.weight": "B.weight", - } + }, + orig_to_new_stacked={ + ".self_attn.q_proj": (".self_attn.qkv_proj", "q"), + ".self_attn.k_proj": (".self_attn.qkv_proj", "k"), + ".self_attn.v_proj": (".self_attn.qkv_proj", "v"), + }, ) @classmethod diff --git a/vllm/model_executor/utils.py b/vllm/model_executor/utils.py index a0269be855..70acb33e36 100644 --- a/vllm/model_executor/utils.py +++ b/vllm/model_executor/utils.py @@ -124,15 +124,16 @@ def get_packed_modules_mapping(model: torch.nn.Module) -> dict[str, list[str]]: def get_moe_expert_mapping( model: torch.nn.Module, ) -> list[tuple[str, str, int, str]]: - if parent_map := getattr(model, "get_expert_mapping", None): - return parent_map() - else: - # We only check main components instead of whole model submodules - for child in model.children(): - child_map = getattr(child, "get_expert_mapping", None) - if child_map is not None: - return child_map() - return [] + """Get the expert mapping from a model. + + It will be retrieved from the first module that has a `get_expert_mapping` method. + If the model manually implements `get_expert_mapping`, it will be used. + Otherwise, it will use the first RoutedExperts layer.""" + for _, module in model.named_modules(): + get_mapping = getattr(module, "get_expert_mapping", None) + if get_mapping is not None: + return get_mapping() + raise ValueError("No module in the model has a `get_expert_mapping` method.") def maybe_disable_graph_partition(current_backend: str) -> dict[str, bool]: diff --git a/vllm/models/deepseek_v4/nvidia/model.py b/vllm/models/deepseek_v4/nvidia/model.py index f1bcd534e9..fed4b5d6a1 100644 --- a/vllm/models/deepseek_v4/nvidia/model.py +++ b/vllm/models/deepseek_v4/nvidia/model.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import typing -from collections.abc import Callable, Iterable, MutableSequence, Sequence +from collections.abc import Callable, Iterable from itertools import islice import regex as re @@ -1365,7 +1365,6 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, DeepseekV4MixtureOfExperts): self.set_moe_parameters() def set_moe_parameters(self) -> None: - self.expert_weights: MutableSequence[Sequence[torch.Tensor]] = [] self.num_expert_groups = getattr(self.config, "n_group", 1) self.num_moe_layers = self.config.num_hidden_layers self.moe_layers: list[nn.Module] = []