Remove more unnecessary load_weights methods (#47058)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-06-30 15:22:16 +01:00
committed by GitHub
parent 0fc2512094
commit 1ab9522935
63 changed files with 1024 additions and 3774 deletions
@@ -17,7 +17,6 @@ from vllm.model_executor.models.moss_audio import (
MOSS_AUDIO_PLACEHOLDER,
MOSS_AUDIO_TOKEN,
MOSS_AUDIO_TOKEN_ID,
GatedMLP,
MossAudioConfig,
MossAudioDummyInputsBuilder,
MossAudioEncoder,
@@ -594,53 +593,6 @@ def test_moss_qwen3_deepstack_keys_for_pp(monkeypatch):
assert set(forward_tensors.tensors) == set(tensors.tensors)
@pytest.mark.parametrize("tp_size", [1, 2])
def test_moss_audio_gated_mlp_tp_shapes_and_loading(monkeypatch, tp_size):
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.device import DeviceConfig
_patch_tensor_parallel_for_linear_layers(monkeypatch, tp_size=tp_size)
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6)
params = dict(mlp.named_parameters())
assert params["gate_up_proj.weight"].shape == torch.Size([16 // tp_size, 4])
assert params["down_proj.weight"].shape == torch.Size([6, 8 // tp_size])
gate_weight = torch.arange(32, dtype=torch.float32).reshape(8, 4)
up_weight = torch.arange(100, 132, dtype=torch.float32).reshape(8, 4)
down_weight = torch.arange(48, dtype=torch.float32).reshape(6, 8)
loaded = mlp.load_weights(
[
("gate_proj.weight", gate_weight),
("up_proj.weight", up_weight),
("down_proj.weight", down_weight),
]
)
assert loaded == {"gate_up_proj.weight", "down_proj.weight"}
shard = 8 // tp_size
assert torch.equal(params["gate_up_proj.weight"][:shard], gate_weight[:shard])
assert torch.equal(params["gate_up_proj.weight"][shard:], up_weight[:shard])
assert torch.equal(params["down_proj.weight"], down_weight[:, : 8 // tp_size])
with set_current_vllm_config(VllmConfig(device_config=DeviceConfig(device="cpu"))):
packed_mlp = GatedMLP(input_size=4, hidden_size=8, output_size=6)
packed_params = dict(packed_mlp.named_parameters())
loaded = packed_mlp.load_weights(
[("gate_up_proj.weight", torch.cat([gate_weight, up_weight], dim=0))]
)
assert loaded == {"gate_up_proj.weight"}
assert torch.equal(
packed_params["gate_up_proj.weight"][:shard],
gate_weight[:shard],
)
assert torch.equal(
packed_params["gate_up_proj.weight"][shard:],
up_weight[:shard],
)
def test_moss_audio_encoder_loads_realistic_attention_weight_names(monkeypatch):
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.device import DeviceConfig
+1
View File
@@ -39,6 +39,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
self._ep_check()
routed_experts = self.base_layer.routed_experts
routed_experts.lora_base_layer_prefix = "base_layer."
assert not routed_experts.quant_method.is_monolithic, (
"Monolithic kernels are not supported for Fused MoE LoRA."
)
+20 -26
View File
@@ -372,32 +372,26 @@ def process_packed_modules_mapping(
model: nn.Module, force_2d_moe: bool = False
) -> dict[str, list[str]]:
if is_moe_model(model):
if moe_packed_mapping := get_moe_expert_mapping(model):
# This method generates and returns a dictionary mapping packed module
# names to lists of their corresponding submodule names. It includes
# both static mappings and dynamic mappings for expert layers, where
# the expert indices are expanded based on the configured number
# of routed experts.
packed_modules_mapping = get_packed_modules_mapping(model)
# The 2D mapping is needed when the model itself is 2D, or when
# the engine forces the universal 2D wrapper via
# enable_mixed_moe_lora_format (so 3D models can also load 2D
# adapters through FusedMoEWithLoRA).
if (not model.is_3d_moe_weight) or force_2d_moe:
# Filter out malformed entries: non-gated MoE has empty
# ckpt_up_proj_name which results in weight_name containing ".."
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
packed_modules_mapping["experts"] = [
weight_name.rstrip(".")
for _, weight_name, _, _ in moe_packed_mapping
if ".." not in weight_name
]
# This method generates and returns a dictionary mapping packed module
# names to lists of their corresponding submodule names. It includes
# both static mappings and dynamic mappings for expert layers, where
# the expert indices are expanded based on the configured number
# of routed experts.
packed_modules_mapping = get_packed_modules_mapping(model)
# The 2D mapping is needed when the model itself is 2D, or when
# the engine forces the universal 2D wrapper via
# enable_mixed_moe_lora_format (so 3D models can also load 2D
# adapters through FusedMoEWithLoRA).
if (not model.is_3d_moe_weight) or force_2d_moe:
# Filter out malformed entries: non-gated MoE has empty
# ckpt_up_proj_name which results in weight_name containing ".."
# (e.g., "experts.0.." instead of "experts.0.layer_name.")
packed_modules_mapping["experts"] = [
weight_name.rstrip(".")
for _, weight_name, _, _ in get_moe_expert_mapping(model)
if ".." not in weight_name
]
return packed_modules_mapping
else:
raise AttributeError(
"To support LoRA for MoE model, "
"'get_expert_mapping' must be implemented"
)
return packed_modules_mapping
else:
return get_packed_modules_mapping(model)
@@ -127,7 +127,7 @@ def FusedMoE(
num_redundant_experts: int = 0,
has_bias: bool = False,
is_sequence_parallel: bool = False,
expert_mapping: list[tuple[str, str, int, str]] | None = None,
ckpt_names: tuple[str, str, str] = ("gate_proj", "down_proj", "up_proj"),
n_shared_experts: int | None = None,
router_logits_dtype: torch.dtype | None = None,
gate: torch.nn.Module | None = None,
@@ -356,7 +356,9 @@ def FusedMoE(
moe_config,
quant_config,
expert_map_manager=expert_map_manager,
expert_mapping=expert_mapping,
ckpt_gate_proj_name=ckpt_names[0],
ckpt_down_proj_name=ckpt_names[1],
ckpt_up_proj_name=ckpt_names[2],
# Extra params that are needed by quant_methods, pass along for now
# Prefer getting these from other sources, e.g. moe_config or
# router object
@@ -58,7 +58,9 @@ class RoutedExperts(PluggableLayer):
moe_config: FusedMoEConfig,
quant_config: QuantizationConfig | None,
expert_map_manager: ExpertMapManager,
expert_mapping: list[tuple[str, str, int, str]] | None = None,
ckpt_gate_proj_name: str = "gate_proj",
ckpt_down_proj_name: str = "down_proj",
ckpt_up_proj_name: str = "up_proj",
#
# Extra params that are needed by quant_methods, pass along for now
# Prefer getting these from other sources, e.g. moe_config or
@@ -81,7 +83,9 @@ class RoutedExperts(PluggableLayer):
self.layer_name = layer_name
self.moe_config = moe_config
self.quant_config = quant_config
self.expert_mapping = expert_mapping
self.ckpt_gate_proj_name = ckpt_gate_proj_name
self.ckpt_down_proj_name = ckpt_down_proj_name
self.ckpt_up_proj_name = ckpt_up_proj_name
self.expert_map_manager = expert_map_manager
self.hidden_size = moe_config.hidden_dim
self.global_num_experts = moe_config.num_experts
@@ -167,6 +171,8 @@ class RoutedExperts(PluggableLayer):
self.quant_method.create_weights(layer=self, **moe_quant_params)
self.lora_base_layer_prefix = ""
# TODO(bnell): Temporary hack. Get rid of this.
def _replace_quant_method(self, quant_method: FusedMoEMethodBase):
self.quant_method = quant_method
@@ -863,26 +869,33 @@ class RoutedExperts(PluggableLayer):
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
if (expert_mapping := self.expert_mapping) is None:
raise ValueError(
"`self.expert_mapping` must be provided to "
"load weights using `self.load_weights`."
)
expert_mapping = self.get_expert_mapping(include_fused=True)
unpadded_hidden = self.moe_config.hidden_dim_unpadded
for expert_name, loaded_weight in weights:
qual_name = f"{self.layer_name}.{expert_name}"
# Fused expert weights can be identified by their 3D tensors
is_fused = loaded_weight.dim() == 3
matched = False
for param_name, weight_name, expert_id, shard_id in expert_mapping:
if weight_name not in qual_name:
if matched and is_fused:
break
continue
matched = True
weight_name = qual_name.replace(weight_name, param_name)
param_name = weight_name.removeprefix(f"{self.layer_name}.")
param = getattr(self, param_name)
# Fused expert weights can be identified by their 3D tensors
if loaded_weight.dim() == 3:
# Repurpose expert_id as shard_idx for deconcatenating w1 and w3
if is_fused:
if shard_id in {"w1", "w3"}:
shard_idx = expert_id
experts_shard = loaded_weight.chunk(2, dim=1)[shard_idx]
if loaded_weight.shape[-1] != unpadded_hidden:
# [..., hidden, intermediate] -> [..., intermediate, hidden]
loaded_weight = loaded_weight.transpose(-1, -2)
# Repurpose expert_id for deconcatenating w1 and w3
experts_shard = loaded_weight.chunk(2, dim=1)[expert_id]
else:
if loaded_weight.shape[-2] != unpadded_hidden:
# [..., intermediate, hidden] -> [..., hidden, intermediate]
loaded_weight = loaded_weight.transpose(-1, -2)
experts_shard = loaded_weight
start = 0
else:
@@ -912,6 +925,27 @@ class RoutedExperts(PluggableLayer):
)
yield param_name
def get_expert_mapping(
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
moe_config = self.moe_config
num_fused_shared_experts = self.expert_map_manager.num_fused_shared_experts
num_redundant_experts = moe_config.num_experts - moe_config.num_logical_experts
return self.build_expert_params_mapping(
ckpt_gate_proj_name or self.ckpt_gate_proj_name,
ckpt_down_proj_name or self.ckpt_down_proj_name,
ckpt_up_proj_name or self.ckpt_up_proj_name,
num_experts=moe_config.num_logical_experts + num_fused_shared_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
lora_base_layer_prefix=self.lora_base_layer_prefix,
include_fused=include_fused,
)
@staticmethod
def make_expert_params_mapping(
model: torch.nn.Module,
@@ -921,6 +955,36 @@ class RoutedExperts(PluggableLayer):
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
) -> list[tuple[str, str, int, str]]:
"""Build the expert mapping, detecting the LoRA `base_layer.` prefix by
scanning `model`'s parameters.
Legacy entry point for models that still hand-roll `load_weights`; the
`RoutedExperts` weight loader uses `get_expert_mapping` /
`build_expert_params_mapping` instead (which take the prefix directly).
See `build_expert_params_mapping` for the returned tuple format.
"""
has_base_layer = any(".base_layer." in n for n, _ in model.named_parameters())
return RoutedExperts.build_expert_params_mapping(
ckpt_gate_proj_name,
ckpt_down_proj_name,
ckpt_up_proj_name,
num_experts,
num_redundant_experts,
routed_experts_prefix,
"base_layer." if has_base_layer else "",
)
@staticmethod
def build_expert_params_mapping(
ckpt_gate_proj_name: str,
ckpt_down_proj_name: str,
ckpt_up_proj_name: str,
num_experts: int,
num_redundant_experts: int = 0,
routed_experts_prefix: str = "routed_experts",
lora_base_layer_prefix: str = "",
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
"""
Create expert parameter mapping for weight loading with redundant experts.
@@ -929,12 +993,13 @@ class RoutedExperts(PluggableLayer):
when loading weights with EPLB redundant experts.
Args:
model: The model containing the MoE layer
ckpt_gate_proj_name: Name of gate projection in checkpoint
ckpt_down_proj_name: Name of down projection in checkpoint
ckpt_up_proj_name: Name of up projection in checkpoint
num_experts: Number of logical (non-redundant) experts
num_redundant_experts: Number of redundant experts
lora_base_layer_prefix: Prefix to add if this layer is a LoRA base layer
include_fused: Prepend the fused pre-fused-checkpoint entries
Returns:
List of tuples (param_name, weight_name, expert_id, shard_id)
@@ -956,22 +1021,39 @@ class RoutedExperts(PluggableLayer):
)
)
base_layer = (
"base_layer."
if any(".base_layer." in name for name, _ in model.named_parameters())
else ""
)
if routed_experts_prefix != "":
routed_experts_prefix = f"{routed_experts_prefix}."
return [
w13 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w13_"
w2 = f"experts.{routed_experts_prefix}{lora_base_layer_prefix}w2_"
fused_mapping = []
if include_fused:
gate_up = None
if ckpt_gate_proj_name == "gate_proj" and ckpt_up_proj_name == "up_proj":
gate_up = "gate_up_proj"
elif ckpt_gate_proj_name == "w1" and ckpt_up_proj_name == "w3":
gate_up = "w13"
else:
logger.warning(
"Unexpected gate/up projection names: %s, %s. "
"Fused gate/up mapping will be skipped.",
ckpt_gate_proj_name,
ckpt_up_proj_name,
)
if gate_up is not None:
fused_mapping = [
# (param_name, weight_name, expert_id, shard_id)
(f"{w13}weight", f"experts.{gate_up}", 0, "w1"),
(f"{w13}weight", f"experts.{gate_up}", 1, "w3"),
(f"{w2}weight", f"experts.{ckpt_down_proj_name}", 0, "w2"),
]
per_expert_mapping = [
# (param_name, weight_name, expert_id, shard_id)
(
f"experts.{routed_experts_prefix}{base_layer}w13_"
if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name]
else f"experts.{routed_experts_prefix}{base_layer}w2_",
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{base_layer}",
w13 if weight_name in [ckpt_gate_proj_name, ckpt_up_proj_name] else w2,
f"experts.{physical_to_logical_map[expert_id]}.{weight_name}.{lora_base_layer_prefix}",
expert_id,
shard_id,
)
@@ -983,6 +1065,8 @@ class RoutedExperts(PluggableLayer):
]
]
return fused_mapping + per_expert_mapping
def get_expert_weights(self) -> Iterable[torch.Tensor]:
def _maybe_make_contiguous(
name: str, p: torch.nn.Parameter
@@ -292,6 +292,11 @@ class MoERunner(MoERunnerInterface):
# For smuggling this layer into the fused moe custom op
register_layer_for_moe_forward_op(get_current_vllm_config(), self)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
def _select_forward(self) -> Callable:
if current_platform.is_tpu() or current_platform.is_cpu():
# TODO: Once the OOM issue for the TPU backend is resolved, we
@@ -567,12 +567,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
if is_moe_model(model):
self.expert_params_mapping = get_moe_expert_mapping(model)
if not self.expert_params_mapping:
raise AttributeError(
f"MoE Model {type(model).__name__} does not support "
"BitsAndBytes quantization yet. Ensure this model has "
"'get_expert_mapping' method."
)
# For some models like Molmo, we need to use hf_to_vllm_mapper
# to ensure correct loading of weights.
if hf_to_vllm_mapper := getattr(model, "hf_to_vllm_mapper", None):
+12 -125
View File
@@ -2,8 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only AfMoE model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
import torch
@@ -21,7 +20,6 @@ from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
MoERunner,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -36,10 +34,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.interfaces import (
EagleModelMixin,
MixtureOfExperts,
@@ -53,7 +47,6 @@ from vllm.model_executor.models.utils import (
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -481,120 +474,6 @@ class AfmoeModel(nn.Module, EagleModelMixin):
}
)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if (weight_name not in name) or ("self_attn.gate_proj" in name):
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
# Do not modify `name` since the loop may continue here
# Instead, create a new variable
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class AfmoeForCausalLM(
nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA, MixtureOfExperts
@@ -615,6 +494,17 @@ class AfmoeForCausalLM(
orig_to_new_suffix={
".router.gate.weight": ".gate.weight",
},
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# `.self_attn.gate_proj` is a gated-attention projection (not fused).
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
},
)
fall_back_to_pt_during_load = False
@@ -705,6 +595,3 @@ class AfmoeForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+18 -32
View File
@@ -20,7 +20,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper
from vllm.transformers_utils.configs.ovis import AIMv2Config
@@ -194,6 +194,13 @@ class AIMv2Transformer(nn.Module):
class AIMv2Model(torch.nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".fc1": (".fc13", 0),
".fc3": (".fc13", 1),
}
)
def __init__(
self,
config: AIMv2Config,
@@ -218,34 +225,13 @@ class AIMv2Model(torch.nn.Module):
return x
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".fc13", ".fc1", 0),
(".fc13", ".fc3", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# post_layernorm is optional in SiglipVisionModel
if (
name.startswith("trunk.post_trunk_norm")
and self.trunk.post_trunk_norm is None
):
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(
self,
# post_trunk_norm is optional (absent for clip-skip backbones).
skip_prefixes=(
["trunk.post_trunk_norm."]
if self.trunk.post_trunk_norm is None
else None
),
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+9 -62
View File
@@ -53,10 +53,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.v1.attention.backend import AttentionType
@@ -70,8 +66,8 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -404,66 +400,17 @@ class ApertusModel(nn.Module, EagleModelMixin):
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
]
params_dict = dict(self.named_parameters())
# we need to load the buffers for beta and eps (XIELU)
for name, buffer in self.named_buffers():
if name.endswith(".beta") or name.endswith(".eps"):
params_dict[name] = buffer
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
if "scale" in name or "zero_point" in name:
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class ApertusForCausalLM(
nn.Module, SupportsLoRA, SupportsPP, SupportsEagle, SupportsEagle3
):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
# LoRA specific attributes
@@ -548,4 +495,4 @@ class ApertusForCausalLM(
self,
skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
)
return loader.load_weights(weights)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+10 -40
View File
@@ -369,6 +369,14 @@ class BertModel(nn.Module, SupportsQuant):
packed_modules_mapping = {"qkv_proj": ["query", "key", "value"]}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self.query": (".self.qkv_proj", "q"),
".self.key": (".self.qkv_proj", "k"),
".self.value": (".self.qkv_proj", "v"),
}
)
def __init__(
self,
*,
@@ -400,43 +408,9 @@ class BertModel(nn.Module, SupportsQuant):
return self.encoder(hidden_states)
def _load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "query", "q"),
("qkv_proj", "key", "k"),
("qkv_proj", "value", "v"),
]
loaded_stacked_params = []
other_weights = []
params_dict = dict(self.named_parameters())
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
loaded_stacked_params.append(name)
break
else:
if name in params_dict:
other_weights.append((name, loaded_weight))
return other_weights, loaded_stacked_params
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
other_weights, loaded_stacked_params = self._load_weights(weights)
loader = AutoWeightsLoader(self, skip_prefixes=["pooler."])
loaded_params = loader.load_weights(other_weights)
loaded_params.update(loaded_stacked_params)
return loaded_params
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class BertPoolingModel(BertModel):
@@ -458,12 +432,8 @@ class BertPoolingModel(BertModel):
self.pooler = BertPooler(vllm_config.model_config)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
other_weights, loaded_stacked_params = self._load_weights(weights)
loader = AutoWeightsLoader(self)
loaded_params = loader.load_weights(other_weights)
loaded_params.update(loaded_stacked_params)
return loaded_params
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@default_pooling_type(seq_pooling_type="CLS")
+21 -33
View File
@@ -19,9 +19,9 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .interfaces import SupportsQuant
from .utils import AutoWeightsLoader, WeightsMapper
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -268,6 +268,14 @@ class BlipVisionModel(nn.Module, SupportsQuant):
main_input_name = "pixel_values"
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv", "q"),
".k_proj": (".qkv", "k"),
".v_proj": (".qkv", "v"),
}
)
def __init__(
self,
config: BlipVisionConfig,
@@ -316,38 +324,18 @@ class BlipVisionModel(nn.Module, SupportsQuant):
return self.post_layernorm(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
layer_count = len(self.encoder.layers)
skip_prefixes: list[str] = []
if self.post_layernorm is None:
skip_prefixes.append("post_layernorm.")
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
for name, loaded_weight in weights:
# post_layernorm is not needed in BlipVisionModel
if name.startswith("post_layernorm") and self.post_layernorm is None:
continue
# omit layers when num_hidden_layers_override is set
if name.startswith("encoder.layers"):
layer_idx = int(name.split(".")[2])
if layer_idx >= layer_count:
# omit layers when num_hidden_layers_override is set
def _filter(ws):
for name, weight in ws:
if name.startswith("encoder.layers.") and int(
name.split(".")[2]
) >= len(self.encoder.layers):
continue
yield name, weight
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
+16 -83
View File
@@ -37,10 +37,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
row_parallel_weight_loader,
)
from vllm.model_executor.model_loader.weight_utils import row_parallel_weight_loader
from vllm.model_executor.utils import set_weight_attrs
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -66,7 +63,8 @@ from .interfaces import (
SupportsQuant,
)
from .utils import (
is_pp_missing_parameter,
AutoWeightsLoader,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -922,6 +920,16 @@ class ChameleonForConditionalGeneration(
"gate_up_proj": ["gate_proj", "up_proj"],
}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
@@ -1023,81 +1031,6 @@ class ChameleonForConditionalGeneration(
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
# With tie_word_embeddings, we can skip lm_head.weight
# The weight might appear unnecessarily in the files if the model is
# processed with quantization, LoRA, fine-tuning, etc.
if self.config.tie_word_embeddings and "lm_head.weight" in name:
continue
use_default_weight_loading = False
if "vqmodel" in name:
if self.model.vqmodel is not None:
# We only do sharding for language model and
# not vqvae for now.
use_default_weight_loading = True
else:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
if name.endswith("kv_scale"):
remapped_kv_scale_name = name.replace(
".kv_scale", ".attn.kv_scale"
)
if remapped_kv_scale_name not in params_dict:
logger.warning_once(
"Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501
name,
remapped_kv_scale_name,
)
continue
else:
name = remapped_kv_scale_name
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
if use_default_weight_loading and name in params_dict:
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
skip_prefixes = ["lm_head."] if self.config.tie_word_embeddings else None
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+31 -60
View File
@@ -29,7 +29,6 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import SupportsQuant
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -56,7 +55,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import MultiModalEmbeddings, SupportsMultiModal
from .interfaces_base import default_pooling_type
from .utils import AutoWeightsLoader, maybe_prefix
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
from .vision import (
VisionEncoderInfo,
VisionFeatureSelectStrategy,
@@ -555,6 +554,14 @@ class CLIPEncoder(nn.Module):
class CLIPTextTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: CLIPTextConfig,
@@ -605,34 +612,19 @@ class CLIPTextTransformer(nn.Module):
return last_hidden_state
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class CLIPVisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: CLIPVisionConfig,
@@ -714,42 +706,21 @@ class CLIPVisionTransformer(nn.Module):
return encoder_outputs
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
layer_count = len(self.encoder.layers)
skip_prefixes: list[str] = []
if self.post_layernorm is None:
skip_prefixes.append("post_layernorm.")
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
for name, loaded_weight in weights:
# post_layernorm is not needed in CLIPVisionModel
if name.startswith("post_layernorm") and self.post_layernorm is None:
continue
# omit layers when num_hidden_layers_override is set
if name.startswith("encoder.layers"):
layer_idx = int(name.split(".")[2])
if layer_idx >= layer_count:
# Drop layers beyond num_hidden_layers_override.
def _filter(ws):
for name, w in ws:
if name.startswith("encoder.layers.") and int(
name.split(".")[2]
) >= len(self.encoder.layers):
continue
yield name, w
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
class CLIPVisionModel(nn.Module):
+15 -83
View File
@@ -18,7 +18,6 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -31,8 +30,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
row_parallel_weight_loader,
)
from vllm.model_executor.utils import set_weight_attrs
@@ -43,8 +40,8 @@ from .commandr import LayerNorm
from .interfaces import SupportsPP, SupportsQuant
from .utils import (
AutoWeightsLoader,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -462,88 +459,23 @@ class Cohere2MoeModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
for param_name, shard_name, shard_id in stacked_params_mapping:
if shard_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(shard_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant):
is_text_generation_model = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
)
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -599,4 +531,4 @@ class Cohere2MoeForCausalLM(nn.Module, SupportsPP, SupportsQuant):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self, skip_prefixes=["lm_head."])
return loader.load_weights(weights)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+14 -41
View File
@@ -20,10 +20,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.deepseek_v2 import (
DeepseekV2ForCausalLM,
DeepseekV2MLAAttention,
@@ -34,6 +30,7 @@ from vllm.multimodal.inputs import NestedTensors
from .interfaces import LocalArgmaxMixin
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_draft_quant_config,
maybe_prefix,
process_eagle_weight,
@@ -270,44 +267,20 @@ class DeepseekV2Eagle3Model(nn.Module):
return hidden_states, aux_output
# midlayer rename + gate_up / MLA fused_qkv_a merges
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={"midlayer.": "layers.0."},
orig_to_new_stacked={
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
".q_a_proj": (".fused_qkv_a_proj", 0),
".kv_a_proj_with_mqa": (".fused_qkv_a_proj", 1),
},
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
(".fused_qkv_a_proj", ".q_a_proj", 0),
(".fused_qkv_a_proj", ".kv_a_proj_with_mqa", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "midlayer." in name:
name = name.replace("midlayer.", "layers.0.")
# Remapping the name FP8 kv-scale
if "scale" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Eagle3DeepseekV2ForCausalLM(LocalArgmaxMixin, DeepseekV2ForCausalLM):
+4 -31
View File
@@ -32,7 +32,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
@@ -355,36 +354,6 @@ class DotsSwiGLUFFN(nn.Module):
x, _ = self.fc2(x)
return x
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("fc13", "fc1", 0),
("fc13", "fc3", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class DotsPatchEmbed(nn.Module):
def __init__(self, config):
@@ -622,6 +591,10 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
".attn.qkv_proj.": ".attn.qkv.",
".attn.out_proj.": ".attn.proj.",
},
orig_to_new_stacked={
".fc1.": (".fc13.", 0),
".fc3.": (".fc13.", 1),
},
orig_to_new_prefix={
"lm_head.": "language_model.lm_head.",
"model.": "language_model.model.",
+34 -93
View File
@@ -27,7 +27,6 @@ from collections.abc import Iterable
import torch
import torch.nn as nn
from transformers import PretrainedConfig
from vllm.config import VllmConfig
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -36,11 +35,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .llama import LlamaDecoderLayer
from .utils import is_pp_missing_parameter, maybe_prefix
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
class ErnieMultiTokenPredictorLayer(nn.Module):
@@ -147,6 +145,28 @@ class ErnieMTP(nn.Module):
super().__init__()
self.config = vllm_config.model_config.hf_config
# MTP weights are stored under a flat `mtp_*.0.` block in the
# checkpoint; rewrite them into `model.layers.{spec_layer}.*`.
spec_layer = self.config.num_hidden_layers
self.hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"model.mtp_emb_norm.0.": f"model.layers.{spec_layer}.mtp_emb_norm.",
"model.mtp_hidden_norm.0.": (
f"model.layers.{spec_layer}.mtp_hidden_norm."
),
"model.mtp_linear_proj.0.": (
f"model.layers.{spec_layer}.mtp_linear_proj."
),
"model.mtp_block.0.": f"model.layers.{spec_layer}.mtp_block.",
},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
self.model = ErnieMultiTokenPredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
@@ -185,94 +205,15 @@ class ErnieMTP(nn.Module):
return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Checkpoint bundles the full base model; only MTP, embed_tokens and
# lm_head weights belong to this module.
def _filter(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
for name, weight in weights:
if any(k in name for k in ("mtp", "embed_tokens", "lm_head")):
yield name, weight
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if self.config.tie_word_embeddings and name.endswith("lm_head.weight"):
continue
if "rotary_emb.inv_freq" in name:
continue
if "mtp" in name:
name = self._rewrite_spec_layer_name(self.config, name)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
if "mtp" not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# According to DeepSeek-V3 Technical Report, MTP modules
# shares embedding layer. We only load the first weights.
if "mtp_" not in name and (
"embed_tokens" not in name and "lm_head" not in name
):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def _rewrite_spec_layer_name(self, config: PretrainedConfig, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
"""
spec_layer_weight_names = [
"embed_tokens",
"mtp_emb_norm",
"mtp_hidden_norm",
"mtp_linear_proj",
]
layer_idx = config.num_hidden_layers
for weight_name in spec_layer_weight_names:
if weight_name in name:
name = name.replace(
f"model.{weight_name}.0.",
f"model.layers.{layer_idx}.{weight_name}.",
)
return name
name = name.replace(
"model.mtp_block.0.", f"model.layers.{layer_idx}.mtp_block."
)
return name
skip_prefixes = ["lm_head"] if self.config.tie_word_embeddings else []
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
+29 -143
View File
@@ -15,8 +15,7 @@
# limitations under the License.
"""Inference-only K-EXAONE-236B-A22B model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
import torch
@@ -32,7 +31,6 @@ from vllm.distributed import (
)
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import ReplicatedLinear
@@ -43,10 +41,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from .exaone4 import Exaone4Attention as ExaoneMoeAttention
@@ -55,8 +49,8 @@ from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -325,142 +319,21 @@ class ExaoneMoeModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
# Skip loading extra parameters for GPTQ/modelopt models.
ignore_suffixes = (
".bias",
"_bias",
".k_scale",
"_k_scale",
".v_scale",
"_v_scale",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if name.startswith("mtp."):
continue
if "rotary_emb.inv_freq" in name:
continue
if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
# Models trained using ColossalAI may include these tensors in
# the checkpoint. Skip them.
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
# Do not modify `name` since the loop may continue here
# Instead, create a new variable
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
# Skip loading extra parameters for GPTQ/modelopt models.
if (
name_mapped.endswith(ignore_suffixes)
and name_mapped not in params_dict
):
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip loading extra parameters for GPTQ/modelopt models.
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_experts.gate_proj": (".shared_experts.gate_up_proj", 0),
".shared_experts.up_proj": (".shared_experts.gate_up_proj", 1),
}
)
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -554,5 +427,18 @@ class ExaoneMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
skip_prefixes=(
["lm_head.", "mtp."] if self.config.tie_word_embeddings else ["mtp."]
),
# Skip loading extra parameters for GPTQ/modelopt models.
ignore_unexpected_suffixes=[
".bias",
"_bias",
".k_scale",
"_k_scale",
".v_scale",
"_v_scale",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
],
)
return loader.load_weights(weights)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+14 -55
View File
@@ -18,15 +18,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.exaone_moe import ExaoneMoeDecoderLayer
from vllm.sequence import IntermediateTensors
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
maybe_prefix,
)
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
logger = init_logger(__name__)
@@ -35,6 +30,17 @@ KVCache = tuple[torch.Tensor, torch.Tensor]
@support_torch_compile
class ExaoneMoeMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# Scope to dense mlp; experts are handled separately.
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -128,55 +134,8 @@ class ExaoneMoeMultiTokenPredictor(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@support_torch_compile
+16 -63
View File
@@ -35,10 +35,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.config import set_default_rope_theta
@@ -52,7 +48,7 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -496,63 +492,6 @@ class FalconH1Model(nn.Module):
hidden_states = self.final_layernorm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "A_log" in name:
name = name.replace("A_log", "A")
if "mamba" in name:
name = name.replace("mamba", "mamba.mamba")
if "scale" in name:
# Remapping the name of kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class FalconH1ForCausalLM(
nn.Module,
@@ -567,6 +506,20 @@ class FalconH1ForCausalLM(
"gate_up_proj": ["gate_proj", "up_proj"],
}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
"A_log": "A",
"mamba": "mamba.mamba",
},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
embedding_modules = {
"embed_tokens": "input_embeddings",
"lm_head": "output_embeddings",
@@ -694,4 +647,4 @@ class FalconH1ForCausalLM(
self,
skip_prefixes=(["lm_head."] if self.tie_word_embeddings else None),
)
return loader.load_weights(weights)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+10 -30
View File
@@ -31,7 +31,6 @@ from vllm.model_executor.layers.linear import (
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.whisper_utils import (
ISO639_1_SUPPORTED_LANGS,
)
@@ -618,6 +617,14 @@ class FunASRAudioInputs(TensorSchema):
class FunASREncoder(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self_attn.q_proj": (".self_attn.qkv", "q"),
".self_attn.k_proj": (".self_attn.qkv", "k"),
".self_attn.v_proj": (".self_attn.qkv", "v"),
}
)
def __init__(
self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False
):
@@ -637,35 +644,8 @@ class FunASREncoder(nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with mapping from HuggingFace format."""
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("self_attn.qkv.", "self_attn.q_proj.", "q"),
("self_attn.qkv.", "self_attn.k_proj.", "k"),
("self_attn.qkv.", "self_attn.v_proj.", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict.get(name)
if param is not None:
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class FunASRModel(nn.Module):
+4 -39
View File
@@ -44,7 +44,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .gemma4 import Gemma4MLP, _get_text_config
@@ -412,44 +411,6 @@ class Gemma4MultiTokenPredictor(nn.Module):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids) * self.normalizer
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
params_dict.update(dict(self.named_buffers()))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def forward(
self,
input_ids: torch.Tensor,
@@ -502,6 +463,10 @@ class Gemma4MTP(nn.Module):
"pre_projection.": "model.pre_projection.",
"post_projection.": "model.post_projection.",
},
orig_to_new_stacked={
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+12 -28
View File
@@ -76,7 +76,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -618,6 +617,16 @@ class Glm4vVisionEmbeddings(nn.Module):
class Glm4vVisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".attn.q.": (".attn.qkv.", "q"),
".attn.k.": (".attn.qkv.", "k"),
".attn.v.": (".attn.qkv.", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
}
)
def __init__(
self,
text_config: Glm4vTextConfig,
@@ -958,33 +967,8 @@ class Glm4vVisionTransformer(nn.Module):
return x
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("attn.qkv.", "attn.q.", "q"),
("attn.qkv.", "attn.k.", "k"),
("attn.qkv.", "attn.v.", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Glm4vProcessingInfo(BaseProcessingInfo):
+10 -26
View File
@@ -48,7 +48,6 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -431,6 +430,14 @@ class HunYuanVisionPatchMerger(nn.Module):
class HunYuanVisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv", "q"),
".k_proj": (".qkv", "k"),
".v_proj": (".qkv", "v"),
}
)
def __init__(
self,
vision_config: HunYuanVLVisionConfig,
@@ -529,31 +536,8 @@ class HunYuanVisionTransformer(nn.Module):
return image_embeds_list
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv", ".q_proj", "q"),
(".qkv", ".k_proj", "k"),
(".qkv", ".v_proj", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _hunyuan_vl_field_config(hf_inputs: Mapping[str, torch.Tensor]):
@@ -38,8 +38,8 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .utils import AutoWeightsLoader, WeightsMapper
from .vision import is_vit_use_data_parallel, run_dp_sharded_vision_model
@@ -352,6 +352,14 @@ class Idefics2Encoder(nn.Module):
class Idefics2VisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: Idefics2VisionConfig,
@@ -453,42 +461,22 @@ class Idefics2VisionTransformer(nn.Module):
return last_hidden_state
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
# head is a pooling header absent from this model.
skip_prefixes = ["head."]
if not self.require_post_norm:
skip_prefixes.append("post_layernorm.")
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
layer_count = len(self.encoder.layers)
for name, loaded_weight in weights:
# skip pooling header
if name.startswith("head."):
continue
# post_layernorm is optional
if name.startswith("post_layernorm.") and not self.require_post_norm:
continue
# omit layers when num_hidden_layers_override is set
if name.startswith("encoder.layers."):
layer_idx = int(name.split(".")[2])
if layer_idx >= layer_count:
def _filter(ws: Iterable[tuple[str, torch.Tensor]]):
# Drop layers beyond num_hidden_layers_override.
for name, w in ws:
if (
name.startswith("encoder.layers.")
and int(name.split(".")[2]) >= layer_count
):
continue
yield name, w
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name or self.use_data_parallel:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
+10 -28
View File
@@ -25,9 +25,6 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
)
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
SupportsLoRA,
@@ -652,6 +649,14 @@ class Siglip2Encoder(nn.Module):
class Siglip2VisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: PixelShuffleSiglip2VisionConfig,
@@ -715,31 +720,8 @@ class Siglip2VisionTransformer(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _resolve_vision_token_id(model_config: ModelConfig, vision_token: str) -> int:
+10 -52
View File
@@ -34,10 +34,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -76,7 +72,6 @@ from .utils import (
AutoWeightsLoader,
WeightsMapper,
init_vllm_registered_model,
is_pp_missing_parameter,
maybe_prefix,
)
from .vision import is_vit_use_data_parallel
@@ -718,6 +713,14 @@ class KeyeSiglipVisionModel(nn.Module):
config_class = PretrainedConfig
main_input_name = "pixel_values"
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: PretrainedConfig,
@@ -776,53 +779,8 @@ class KeyeSiglipVisionModel(nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "head.attention" in name or "head.layernorm" in name:
continue
if "head.mlp" in name or "head.probe" in name:
continue
for (
param_name,
weight_name,
shard_id,
) in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param,
"weight_loader",
default_weight_loader,
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self, skip_prefixes=["vision_model.head."])
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Projector(nn.Module):
+10 -32
View File
@@ -17,7 +17,6 @@ from vllm.config.multimodal import BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import PromptType, TokensPrompt
from vllm.model_executor.model_loader import DefaultModelLoader
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import (
SupportsMultiModal,
SupportsPP,
@@ -81,6 +80,14 @@ class KimiAudioWhisperEncoder(WhisperEncoder):
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
},
)
def __init__(
self, *, vllm_config: VllmConfig, prefix: str = "", init_in_fp32: bool = False
):
@@ -102,37 +109,8 @@ class KimiAudioWhisperEncoder(WhisperEncoder):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
# -----------------------------------------------------------------------------
+18 -42
View File
@@ -32,7 +32,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP, SupportsQuant
@@ -41,7 +40,6 @@ from .utils import (
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -297,6 +295,20 @@ class Lfm2ShortConvDecoderLayer(nn.Module):
@support_torch_compile
class Lfm2Model(nn.Module):
# HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision
# with the inner .conv.conv child (ShortConv has a child self.conv, so
# naming the container .conv too makes _match_target_modules match both).
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={".conv.": ".short_conv."},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".w1": (".w13", 0),
".w3": (".w13", 1),
},
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -375,40 +387,8 @@ class Lfm2Model(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".w13", ".w1", 0),
(".w13", ".w3", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if ".conv." in name:
name = name.replace(".conv.", ".short_conv.", 1)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Use segment-boundary matching (trailing dot) to prevent
# e.g. ".w1" from matching inside ".w13" in pre-fused keys.
if weight_name + "." not in name:
continue
name = name.replace(weight_name + ".", param_name + ".")
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Lfm2ForCausalLM(
@@ -427,12 +407,8 @@ class Lfm2ForCausalLM(
"in_proj": ["in_proj"],
}
# HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision
# with the inner .conv.conv child (ShortConv has a child self.conv, so
# naming the container .conv too makes _match_target_modules match both)
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={".conv.": ".short_conv."},
)
# Reuse the backbone mapper so LoRA/quantization see the same name mapping.
hf_to_vllm_mapper = Lfm2Model.hf_to_vllm_mapper
# LoRA specific attributes
embedding_modules = {
+23 -37
View File
@@ -23,9 +23,8 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from .utils import maybe_prefix
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
from .vision import (
is_vit_use_data_parallel,
resolve_visual_encoder_outputs,
@@ -458,6 +457,14 @@ class Siglip2VisionTransformer(nn.Module):
class Siglip2Model(torch.nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: Siglip2VisionConfig,
@@ -501,42 +508,21 @@ class Siglip2Model(torch.nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
skip_prefixes = []
if self.vision_model.post_layernorm is None:
skip_prefixes.append("vision_model.post_layernorm.")
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
# Drop layers omitted by num_hidden_layers_override.
layer_count = len(self.vision_model.encoder.layers)
for name, loaded_weight in weights:
# post_layernorm is optional in Siglip2Model
if (
name.startswith("vision_model.post_layernorm")
and self.vision_model.post_layernorm is None
):
continue
# omit layers when num_hidden_layers_override is set
if name.startswith("vision_model.encoder.layers"):
layer_idx = int(name.split(".")[3])
if layer_idx >= layer_count:
def _filter(ws):
for n, w in ws:
if (
n.startswith("vision_model.encoder.layers.")
and int(n.split(".")[3]) >= layer_count
):
continue
yield n, w
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
+19 -30
View File
@@ -32,18 +32,33 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.llama4 import Llama4DecoderLayer, Llama4ForCausalLM
from vllm.model_executor.models.utils import extract_layer_index
from .interfaces import SupportsMultiModal
from .utils import AutoWeightsLoader, maybe_prefix, process_eagle_weight
from .utils import (
AutoWeightsLoader,
WeightsMapper,
maybe_prefix,
process_eagle_weight,
)
logger = init_logger(__name__)
@support_torch_compile
class LlamaModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={"model.": ""},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def __init__(
self,
*,
@@ -108,34 +123,8 @@ class LlamaModel(nn.Module):
return hidden_states, hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
name = name.removeprefix("model.")
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
for name in params_dict:
assert name in loaded_params, f"{name} is not loaded!"
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def validate_and_update_config(
self, start_layer_id: int, quant_config: QuantizationConfig | None = None
+14 -34
View File
@@ -14,14 +14,11 @@ from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_draft_quant_config,
maybe_prefix,
process_eagle_weight,
@@ -53,6 +50,17 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
@support_torch_compile
class LlamaModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
}
)
def __init__(
self,
*,
@@ -116,36 +124,8 @@ class LlamaModel(nn.Module):
return hidden_states, hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
# Remapping the name FP8 kv-scale or zero point.
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class EagleLlamaForCausalLM(LlamaForCausalLM):
+14 -36
View File
@@ -18,15 +18,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.llama import LlamaDecoderLayer, LlamaForCausalLM
from vllm.multimodal.inputs import NestedTensors
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_draft_quant_config,
maybe_prefix,
process_eagle_weight,
@@ -253,39 +250,20 @@ class LlamaModel(nn.Module):
return hidden_states, aux_output
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={"midlayer.": "layers.0."},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "midlayer." in name:
name = name.replace("midlayer.", "layers.0.")
# Remapping the name FP8 kv-scale or zero point.
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Eagle3LlamaForCausalLM(LlamaForCausalLM):
+31 -103
View File
@@ -34,11 +34,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen2 import Qwen2DecoderLayer
from vllm.sequence import IntermediateTensors
from .utils import maybe_prefix
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
class MiMoMultiTokenPredictorLayer(nn.Module):
@@ -163,6 +162,28 @@ class MiMoMTP(nn.Module):
self.config.hidden_size,
prefix=maybe_prefix(prefix, "lm_head"),
)
# Checkpoint stores MTP layers 0-indexed and without the `mtp_block`
# wrapper around the transformer block; remap onto the offset index.
start = self.config.num_hidden_layers
self.hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={
".self_attn.": ".mtp_block.self_attn.",
".mlp.": ".mtp_block.mlp.",
".input_layernorm.": ".mtp_block.input_layernorm.",
".post_attention_layernorm.": ".mtp_block.post_attention_layernorm.",
},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
orig_to_new_prefix={
f"model.mtp_layers.{i}.": f"model.mtp_layers.{i + start}."
for i in range(self.config.num_nextn_predict_layers)
},
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
@@ -190,105 +211,12 @@ class MiMoMTP(nn.Module):
return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# The checkpoint carries the full model; keep only the MTP layers and
# the shared embedding/head.
def mtp_weights():
for name, weight in weights:
if "mtp_layers" in name or "embed_tokens" in name or "lm_head" in name:
yield name, weight
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
name = self.map_model_name_to_mtp_param_name(name)
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
if "mtp_layers" not in name:
break
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if "mtp_layers" not in name and (
"embed_tokens" not in name and "lm_head" not in name
):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
def map_model_name_to_mtp_param_name(self, name: str) -> str:
import regex as re
# append mtp_start_layer_idx
pattern = r"(model\.mtp_layers\.)(\d+)(\.)"
match = re.match(pattern, name)
if match:
original_num = int(match.group(2))
new_num = original_num + self.config.num_hidden_layers
name = name.replace(match.group(), f"{match.group(1)}{new_num}.")
# check for early turn
name_without_prefix = [
"token_layernorm",
"hidden_layernorm",
"input_proj",
"final_layernorm",
]
for sub_name in name_without_prefix:
if sub_name in name:
return name
# add mtp_block
pattern = r"(model\.mtp_layers\.\d+\.)"
match = re.match(pattern, name)
if match:
name = name.replace(match.group(), match.group() + "mtp_block.")
return name
def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str:
"""
Rewrite the weight name to match the format of the original model.
Add .mtp_block for modules in transformer layer block for spec layer
"""
spec_layer_weight_names = [
"embed_tokens",
"enorm",
"hnorm",
"eh_proj",
"shared_head",
]
spec_layer_weight = False
for weight_name in spec_layer_weight_names:
if weight_name in name:
spec_layer_weight = True
break
if not spec_layer_weight:
# treat rest weights as weights for transformer layer block
name = name.replace(
f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
)
return name
loader = AutoWeightsLoader(self)
return loader.load_weights(mtp_weights(), mapper=self.hf_to_vllm_mapper)
+9 -23
View File
@@ -29,7 +29,6 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.vision import is_vit_use_data_parallel
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargsItems
@@ -378,6 +377,13 @@ class MiMoVisionBlock(nn.Module):
class MiMoVisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
"mlp.gate_proj": ("mlp.gate_up_proj", 0),
"mlp.up_proj": ("mlp.gate_up_proj", 1),
}
)
def __init__(
self,
vision_cfg: PretrainedConfig,
@@ -627,28 +633,8 @@ class MiMoVisionTransformer(nn.Module):
return x
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("mlp.gate_up_proj", "mlp.gate_proj", 0),
("mlp.gate_up_proj", "mlp.up_proj", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiMoV2OmniProcessingInfo(BaseProcessingInfo):
+10 -26
View File
@@ -25,7 +25,6 @@ from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateShapeCalculator,
)
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -619,6 +618,14 @@ class MiniCPMV4_6ProcessingInfo(MiniCPMVProcessingInfo):
class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
"q_proj": ("qkv_proj", "q"),
"k_proj": ("qkv_proj", "k"),
"v_proj": ("qkv_proj", "v"),
}
)
def __init__(
self,
config,
@@ -667,31 +674,8 @@ class MiniCPMV4_6ViTWindowAttentionSelfAttn(nn.Module):
return out
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
mapped_name = name.replace(weight_name, param_name, 1)
if mapped_name not in params_dict:
continue
param = params_dict[mapped_name]
param.weight_loader(param, loaded_weight, shard_id)
break
else:
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiniCPMV4_6ViTWindowAttentionMerger(nn.Module):
+24 -125
View File
@@ -41,7 +41,6 @@ from vllm.distributed import (
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -57,17 +56,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from .interfaces import EagleModelMixin, SupportsEagle3, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -111,6 +106,7 @@ class MiniMaxM2MoE(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.experts",
router_logits_dtype=torch.float32,
ckpt_names=("w1", "w2", "w3"),
)
self.gate = GateLinear(
@@ -327,6 +323,15 @@ class MiniMaxM2DecoderLayer(nn.Module):
class MiniMaxM2Model(nn.Module, EagleModelMixin):
fall_back_to_pt_during_load = False
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -409,115 +414,23 @@ class MiniMaxM2Model(nn.Module, EagleModelMixin):
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = self.get_expert_mapping()
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
spec_layer = get_spec_layer_idx_from_weight_name(self.config, name)
if spec_layer is not None:
continue # skip spec decode layers for main model
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if ("mlp.experts." in name) and name not in params_dict:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
# Remap qkv_proj.[kv]_scale to attn.[kv]_scale
if name.endswith((".k_scale", ".v_scale")):
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is not None and remapped_name in params_dict:
param = params_dict[remapped_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
break
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
# Skip spec-decode (MTP) layers; they are appended after the main
# decoder layers and have no destination in the main model.
skip_prefixes = None
num_mtp = getattr(self.config, "num_mtp_modules", 0)
if num_mtp:
base = self.config.num_hidden_layers
skip_prefixes = [f"layers.{base + i}." for i in range(num_mtp)]
loader = AutoWeightsLoader(
self,
skip_prefixes=skip_prefixes,
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
hf_to_vllm_mapper = MiniMaxM2Model.hf_to_vllm_mapper
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -577,17 +490,3 @@ class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
def get_spec_layer_idx_from_weight_name(
config: PretrainedConfig, weight_name: str
) -> int | None:
if hasattr(config, "num_mtp_modules") and (config.num_mtp_modules > 0):
layer_idx = config.num_hidden_layers
for i in range(config.num_mtp_modules):
if weight_name.startswith(f"model.layers.{layer_idx + i}."):
return layer_idx + i
return None
+15 -113
View File
@@ -24,8 +24,7 @@
# limitations under the License.
"""Inference-only Mixtral model."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
import torch
@@ -42,7 +41,6 @@ from vllm.distributed import (
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -57,17 +55,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -142,6 +136,7 @@ class MixtralMoE(nn.Module):
prefix=f"{prefix}.experts",
enable_eplb=self.enable_eplb,
num_redundant_experts=self.n_redundant_experts,
ckpt_names=("w1", "w2", "w3"),
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -295,6 +290,15 @@ class MixtralDecoderLayer(nn.Module):
@support_torch_compile
class MixtralModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -364,114 +368,15 @@ class MixtralModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name_mapped, self):
continue
if (
name_mapped.endswith(".bias") or name_mapped.endswith("_bias")
) and name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
continue
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
fall_back_to_pt_during_load = False
hf_to_vllm_mapper = MixtralModel.hf_to_vllm_mapper
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -582,6 +487,3 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP, MixtureOfExperts):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+10 -32
View File
@@ -672,6 +672,14 @@ class MolmoDecoderNormAfterLayer(MolmoDecoderLayer):
class MolmoVisionBackbone(nn.Module, SupportsQuant):
packed_modules_mapping = {"merged_linear": ["gate_proj", "up_proj"]}
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# image_projector gate_up merge
"gate_proj": ("merged_linear", 0),
"up_proj": ("merged_linear", 1),
}
)
def __init__(
self,
config: PretrainedConfig,
@@ -800,38 +808,8 @@ class MolmoVisionBackbone(nn.Module, SupportsQuant):
return image_features
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("merged_linear", "gate_proj", 0),
("merged_linear", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@support_torch_compile
+16 -37
View File
@@ -709,6 +709,20 @@ class Molmo2VisionBackbone(nn.Module, SupportsQuant):
"merged_linear": ["gate_proj", "up_proj"],
}
# Runs after the top-level mapper, so image_pooling_2d/image_projector
# source names are already renamed to q/k/v_proj and gate/up_proj.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
"wq": ("merged_qkv", "q"),
"wk": ("merged_qkv", "k"),
"wv": ("merged_qkv", "v"),
"k_proj": ("merged_kv", 0),
"v_proj": ("merged_kv", 1),
"gate_proj": ("merged_linear", 0),
"up_proj": ("merged_linear", 1),
},
)
def __init__(
self,
vit_config: VitConfig,
@@ -839,43 +853,8 @@ class Molmo2VisionBackbone(nn.Module, SupportsQuant):
]
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("merged_qkv", "wq", "q"),
("merged_qkv", "wk", "k"),
("merged_qkv", "wv", "v"),
("merged_kv", "k_proj", 0),
("merged_kv", "v_proj", 1),
("merged_linear", "gate_proj", 0),
("merged_linear", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Molmo2Attention(nn.Module):
+5 -33
View File
@@ -33,7 +33,6 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
AudioItem,
@@ -742,37 +741,6 @@ class GatedMLP(nn.Module):
x, _ = self.down_proj(x)
return x
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
target_name = name
for param_name, weight_name, shard_id in stacked_params_mapping:
components = target_name.split(".")
if weight_name not in components:
continue
target_name = ".".join(
param_name if component == weight_name else component
for component in components
)
param = params_dict[target_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[target_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(target_name)
return loaded_params
@support_torch_compile(
dynamic_arg_dims={
@@ -1478,7 +1446,11 @@ class MossAudioModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
"language_model.embed_tokens.": "language_model.model.embed_tokens.",
"language_model.layers.": "language_model.model.layers.",
"language_model.norm.": "language_model.model.norm.",
}
},
orig_to_new_stacked={
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def get_mm_mapping(self) -> MultiModelKeys:
+10 -105
View File
@@ -34,7 +34,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -49,13 +48,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -338,106 +336,16 @@ class OlmoeModel(nn.Module):
hidden_states = self.norm(hidden_states)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
if name.endswith("kv_scale"):
remapped_kv_scale_name = name.replace(
".kv_scale", ".attn.kv_scale"
)
if remapped_kv_scale_name not in params_dict:
logger.warning_once(
"Found kv scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv-scale is not loaded.", # noqa: E501
name,
remapped_kv_scale_name,
)
continue
else:
name = remapped_kv_scale_name
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class OlmoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -496,7 +404,4 @@ class OlmoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+21 -54
View File
@@ -48,10 +48,6 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
@@ -81,7 +77,6 @@ from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
is_pp_missing_parameter,
maybe_prefix,
)
from .vision import get_vit_attn_backend
@@ -884,6 +879,14 @@ class SiglipVisionTransformer(nn.Module):
class SiglipVisionModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config,
@@ -928,55 +931,19 @@ class SiglipVisionModel(nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if "head.attention" in name or "head.layernorm" in name:
continue
if "head.mlp" in name or "head.probe" in name:
continue
if "packing_position_embedding" in name:
continue
for (
param_name,
weight_name,
shard_id,
) in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param,
"weight_loader",
default_weight_loader,
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
# Skip the SigLIP attention pooling head and packing pos embedding
# present in the checkpoint but absent from this vision tower.
loader = AutoWeightsLoader(
self,
skip_substrs=[
"head.attention",
"head.layernorm",
"head.mlp",
"head.probe",
"packing_position_embedding",
],
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@MULTIMODAL_REGISTRY.register_processor(
+11 -84
View File
@@ -37,7 +37,6 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.linear import (
QKVParallelLinear,
@@ -51,16 +50,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -289,6 +284,7 @@ class PhiMoE(nn.Module):
tp_size=tp_size,
custom_routing_function=phimoe_routing_function,
prefix=f"{prefix}.experts",
ckpt_names=("w1", "w2", "w3"),
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -516,84 +512,18 @@ class PhiMoEModel(nn.Module):
hidden_states = self.norm(hidden_states)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="w1",
ckpt_down_proj_name="w2",
ckpt_up_proj_name="w3",
num_experts=self.config.num_local_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
fall_back_to_pt_during_load = False
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -654,7 +584,4 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+12 -27
View File
@@ -65,7 +65,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.evs import (
@@ -613,6 +612,16 @@ class Qwen2_5_VisionPatchMerger(nn.Module):
class Qwen2_5_VisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".attn.q.": (".attn.qkv.", "q"),
".attn.k.": (".attn.qkv.", "k"),
".attn.v.": (".attn.qkv.", "v"),
".mlp.gate_proj.": (".mlp.gate_up_proj.", 0),
".mlp.up_proj.": (".mlp.gate_up_proj.", 1),
}
)
def __init__(
self,
vision_config: Qwen2_5_VLVisionConfig,
@@ -1116,32 +1125,8 @@ class Qwen2_5_VisionTransformer(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("attn.qkv.", "attn.q.", "q"),
("attn.qkv.", "attn.k.", "k"),
("attn.qkv.", "attn.v.", "v"),
("mlp.gate_up_proj.", "mlp.gate_proj.", 0),
("mlp.gate_up_proj.", "mlp.up_proj.", 1),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
+26 -122
View File
@@ -42,7 +42,6 @@ from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -58,14 +57,13 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -418,124 +416,22 @@ class Qwen2MoeModel(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Remapping the name of FP8 kv-scale.
if name.endswith("kv_scale"):
remapped_kv_scale_name = name.replace(
".kv_scale", ".attn.kv_scale"
)
if remapped_kv_scale_name not in params_dict:
logger.warning_once(
"Found kv_scale in the checkpoint (e.g. %s), but not found the expected name in the model (e.g. %s). kv_scale is not loaded.", # noqa: E501
name,
remapped_kv_scale_name,
)
continue
else:
name = remapped_kv_scale_name
# GGUF: make sure that shared_expert_gate is a 2D tensor.
if (
"mlp.shared_expert_gate" in name
and len(loaded_weight.shape) == 1
):
loaded_weight = loaded_weight[None, :]
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
fall_back_to_pt_during_load = False
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
}
)
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -596,8 +492,16 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def _maybe_reshape(
weights: Iterable[tuple[str, torch.Tensor]],
) -> Iterable[tuple[str, torch.Tensor]]:
for name, loaded_weight in weights:
# GGUF: make sure that shared_expert_gate is a 2D tensor.
if "mlp.shared_expert_gate" in name and loaded_weight.dim() == 1:
loaded_weight = loaded_weight[None, :]
yield name, loaded_weight
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
loader = AutoWeightsLoader(self)
return loader.load_weights(
_maybe_reshape(weights), mapper=self.hf_to_vllm_mapper
)
+11 -26
View File
@@ -60,7 +60,6 @@ from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
@@ -522,6 +521,15 @@ class Qwen2VisionPatchMerger(nn.Module):
class Qwen2VisionTransformer(nn.Module):
# Vision checkpoints store qkv pre-fused; merge separate q/k/v into qkv.
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv", "q"),
".k_proj": (".qkv", "k"),
".v_proj": (".qkv", "v"),
}
)
def __init__(
self,
vision_config: Qwen2VLVisionConfig,
@@ -742,31 +750,8 @@ class Qwen2VisionTransformer(nn.Module):
return x
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
def _create_qwen2vl_field_factory(
+30 -224
View File
@@ -24,8 +24,7 @@
# limitations under the License.
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
import torch
from torch import nn
@@ -37,12 +36,7 @@ from vllm.distributed import (
get_pp_group,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm as Qwen3_5RMSNorm,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm as Qwen3_5RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
QwenGatedDeltaNetAttention,
@@ -57,16 +51,9 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_5 import (
Qwen3_5Config,
Qwen3_5TextConfig,
)
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import (
Qwen3_5MoeConfig,
Qwen3_5MoeTextConfig,
@@ -89,6 +76,7 @@ from .qwen3_next import (
Qwen3NextModel,
Qwen3NextSparseMoeBlock,
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from .qwen3_vl import (
Qwen3_VisionTransformer,
@@ -100,9 +88,9 @@ from .qwen3_vl import (
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
_merge_multimodal_embeddings,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -210,6 +198,17 @@ class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
}
)
class Qwen3_5Model(Qwen3NextModel):
# Qwen3.5 ships the GDN in_proj checkpoints separately (qwen3-next
# pre-fuses them); fuse them on top of the qwen3-next QKV/gate_up mapping.
hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_stacked={
".in_proj_qkv": (".in_proj_qkvz", (0, 1, 2)),
".in_proj_z": (".in_proj_qkvz", 3),
".in_proj_b": (".in_proj_ba", 0),
".in_proj_a": (".in_proj_ba", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super(Qwen3NextModel, self).__init__()
@@ -222,6 +221,7 @@ class Qwen3_5Model(Qwen3NextModel):
self.num_redundant_experts = eplb_config.num_redundant_experts
self.config = config
self.quant_config = vllm_config.quant_config
self.vocab_size = config.vocab_size
@@ -251,212 +251,21 @@ class Qwen3_5Model(Qwen3NextModel):
self.aux_hidden_state_layers: tuple[int, ...] = ()
def load_fused_expert_weights(
self,
name: str,
params_dict: dict,
loaded_weight: torch.Tensor,
shard_id: str,
num_experts: int,
) -> bool:
param = params_dict[name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
loaded_local_expert = False
for expert_id in range(num_experts):
curr_expert_weight = loaded_weight[expert_id]
success = weight_loader(
param,
curr_expert_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_local_expert = True
return loaded_local_expert
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
# GDN
("in_proj_qkvz", "in_proj_qkv", (0, 1, 2)),
("in_proj_qkvz", "in_proj_z", 3),
# self attention
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
# mlp
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
("in_proj_ba", "in_proj_b", 0),
("in_proj_ba", "in_proj_a", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
is_fused_expert = False
fused_expert_params_mapping: list[tuple[str, str, int, str]] = []
for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_up_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="gate_up_proj",
num_experts=1,
):
if shard_id == "w3":
continue
parts = ckpt_name.split(".")
fused_expert_params_mapping.append(
(f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id)
mapper = self.hf_to_vllm_mapper
# FSE must match construction (Qwen3NextSparseMoeBlock): reroute the
# shared expert into the extra fused slot only when AITER FSE is both
# requested and compatible with the quant spec.
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
_is_shared_expert_fse_compatible(self.quant_config)
)
if is_fse:
num_routed = self.config.num_experts
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
num_experts = (
self.config.num_experts if hasattr(self.config, "num_experts") else 0
)
from vllm.config import get_current_vllm_config
from .qwen3_next import _is_shared_expert_fse_compatible
is_fse = (
rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
and _is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if name.startswith("mtp."):
continue
# Remapping the name of FP8 kv-scale.
if name.endswith("scale"):
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
# FSE: remap shared_expert weights to fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_experts}.",
)
is_fused_expert = False
expert_params_mapping = self.get_expert_mapping()
for param_name, weight_name, shard_id in stacked_params_mapping:
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
is_fused_expert = True
expert_params_mapping = fused_expert_params_mapping
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# name = apply_attn_prefix(name, params_dict)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name_mapped, self):
continue
if is_fused_expert:
# qwen3.5 no need to transpose
# loaded_weight = loaded_weight.transpose(-1, -2)
if "experts.gate_up_proj" in name:
loaded_weight = loaded_weight.chunk(2, dim=-2)
success_w1 = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight[0],
"w1",
num_experts,
)
success_w3 = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight[1],
"w3",
num_experts,
)
success = success_w1 and success_w3
else:
# down_proj
success = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight,
shard_id,
num_experts,
)
if success:
name = name_mapped
break
else:
# Skip loading extra bias for GPTQ models.
if (
name_mapped.endswith(".bias")
or name_mapped.endswith("_bias")
) and name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
weight_loader = param.weight_loader
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
logger.warning_once(
f"Parameter {name} not found in params_dict, skip loading"
)
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
class Qwen3_5ForCausalLMBase(
@@ -566,9 +375,6 @@ class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLMBase, QwenNextMixtureOfExperts):
# set MoE hyperparameters
self.set_moe_parameters()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
########################################################
# Qwen3_5-Dense
+24 -189
View File
@@ -2,29 +2,32 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Inference-only Qwen3_5 MTP model."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
import torch
from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.interfaces import LocalArgmaxMixin
from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts
from vllm.model_executor.models.qwen3_5 import (
Qwen3_5DecoderLayer,
Qwen3_5Model,
Qwen3_5RMSNorm,
)
from vllm.model_executor.models.qwen3_next import (
QwenNextMixtureOfExperts,
_is_shared_expert_fse_compatible,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
@@ -37,8 +40,8 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
_merge_multimodal_embeddings,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
@@ -58,6 +61,8 @@ logger = init_logger(__name__)
}
)
class Qwen3_5MultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3_5Model.hf_to_vllm_mapper
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -159,188 +164,18 @@ class Qwen3_5MultiTokenPredictor(nn.Module):
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
def load_fused_expert_weights(
self,
name: str,
params_dict: dict,
loaded_weight: torch.Tensor,
shard_id: str,
num_experts: int,
) -> bool:
param = params_dict[name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
loaded_local_expert = False
for expert_id in range(num_experts):
curr_expert_weight = loaded_weight[expert_id]
success = weight_loader(
param,
curr_expert_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_local_expert = True
return loaded_local_expert
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts
if hasattr(self.config, "num_experts")
else 0,
mapper = self.hf_to_vllm_mapper
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled() and (
_is_shared_expert_fse_compatible(get_current_vllm_config().quant_config)
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
is_fused_expert = False
fused_expert_params_mapping: list[tuple[str, str, int, str]] = []
for param_name, ckpt_name, _, shard_id in fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_up_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="gate_up_proj",
num_experts=1,
):
if shard_id == "w3":
continue
parts = ckpt_name.split(".")
fused_expert_params_mapping.append(
(f"{param_name}weight", f"{parts[0]}.{parts[2]}", 0, shard_id)
if is_fse:
num_routed = getattr(self.config, "num_experts", 0)
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
num_experts = (
self.config.num_experts if hasattr(self.config, "num_experts") else 0
)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
is_fused_expert = True
expert_params_mapping = fused_expert_params_mapping
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name_mapped, self):
continue
if is_fused_expert:
# qwen3.5 no need to transpose
# loaded_weight = loaded_weight.transpose(-1, -2)
if "experts.gate_up_proj" in name:
loaded_weight = loaded_weight.chunk(2, dim=-2)
success_w1 = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight[0],
"w1",
num_experts,
)
success_w3 = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight[1],
"w3",
num_experts,
)
success = success_w1 and success_w3
else:
# down_proj
success = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight,
shard_id,
num_experts,
)
if success:
name = name_mapped
break
else:
# Skip loading extra bias for GPTQ models.
if (
name_mapped.endswith(".bias")
or name_mapped.endswith("_bias")
) and name_mapped not in params_dict:
continue
param = params_dict[name_mapped]
weight_loader = param.weight_loader
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
logger.warning_once(
f"Parameter {name} not found in params_dict, skip loading"
)
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile(
+14 -36
View File
@@ -17,15 +17,12 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.qwen3 import Qwen3DecoderLayer, Qwen3ForCausalLM
from vllm.multimodal.inputs import NestedTensors
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_draft_quant_config,
maybe_prefix,
process_eagle_weight,
@@ -132,6 +129,17 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
}
)
class Qwen3Eagle3Model(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={"midlayer.": "layers.0."},
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
},
)
def __init__(
self,
*,
@@ -256,38 +264,8 @@ class Qwen3Eagle3Model(nn.Module):
return hidden_states, aux_output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
(".qkv_proj", ".q_proj", "q"),
(".qkv_proj", ".k_proj", "k"),
(".qkv_proj", ".v_proj", "v"),
(".gate_up_proj", ".gate_proj", 0),
(".gate_up_proj", ".up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "midlayer." in name:
name = name.replace("midlayer.", "layers.0.")
# Remapping the name FP8 kv-scale or zero point.
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
+37 -154
View File
@@ -23,8 +23,7 @@
# limitations under the License.
"""Inference-only Qwen3MoE model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from collections.abc import Iterable
from itertools import islice
from typing import Any
@@ -43,10 +42,7 @@ from vllm.distributed import (
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
@@ -61,10 +57,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
@@ -75,12 +67,13 @@ from .interfaces import (
SupportsEagle3,
SupportsLoRA,
SupportsPP,
SupportsQuant,
)
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -438,6 +431,20 @@ class Qwen3MoeDecoderLayer(nn.Module):
@support_torch_compile
class Qwen3MoeModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
# .experts.gate_up_proj must be handled by MoERunner.load_weights for EP
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
}
)
def __init__(
self,
*,
@@ -516,152 +523,31 @@ class Qwen3MoeModel(nn.Module, EagleModelMixin):
return hidden_states, aux_hidden_states
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=self.config.num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Skip loading extra parameters for GPTQ/modelopt models.
ignore_suffixes = (
".bias",
"_bias",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
loader = AutoWeightsLoader(
self,
ignore_unexpected_suffixes=[
".bias",
"_bias",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
],
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
for name, loaded_weight in weights:
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra parameters for GPTQ/modelopt models.
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
# Do not modify `name` since the loop may continue here
# Instead, create a new variable
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
# Skip loading extra parameters for GPTQ/modelopt models.
if (
name_mapped.endswith(ignore_suffixes)
and name_mapped not in params_dict
):
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
# available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra parameters for GPTQ/modelopt models.
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3MoeForCausalLM(
nn.Module, SupportsPP, SupportsLoRA, SupportsEagle, SupportsEagle3, MixtureOfExperts
nn.Module,
SupportsPP,
SupportsLoRA,
SupportsEagle,
SupportsEagle3,
MixtureOfExperts,
SupportsQuant,
):
hf_to_vllm_mapper = Qwen3MoeModel.hf_to_vllm_mapper
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -769,6 +655,3 @@ class Qwen3MoeForCausalLM(
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+27 -136
View File
@@ -10,12 +10,7 @@ from torch import nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
CacheConfig,
ModelConfig,
VllmConfig,
get_current_vllm_config,
)
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.distributed import (
get_ep_group,
get_pp_group,
@@ -26,7 +21,6 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.fused_moe import (
FusedMoE,
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_qk_norm_rope import fused_qk_rmsnorm_rope_gate
from vllm.model_executor.layers.layernorm import (
@@ -53,10 +47,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.model_executor.models.qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
@@ -76,8 +66,8 @@ from .interfaces import (
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
@@ -132,7 +122,6 @@ class Qwen3NextSparseMoeBlock(nn.Module):
)
# Load balancing settings.
vllm_config = get_current_vllm_config()
eplb_config = vllm_config.parallel_config.eplb_config
self.enable_eplb = parallel_config.enable_eplb
@@ -544,6 +533,19 @@ class Qwen3NextDecoderLayer(nn.Module):
@support_torch_compile
class Qwen3NextModel(nn.Module, EagleModelMixin):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".mlp.gate_proj": (".mlp.gate_up_proj", 0),
".mlp.up_proj": (".mlp.gate_up_proj", 1),
".shared_expert.gate_proj": (".shared_expert.gate_up_proj", 0),
".shared_expert.up_proj": (".shared_expert.gate_up_proj", 1),
}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -625,123 +627,18 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
return hidden_states, aux_hidden_states
return hidden_states
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
num_experts = getattr(self.config, "num_experts", 0)
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
num_experts += 1
return fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=num_experts,
num_redundant_experts=self.num_redundant_experts,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
num_routed = getattr(self.config, "num_experts", 0)
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
if name.startswith("mtp."):
continue
# Remapping the name of FP8 kv-scale.
if name.endswith("scale"):
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
# FSE: remap shared_expert weights to the fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_routed}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# name = apply_attn_prefix(name, params_dict)
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
logger.warning_once(
f"Parameter {name} not found in params_dict, skip loading"
)
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
mapper = self.hf_to_vllm_mapper
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
# AITER fused-shared-experts: route the shared_expert checkpoint
# weights into the extra fused expert slot. Merge (not mutate) so the
# shared class mapper isn't permanently altered.
num_routed = getattr(self.config, "num_experts", 0)
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
class QwenNextMixtureOfExperts(MixtureOfExperts):
@@ -901,11 +798,5 @@ class Qwen3NextForCausalLM(
return self.logits_processor(self.lm_head, hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
skip_prefixes=["mtp."],
)
loader = AutoWeightsLoader(self, skip_prefixes=["mtp."])
return loader.load_weights(weights)
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
return self.model.get_expert_mapping()
+14 -100
View File
@@ -12,18 +12,15 @@ from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.qwen3_next import (
Qwen3NextDecoderLayer,
Qwen3NextModel,
Qwen3NextRMSNorm,
QwenNextMixtureOfExperts,
)
@@ -32,7 +29,7 @@ from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from .utils import (
AutoWeightsLoader,
is_pp_missing_parameter,
WeightsMapper,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
@@ -44,6 +41,8 @@ KVCache = tuple[torch.Tensor, torch.Tensor]
@support_torch_compile
class Qwen3NextMultiTokenPredictor(nn.Module):
hf_to_vllm_mapper = Qwen3NextModel.hf_to_vllm_mapper
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
@@ -146,101 +145,16 @@ class Qwen3NextMultiTokenPredictor(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Params for weights, fp8 weight scales, fp8 activation scales
# (param_name, weight_name, expert_id, shard_id)
is_fse = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
num_experts = self.config.num_experts
if is_fse:
num_experts += 1
expert_params_mapping = fused_moe_make_expert_params_mapping(
self,
ckpt_gate_proj_name="gate_proj",
ckpt_down_proj_name="down_proj",
ckpt_up_proj_name="up_proj",
num_experts=num_experts,
)
num_routed = self.config.num_experts
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
continue
# FSE: remap shared_expert weights to the fused expert slot
if is_fse and "mlp.shared_expert." in name:
name = name.replace(
"mlp.shared_expert.",
f"mlp.experts.{num_routed}.",
)
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
# Skip loading extra bias for GPTQ models.
if (
name.endswith(".bias") or name.endswith("_bias")
) and name not in params_dict:
continue
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(
param,
loaded_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
)
break
else:
# Skip loading extra bias for GPTQ models.
if name.endswith(".bias") and name not in params_dict:
continue
if is_pp_missing_parameter(name, self):
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
mapper = self.hf_to_vllm_mapper
if rocm_aiter_ops.is_fusion_moe_shared_experts_enabled():
# AITER fused-shared-experts: route the shared_expert checkpoint
# weights into the extra fused expert slot.
num_routed = self.config.num_experts
mapper = mapper | WeightsMapper(
orig_to_new_substr={"mlp.shared_expert.": f"mlp.experts.{num_routed}."}
)
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=mapper)
@support_torch_compile
@@ -63,7 +63,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.qwen2_audio import Qwen2AudioProcessingInfo
from vllm.multimodal import MULTIMODAL_REGISTRY
@@ -319,6 +318,14 @@ class Qwen3OmniMoeAudioEncoderLayer(nn.Module):
class Qwen3OmniMoeAudioEncoder(nn.Module):
"""vLLM-native Qwen3-Omni Audio Encoder."""
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".self_attn.q_proj.": (".self_attn.qkv.", "q"),
".self_attn.k_proj.": (".self_attn.qkv.", "k"),
".self_attn.v_proj.": (".self_attn.qkv.", "v"),
}
)
def __init__(
self,
config: Qwen3OmniMoeAudioEncoderConfig,
@@ -531,35 +538,8 @@ class Qwen3OmniMoeAudioEncoder(nn.Module):
return lengths
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with mapping from HuggingFace format."""
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("self_attn.qkv.", "self_attn.q_proj.", "q"),
("self_attn.qkv.", "self_attn.k_proj.", "k"),
("self_attn.qkv.", "self_attn.v_proj.", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict.get(name)
if param is not None:
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3_VisionPatchEmbed(nn.Module):
@@ -737,6 +717,14 @@ class Qwen3_VisionPatchMerger(nn.Module):
class Qwen3Omni_VisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".attn.q.": (".attn.qkv.", "q"),
".attn.k.": (".attn.qkv.", "k"),
".attn.v.": (".attn.qkv.", "v"),
}
)
def __init__(
self,
vision_config,
@@ -1044,31 +1032,8 @@ class Qwen3Omni_VisionTransformer(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("attn.qkv.", "attn.q.", "q"),
("attn.qkv.", "attn.k.", "k"),
("attn.qkv.", "attn.v.", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@support_torch_compile(
+10 -26
View File
@@ -67,7 +67,6 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.evs import (
@@ -518,6 +517,14 @@ class Qwen3_VisionPatchMerger(nn.Module):
class Qwen3_VisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
"attn.q.": ("attn.qkv.", "q"),
"attn.k.": ("attn.qkv.", "k"),
"attn.v.": ("attn.qkv.", "v"),
}
)
def __init__(
self,
vision_config: Qwen3VLVisionConfig,
@@ -834,31 +841,8 @@ class Qwen3_VisionTransformer(nn.Module):
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("attn.qkv.", "attn.q.", "q"),
("attn.qkv.", "attn.k.", "k"),
("attn.qkv.", "attn.v.", "v"),
]
params_dict = dict(self.named_parameters(remove_duplicate=False))
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
+1 -203
View File
@@ -24,8 +24,6 @@
# limitations under the License.
"""Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights."""
import typing
from collections.abc import Callable, Iterable
from itertools import islice
import torch
@@ -39,10 +37,6 @@ from vllm.distributed import get_pp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
@@ -60,7 +54,7 @@ from .qwen3_vl import (
Qwen3VLMultiModalProcessor,
Qwen3VLProcessingInfo,
)
from .utils import is_pp_missing_parameter, maybe_prefix
from .utils import maybe_prefix
logger = init_logger(__name__)
@@ -135,202 +129,6 @@ class Qwen3MoeLLMModel(Qwen3MoeModel):
return hidden_states, aux_hidden_states
return hidden_states
def load_fused_expert_weights(
self,
name: str,
params_dict: dict,
loaded_weight: torch.Tensor,
shard_id: str,
num_experts: int,
) -> bool:
param = params_dict[name]
weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
loaded_local_expert = False
for expert_id in range(num_experts):
curr_expert_weight = loaded_weight[expert_id]
success = weight_loader(
param,
curr_expert_weight,
name,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
loaded_local_expert = True
return loaded_local_expert
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1),
]
# Skip loading extra parameters for GPTQ/modelopt models.
ignore_suffixes = (
".bias",
"_bias",
".weight_scale",
"_weight_scale",
".input_scale",
"_input_scale",
)
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
expert_params_mapping = self.get_expert_mapping()
is_fused_expert = False
base_layer = (
"base_layer." if any(".base_layer." in name for name in params_dict) else ""
)
fused_expert_params_mapping = [
(
f"experts.routed_experts.{base_layer}w13_weight",
"experts.gate_up_proj",
0,
"w1",
),
(
f"experts.routed_experts.{base_layer}w2_weight",
"experts.down_proj",
0,
"w2",
),
]
num_experts = self.config.num_experts
for name, loaded_weight in weights:
if "scale" in name or "zero_point" in name:
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
for param_name, weight_name, shard_id in stacked_params_mapping:
if "experts.gate_up_proj" in name or "experts.down_proj" in name:
is_fused_expert = True
expert_params_mapping = fused_expert_params_mapping
# Skip non-stacked layers and experts (experts handled below).
if weight_name not in name:
continue
# We have mlp.experts[0].gate_proj in the checkpoint.
# Since we handle the experts below in expert_params_mapping,
# we need to skip here BEFORE we update the name, otherwise
# name will be updated to mlp.experts[0].gate_up_proj, which
# will then be updated below in expert_params_mapping
# for mlp.experts[0].gate_gate_up_proj, which breaks load.
if "mlp.experts" in name:
continue
name = name.replace(weight_name, param_name)
# Skip loading extra parameters for GPTQ/modelopt models.
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if weight_loader == default_weight_loader:
weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
break
else:
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
if weight_name not in name:
continue
# Anyway, this is an expert weight and should not be
# attempted to load as other weights later
is_expert_weight = True
name_mapped = name.replace(weight_name, param_name)
if is_pp_missing_parameter(name_mapped, self):
continue
if is_fused_expert:
loaded_weight = loaded_weight.transpose(-1, -2) # no bias
if "experts.gate_up_proj" in name:
loaded_weight = loaded_weight.chunk(2, dim=-2)
success_w1 = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight[0],
"w1",
num_experts,
)
success_w3 = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight[1],
"w3",
num_experts,
)
success = success_w1 and success_w3
else:
# down_proj
success = self.load_fused_expert_weights(
name_mapped,
params_dict,
loaded_weight,
shard_id,
num_experts,
)
else:
# Skip loading extra parameters for GPTQ/modelopt models
if (
name_mapped.endswith(ignore_suffixes)
and name_mapped not in params_dict
):
continue
param = params_dict[name_mapped]
# We should ask the weight loader to return success or
# not here since otherwise we may skip experts with
# other available replicas.
weight_loader = typing.cast(
Callable[..., bool], param.weight_loader
)
success = weight_loader(
param,
loaded_weight,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
return_success=True,
)
if success:
name = name_mapped
break
else:
if is_expert_weight:
# We've checked that this is an expert weight
# However it's not mapped locally to this rank
# So we simply skip it
continue
# Skip loading extra parameters for GPTQ/modelopt models.
if name.endswith(ignore_suffixes) and name not in params_dict:
continue
# Skip layers on other devices.
if is_pp_missing_parameter(name, self):
continue
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+34 -126
View File
@@ -33,10 +33,6 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.pooler import DispatchPooler
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
maybe_remap_kv_scale_name,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
MultiModalFieldConfig,
@@ -62,7 +58,7 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
from .interfaces_base import default_pooling_type
from .utils import AutoWeightsLoader, maybe_prefix
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
from .vision import (
VisionEncoderInfo,
VisionFeatureSelectStrategy,
@@ -571,6 +567,14 @@ class SiglipEncoder(nn.Module):
class SiglipTextTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: SiglipTextConfig,
@@ -615,30 +619,8 @@ class SiglipTextTransformer(nn.Module):
return last_hidden_state
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class SiglipMultiheadAttentionPoolingHead(nn.Module):
@@ -682,6 +664,14 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
class SiglipVisionTransformer(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: SiglipVisionConfig,
@@ -806,46 +796,25 @@ class SiglipVisionTransformer(nn.Module):
return encoder_outputs
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
skip_prefixes = []
if self.post_layernorm is None:
skip_prefixes.append("post_layernorm.")
if self.head is None:
skip_prefixes.append("head.")
loader = AutoWeightsLoader(self, skip_prefixes=skip_prefixes)
layer_count = len(self.encoder.layers)
for name, loaded_weight in weights:
# post_layernorm is not needed in SiglipVisionTransformer
if name.startswith("post_layernorm") and self.post_layernorm is None:
continue
# if the model configuration is not going to use
# the pooling head for inference, don't load its weights
if self.head is None and name.startswith("head"):
continue
# omit layers when num_hidden_layers_override is set
if name.startswith("encoder.layers"):
layer_idx = int(name.split(".")[2])
if layer_idx >= layer_count:
def _filter(ws):
for name, w in ws:
# omit layers when num_hidden_layers_override is set
if name.startswith("encoder.layers.") and (
int(name.split(".")[2]) >= layer_count
):
continue
yield name, w
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
return loader.load_weights(_filter(weights), mapper=self.hf_to_vllm_mapper)
class SiglipVisionModel(nn.Module):
@@ -896,67 +865,6 @@ class SiglipVisionModel(nn.Module):
feature_select_strategy=feature_select_strategy,
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
layer_count = len(self.vision_model.encoder.layers)
for name, loaded_weight in weights:
# post_layernorm is optional in SiglipVisionModel
if (
name.startswith("vision_model.post_layernorm")
and self.vision_model.post_layernorm is None
):
continue
# if the model configuration is not going to use
# the pooling head for inference, don't load its weights
if self.vision_model.head is None and name.startswith("vision_model.head"):
continue
# omit layers when num_hidden_layers_override is set
if name.startswith("vision_model.encoder.layers"):
layer_idx = int(name.split(".")[3])
if layer_idx >= layer_count:
continue
# Check if this is a scale parameter that needs remapping first
if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")):
# Try to remap the scale name first
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is not None and remapped_name in params_dict:
# Successfully remapped, use the remapped name
param = params_dict[remapped_name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader(param, loaded_weight)
loaded_params.add(remapped_name)
continue
# If remapping failed, continue with normal processing
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
# Adapted from: https://github.com/huggingface/transformers/blob/v4.54.1/src/transformers/models/siglip/modeling_siglip.py#L200
class SiglipTextEmbeddings(nn.Module):
+11 -27
View File
@@ -26,10 +26,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding.common import (
ApplyRotaryEmb,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.platforms import current_platform
from .utils import maybe_prefix
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
from .vision import is_vit_use_data_parallel
@@ -588,6 +587,14 @@ class Siglip2VisionTransformer(nn.Module):
class Siglip2NavitModel(torch.nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
def __init__(
self,
config: Siglip2VisionConfig,
@@ -613,28 +620,5 @@ class Siglip2NavitModel(torch.nn.Module):
)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(name)
return loaded_params
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+15 -47
View File
@@ -16,7 +16,6 @@
# limitations under the License.
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
from collections.abc import Iterable
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any
@@ -82,11 +81,6 @@ class TransformersMoERunner(MoERunner):
) -> torch.Tensor:
return super().forward(hidden_states, topk_weights)
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
return self.routed_experts.load_weights(weights)
def _transformers_moe_forward(
hidden_states: torch.Tensor,
@@ -119,6 +113,20 @@ direct_register_custom_op(
)
class TransformersRoutedExperts(RoutedExperts):
def get_expert_mapping(
self, include_fused: bool = False
) -> list[tuple[str, str, int, str]]:
common_names = ("gate_proj", "down_proj", "up_proj")
common_map = super().get_expert_mapping(*common_names, include_fused)
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
if not include_fused:
return common_map + mixtral_map
common_fused, common_unfused = common_map[:3], common_map[3:]
mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:]
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
class MoEMixin(MixtureOfExperts):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
self.check_version("5.0.0", "MoE models support")
@@ -140,43 +148,6 @@ class MoEMixin(MixtureOfExperts):
mlp.n_redundant_experts = self.num_redundant_experts
mlp.experts.update_expert_map()
def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
"""
Params for weights, fp8 weight scales, fp8 activation scales
(param_name, weight_name, expert_id, shard_id)
"""
# Models saved with fused experts. These are checkpoints released:
# - After Transformers v5
# - Before Transformers v5, but re-saved with save_original_format=False
# In the fused experts case, we repurpose the expert_id as shard_idx for
# deconcatenating w1 and w3 in FusedMoE.load_weights.
expert_mapping = [
("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
("experts.w13_weight", "experts.gate_up_proj", 1, "w3"),
("experts.w2_weight", "experts.down_proj", 0, "w2"),
]
# Models saved with ModuleList experts
ckpt_names = [
# (ckpt_gate_proj_name, ckpt_down_proj_name, ckpt_up_proj_name)
("gate_proj", "down_proj", "up_proj"), # Most common MoE style
("w1", "w2", "w3"), # Granite, Mixtral, Phi MoE style
]
num_experts = self.model_config.get_num_experts()
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
for gate_proj, down_proj, up_proj in ckpt_names:
expert_mapping.extend(
RoutedExperts.make_expert_params_mapping(
self,
ckpt_gate_proj_name=gate_proj,
ckpt_down_proj_name=down_proj,
ckpt_up_proj_name=up_proj,
num_experts=num_experts,
num_redundant_experts=num_redundant_experts,
routed_experts_prefix="",
)
)
return expert_mapping
def recursive_replace(self):
"""Initialize the MoE layers."""
text_config = self.text_config
@@ -218,9 +189,6 @@ class MoEMixin(MixtureOfExperts):
if "gptoss" in wrapped_arch:
activation = "swigluoai"
# Expert mapping for `AutoWeightsLoader`
expert_mapping = self.get_expert_mapping()
# Expert parallel load balancing kwargs
enable_eplb = self.parallel_config.enable_eplb
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
@@ -308,12 +276,12 @@ class MoEMixin(MixtureOfExperts):
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
expert_mapping=expert_mapping,
custom_routing_function=partial(
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersMoERunner,
routed_experts_cls=TransformersRoutedExperts,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
+6 -30
View File
@@ -43,7 +43,6 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm.sequence import IntermediateTensors
from .interfaces import HasInnerState, IsHybrid, SupportsMambaPrefixCaching
@@ -804,34 +803,6 @@ class Zamba2Model(nn.Module):
hidden_states = self.final_layernorm(hidden_states)
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [
# (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"),
("qkv_proj", "v_proj", "v"),
]
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
for chkpt_weight_name, loaded_weight in weights:
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in chkpt_weight_name:
continue
chkpt_weight_name = chkpt_weight_name.replace(weight_name, param_name)
param = params_dict[chkpt_weight_name]
weight_loader = param.weight_loader
weight_loader(param, loaded_weight, shard_id)
break
else:
if chkpt_weight_name not in params_dict:
continue
param = params_dict[chkpt_weight_name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight)
loaded_params.add(chkpt_weight_name)
return loaded_params
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixCaching):
"""Zamba2 model with causal language modeling head.
@@ -849,7 +820,12 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsMambaPrefixC
"A_log": "A",
"0.weight": "A.weight",
"1.weight": "B.weight",
}
},
orig_to_new_stacked={
".self_attn.q_proj": (".self_attn.qkv_proj", "q"),
".self_attn.k_proj": (".self_attn.qkv_proj", "k"),
".self_attn.v_proj": (".self_attn.qkv_proj", "v"),
},
)
@classmethod
+10 -9
View File
@@ -124,15 +124,16 @@ def get_packed_modules_mapping(model: torch.nn.Module) -> dict[str, list[str]]:
def get_moe_expert_mapping(
model: torch.nn.Module,
) -> list[tuple[str, str, int, str]]:
if parent_map := getattr(model, "get_expert_mapping", None):
return parent_map()
else:
# We only check main components instead of whole model submodules
for child in model.children():
child_map = getattr(child, "get_expert_mapping", None)
if child_map is not None:
return child_map()
return []
"""Get the expert mapping from a model.
It will be retrieved from the first module that has a `get_expert_mapping` method.
If the model manually implements `get_expert_mapping`, it will be used.
Otherwise, it will use the first RoutedExperts layer."""
for _, module in model.named_modules():
get_mapping = getattr(module, "get_expert_mapping", None)
if get_mapping is not None:
return get_mapping()
raise ValueError("No module in the model has a `get_expert_mapping` method.")
def maybe_disable_graph_partition(current_backend: str) -> dict[str, bool]:
+1 -2
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import typing
from collections.abc import Callable, Iterable, MutableSequence, Sequence
from collections.abc import Callable, Iterable
from itertools import islice
import regex as re
@@ -1365,7 +1365,6 @@ class DeepseekV4ForCausalLM(nn.Module, SupportsPP, DeepseekV4MixtureOfExperts):
self.set_moe_parameters()
def set_moe_parameters(self) -> None:
self.expert_weights: MutableSequence[Sequence[torch.Tensor]] = []
self.num_expert_groups = getattr(self.config, "n_group", 1)
self.num_moe_layers = self.config.num_hidden_layers
self.moe_layers: list[nn.Module] = []